import { AsyncLocalStorage } from "node:async_hooks"; import { createHash, createHmac, generateKeyPairSync, randomUUID, } from "node:crypto"; import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync, } from "node:fs"; import { createServer, type Server } from "node:http"; import type { AddressInfo } from "node:net"; import os from "node:os"; import path from "node:path"; import { Readable } from "node:stream"; import { crc32 } from "node:zlib"; import express from "express"; import sharp from "sharp"; import request from "supertest"; import { and, asc, desc, eq, inArray, isNotNull, like, or, sql, } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, agentWakeupRequests, activityLog, assets, authUsers, chatActions, chatConversations, chatDeliveries, chatDiscordCommandOwners, chatEndpointLeases, chatEndpointResources, chatEndpoints, chatExternalPrincipals, chatIdentityLinks, chatMessageLinks, chatPublications, chatSdkState, chatTeamsFileTransfers, completionContracts, companySecretBindings, companySecrets, companies, companyMemberships, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issueAttachments, issueWorkProducts, issueQuestionResponseDeliveries, issueRecoveryActions, issueThreadInteractions, issues, nativeRunResults, nativeRunFinalizations, environmentLeases, principalPermissionGrants, toolConnections, } from "@paperclipai/db"; import type { ChatProvider } from "@paperclipai/shared"; import { isPaperclipExternalChatTurn } from "@paperclipai/adapter-utils/server-utils"; import type { Attachment, Author, Message, Thread } from "chat"; import { errorHandler } from "../middleware/index.js"; import { issueRoutes } from "../routes/issues.js"; import { deliverReconciledExecutions } from "../services/execution-recovery-resolution.js"; import { unadmittedChatWakeupCondition, authorizeFailedChatRunRetryWake, } from "../services/durable-chat-wakeup.js"; import { createChatWebhookDiagnostics, type ChatWebhookDiagnosticEvent, } from "../services/chat-webhook-diagnostics.js"; import { chatChannelRoutes, chatWebhookRoutes, } from "../routes/chat-channels.js"; import { chatChannelService, type ChatChannelServiceOptions, type ChatChannelService, } from "../services/chat-channels.js"; import type { CreateChatSdkEndpointRuntimeOptions, ChatSdkMessageTrigger, ChatSdkRuntime, } from "../services/chat-sdk-runtime.js"; import { createChatSdkEndpointRuntime } from "../services/chat-sdk-runtime.js"; import type { TelegramDraftControl } from "../services/chat-telegram-draft-stop.js"; // Opt-in private physical candidate; normal CI uses the staged pinned package. vi.mock("@chat-adapter/telegram", async (importOriginal) => { const candidate = process.env.PAPERCLIP_TELEGRAM_STOP_ADAPTER_MODULE; return candidate ? import(/* @vite-ignore */ candidate) : importOriginal(); }); import { createDiscordAdapter } from "@chat-adapter/discord"; import { createTeamsAdapter } from "@chat-adapter/teams"; import { bindTeamsPersonalRecipient, parseTeamsPersonalRecipient, parseTeamsPersonalRecipientBinding, type TeamsPersonalRecipientAdmission, } from "../services/chat-teams-personal-recipient.js"; import * as discordQuestionForms from "../services/chat-discord-question-forms.js"; import { issueService } from "../services/issues.js"; import { getExternalChannelBindingSummary } from "../services/chat-channel-binding.js"; import { PaperclipRunnerToolAuthority } from "../services/native-runtime/paperclip-runner-tool-authority.js"; import { NativeChatAttachmentReadScope } from "../services/native-runtime/chat-attachment-read.js"; import { logActivity } from "../services/activity-log.js"; import { subscribeCompanyLiveEvents } from "../services/live-events.js"; import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; import { questionResponseDeliveryService } from "../services/question-response-delivery.js"; import * as chatQuestionForms from "../services/chat-question-forms.js"; import type { StorageService } from "../storage/types.js"; import { TELEGRAM_CALLBACK_DATA_LIMIT_BYTES, telegramChatSdkCallbackData, } from "../services/chat-interaction-publications.js"; import { enqueueChatRunMilestones, resolveChatRunPresentationAuthorizationReason, } from "../services/chat-run-publications.js"; import { heartbeatService, resolveExternalChatWakeProvider, } from "../services/heartbeat.js"; import { registerServerAdapter, unregisterServerAdapter, } from "../adapters/index.js"; import { projectSafeChatPublicationText } from "../services/chat-publication-projection.js"; import { nativePublicationTextFits, renderPublicationTransportText, } from "../services/chat-publication-text-parts.js"; import { MAX_ATTACHMENT_BYTES, formatAttachmentSize, } from "../attachment-types.js"; import { TELEGRAM_VIDEO_NOTE_MP4 } from "./fixtures/telegram-video-note.js"; import { TELEGRAM_VOICE_OGG } from "./fixtures/telegram-voice.js"; import { GITHUB_ATTACHMENT_BATCH_TIMEOUT_MS, githubAttachmentLocator, githubPublicAttachmentsFromMessage, rehydrateGitHubPublicAttachment, } from "../services/chat-github-attachments.js"; import * as attachmentEgress from "../services/remote-http-fetch.js"; import { logger as chatAttachmentLogger } from "../middleware/logger.js"; import type { PrpStructuredRunResult, PrpTerminalState, } from "../vendor/paperclip-runner/index.js"; import { finalizeNativeRun, repairCommittedNativeChatResponse, } from "../services/native-runtime/native-run-finalizer.js"; import { NativeRunCoordinatorStore } from "../services/native-runtime/native-run-coordinator-store.js"; import { reconcileNativeFinalizations } from "../services/native-runtime/native-finalization-reconciler.js"; import { PaperclipControlPlanePort } from "../services/native-runtime/paperclip-control-plane-port.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; const externalTestDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL; const embeddedPostgresSupport = externalTestDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe.sequential : describe.skip; if (!embeddedPostgresSupport.supported) { console.warn( `Skipping chat-channel integration tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, ); } type TestDb = ReturnType; class FakeEndpointRuntime { readonly initialize = vi.fn(async () => { await this.initializeHook?.(this.options.endpointId); }); readonly shutdown = vi.fn(async () => undefined); readonly posts: Array<{ threadId: string; text: string; attachments?: unknown[]; chunks?: string[]; files?: unknown[]; }> = []; readonly edits: Array<{ threadId: string; messageId: string; text: string; }> = []; readonly editAttempts: Array<{ threadId: string; messageId: string; }> = []; readonly reactions: Array<{ threadId: string; messageId: string; emoji: string; }> = []; readonly reactionErrors: Error[] = []; readonly removedReactions: Array<{ threadId: string; messageId: string; emoji: string; }> = []; readonly removeReactionErrors: Error[] = []; readonly rehydratedAttachmentDescriptors: unknown[] = []; readonly postResultIds: string[] = []; readonly slackFileReceiptLookups: Array<{ fileIds: string[]; threadId: string; }> = []; readonly slackFileReceiptResultIds: Array = []; slackFileReceiptCaptureRepeats = 1; slackFilePublicationAttempts = 0; slackFilePostAcceptanceError: Error | null = null; slackFilePostAcceptanceHook: (() => Promise) | undefined; slackFileReceiptHook: (() => Promise) | undefined; readonly ensuredDiscordRootThreads: Array<{ channelId: string; content: string; messageId: string; }> = []; readonly recordedMicrosoftTeamsRoutes: Array<{ threadId: string; serviceUrl: unknown; }> = []; private nextPostId = 0; postError: Error | null = null; editError: Error | null = null; postHook: (() => Promise) | undefined; webhookHook: ((request: Request) => Promise) | undefined; webhookRequest: Request | null = null; webhookResponse = new Response("accepted", { status: 202, headers: { "x-chat-test": "accepted" }, }); constructor( private readonly options: CreateChatSdkEndpointRuntimeOptions, private readonly attachmentBodies: Map, private readonly initializeHook?: (endpointId: string) => Promise, ) {} get provider() { return this.options.providerConfig.provider; } async handleWebhook(request: Request) { this.webhookRequest = request; await this.webhookHook?.(request); return this.webhookResponse; } async ensureDiscordRootThread(input: { channelId: string; content: string; messageId: string; }) { this.ensuredDiscordRootThreads.push(input); } async recordMicrosoftTeamsRoute(threadId: string, serviceUrl: unknown) { this.recordedMicrosoftTeamsRoutes.push({ threadId, serviceUrl }); } async postSlackFilePublication( threadId: string, message: unknown, onUploadAccepted: (receipt: { version: 1; channelId: string; fileIds: string[]; threadTs: string | null; }) => Promise, ) { this.slackFilePublicationAttempts += 1; await this.postHook?.(); if (this.postError) throw this.postError; const parts = threadId.split(":"); const files = message && typeof message === "object" && "files" in message && Array.isArray((message as { files?: unknown }).files) ? (message as { files: unknown[] }).files : []; for ( let attempt = 0; attempt < this.slackFileReceiptCaptureRepeats; attempt += 1 ) { await onUploadAccepted({ version: 1, channelId: parts[1] ?? "", fileIds: files.map((_, index) => `FTEST${index + 1}`), threadTs: parts[2] || null, }); } await this.slackFilePostAcceptanceHook?.(); if (this.slackFilePostAcceptanceError) { throw this.slackFilePostAcceptanceError; } const postHook = this.postHook; this.postHook = undefined; try { return await this.thread(threadId).post(message); } finally { this.postHook = postHook; } } async resolveSlackFileUploadReceipt(threadId: string, fileIds: string[]) { this.slackFileReceiptLookups.push({ fileIds: [...fileIds], threadId, }); await this.slackFileReceiptHook?.(); return this.slackFileReceiptResultIds.shift() ?? null; } acceptsProviderScope(raw: unknown) { if (this.options.providerConfig.provider !== "microsoft-teams") return true; const expected = this.options.providerConfig.credentials.appTenantId; if (!expected || !raw || typeof raw !== "object") return Boolean(!expected); const payload = raw as { conversation?: { tenantId?: unknown }; channelData?: { tenant?: { id?: unknown } }; recipient?: { isTargeted?: unknown }; }; if (payload.recipient?.isTargeted === true) return false; const tenantIds = [ payload.conversation?.tenantId, payload.channelData?.tenant?.id, ].filter((value): value is string => typeof value === "string"); return ( tenantIds.length > 0 && tenantIds.every((value) => value === expected) ); } async applySlackReceiptReaction(input: { operation: "add" | "remove"; threadId: string; messageId: string; reaction: "eyes"; }) { const adapter = this.thread(input.threadId).adapter; if (input.operation === "add") await adapter.addReaction( input.threadId, input.messageId, input.reaction, ); else await adapter.removeReaction( input.threadId, input.messageId, input.reaction, ); } async applyGitHubReceiptReaction( input: Parameters[0], assertCurrent: () => Promise, ) { await assertCurrent(); await this.applySlackReceiptReaction(input); return input.githubReceipt ?? { botUserId: "9001", reactionId: "880012" }; } async streamTelegramDraft( threadId: string, stream: AsyncIterable, control: TelegramDraftControl, ) { if (!(await control.beforeDraft()) || !(await control.beforeFinal())) return { paperclipDraftStopped: true as const }; return this.thread(threadId).post(stream); } thread(threadId: string) { const parts = threadId.split(":"); const channelId = this.options.providerConfig.provider === "discord" ? (parts[2] ?? threadId) : (parts[1] ?? threadId); const isTelegramDirectMessage = this.options.providerConfig.provider === "telegram" && /^\d+$/.test(channelId); return { id: threadId, channelId, isDM: isTelegramDirectMessage || /^D[A-Z0-9-]*$/i.test(channelId), channel: { id: channelId, name: "command-thread", }, adapter: { addReaction: async ( reactionThreadId: string, messageId: string, emoji: string, ) => { const error = this.reactionErrors.shift(); if (error) throw error; this.reactions.push({ threadId: reactionThreadId, messageId, emoji, }); }, removeReaction: async ( reactionThreadId: string, messageId: string, emoji: string, ) => { const error = this.removeReactionErrors.shift(); if (error) throw error; this.removedReactions.push({ threadId: reactionThreadId, messageId, emoji, }); }, editMessage: async ( editedThreadId: string, messageId: string, editedMessage: unknown, ) => { this.editAttempts.push({ threadId: editedThreadId, messageId }); if (this.editError) throw this.editError; if (this.postError) throw this.postError; const text = editedMessage && typeof editedMessage === "object" && "markdown" in editedMessage ? String((editedMessage as { markdown: unknown }).markdown) : JSON.stringify(editedMessage); this.edits.push({ threadId: editedThreadId, messageId, text, }); return { id: messageId, threadId: editedThreadId }; }, }, startTyping: async () => undefined, subscribe: async () => undefined, post: async (message: unknown) => { await this.postHook?.(); if (this.postError) throw this.postError; let text: string; let attachments: unknown[] | undefined; let chunks: string[] | undefined; let files: unknown[] | undefined; if (typeof message === "string") text = message; else if ( message && typeof message === "object" && Symbol.asyncIterator in message ) { chunks = []; for await (const chunk of message as AsyncIterable) chunks.push(String(chunk)); text = chunks.join(""); } else if ( message && typeof message === "object" && "markdown" in message ) { text = String((message as { markdown: unknown }).markdown); } else text = JSON.stringify(message); if ( message && typeof message === "object" && "attachments" in message && Array.isArray((message as { attachments?: unknown }).attachments) ) { attachments = (message as { attachments: unknown[] }).attachments; } if ( message && typeof message === "object" && "files" in message && Array.isArray((message as { files?: unknown }).files) ) { files = (message as { files: unknown[] }).files; } this.posts.push({ threadId, text, ...(attachments ? { attachments } : {}), ...(chunks ? { chunks } : {}), ...(files ? { files } : {}), }); this.nextPostId += 1; return { id: this.postResultIds.shift() ?? `outbound-${this.nextPostId}`, threadId, }; }, }; } attachmentRecoveryDescriptor(attachment: Attachment) { if (this.options.providerConfig.provider === "github") { const locator = githubAttachmentLocator(attachment); return locator ? { version: 1, provider: "github", attachment: { type: attachment.type, name: attachment.name }, locator, } : null; } const recoveryKey = attachment.fetchMetadata?.testRecoveryKey; if (typeof recoveryKey !== "string") return null; return { version: 1, provider: this.options.providerConfig.provider, attachment: { type: attachment.type, name: attachment.name, mimeType: attachment.mimeType, size: attachment.size, }, locator: { kind: "test_attachment", recoveryKey }, }; } parseTelegramCommandMessage(raw: unknown): Message | null { if ( this.options.providerConfig.provider !== "telegram" || !raw || typeof raw !== "object" ) return null; const document = (raw as { document?: Record }).document; const recoveryKey = typeof document?.file_id === "string" ? document.file_id : null; if (!recoveryKey) return null; return makeMessage({ id: `telegram-command:${recoveryKey}`, text: "", attachments: [ { type: "file", name: typeof document.file_name === "string" ? document.file_name : undefined, mimeType: typeof document.mime_type === "string" ? document.mime_type : undefined, size: typeof document.file_size === "number" ? document.file_size : undefined, fetchMetadata: { testRecoveryKey: recoveryKey }, } as Attachment, ], }); } parseMicrosoftTeamsMessage(raw: unknown): Message | null { if ( this.options.providerConfig.provider !== "microsoft-teams" || !raw || typeof raw !== "object" ) return null; const activity = raw as { conversation?: { conversationType?: unknown; id?: unknown }; from?: { aadObjectId?: unknown; id?: unknown; name?: unknown }; id?: unknown; serviceUrl?: unknown; text?: unknown; }; if ( typeof activity.id !== "string" || typeof activity.conversation?.id !== "string" || typeof activity.serviceUrl !== "string" ) return null; const conversationType = activity.conversation.conversationType; const legacyIsDM = !activity.conversation.id.startsWith("19:"); const explicitIsDM = conversationType === "personal"; const includeConversationType = (conversationType === "channel" || conversationType === "groupChat" || conversationType === "personal") && explicitIsDM !== legacyIsDM; const threadId = [ "teams", Buffer.from(activity.conversation.id).toString("base64url"), ...(includeConversationType ? [conversationType] : []), ].join(":"); const userId = typeof activity.from?.id === "string" ? activity.from.id : "unknown"; const userName = typeof activity.from?.name === "string" ? activity.from.name : userId; return { ...makeMessage({ id: activity.id, raw, text: typeof activity.text === "string" ? activity.text : "", userId, userName, }), threadId, } as Message; } rehydrateAttachment( descriptor: unknown, source?: { threadId: string; messageId: string }, ): Attachment | null { this.rehydratedAttachmentDescriptors.push(descriptor); if (!descriptor || typeof descriptor !== "object") return null; const value = descriptor as { version?: unknown; provider?: unknown; attachment?: Attachment; locator?: { kind?: unknown; recoveryKey?: unknown }; }; if ( this.options.providerConfig.provider === "github" && value.provider === "github" && value.version === 1 && source ) { return rehydrateGitHubPublicAttachment(value.locator, source); } if ( value.version !== 1 || value.provider !== this.options.providerConfig.provider || value.locator?.kind !== "test_attachment" || typeof value.locator.recoveryKey !== "string" || !value.attachment ) { return null; } const body = this.attachmentBodies.get(value.locator.recoveryKey); if (!body) return null; return { ...value.attachment, fetchData: async () => body, fetchMetadata: { testRecoveryKey: value.locator.recoveryKey }, } as Attachment; } async resolveGitHubAttachmentComment( _request: { url: string; accept: string }, _signal: AbortSignal, ): Promise { return null; } } class FakeChatSdkRuntime { readonly endpoints = new Map(); readonly configurations = new Map< string, CreateChatSdkEndpointRuntimeOptions >(); initializeHook: ((endpointId: string) => Promise) | undefined; replaceCount = 0; constructor(readonly attachmentBodies: Map = new Map()) {} get(endpointId: string) { return this.endpoints.get(endpointId) ?? null; } async replaceEndpoint(options: CreateChatSdkEndpointRuntimeOptions) { this.replaceCount += 1; this.configurations.set(options.endpointId, options); const endpoint = new FakeEndpointRuntime( options, this.attachmentBodies, this.initializeHook, ); this.endpoints.set(options.endpointId, endpoint); return endpoint; } async removeEndpoint(endpointId: string) { const endpoint = this.endpoints.get(endpointId); if (!endpoint) return false; this.endpoints.delete(endpointId); await endpoint.shutdown(); return true; } async shutdown() { await Promise.all( [...this.endpoints.values()].map(async (endpoint) => endpoint.shutdown()), ); this.endpoints.clear(); } } const TEST_SLACK_BOT_SCOPES = "app_mentions:read,assistant:write,channels:history,channels:read,chat:write,commands,files:read,files:write,groups:history,groups:read,im:history,im:read,mpim:history,mpim:read,reactions:read,reactions:write,users:read"; let discordApplicationSequence = 0n; function uniqueDiscordApplicationId() { discordApplicationSequence += 1n; return (123_456_789_012_345_678n + discordApplicationSequence).toString(); } function fakeSlackFetch(botId = `U-BOT-${randomUUID()}`) { return (input: string | URL | Request) => { const url = String(input); if (url === "https://slack.com/api/auth.test") { return Promise.resolve( new Response( JSON.stringify({ ok: true, team_id: "T-PAPERCLIP", team: "Paperclip Test", user_id: botId, user: `maya-${botId.slice(-8)}`, }), { status: 200, headers: { "content-type": "application/json", "x-oauth-scopes": TEST_SLACK_BOT_SCOPES, }, }, ), ); } if (url.startsWith("https://slack.com/api/conversations.list")) { return Promise.resolve( new Response( JSON.stringify({ ok: true, channels: [], response_metadata: { next_cursor: "" }, }), { status: 200, headers: { "content-type": "application/json" } }, ), ); } if (url.startsWith("https://slack.com/api/conversations.info")) { const channelId = new URL(url).searchParams.get("channel"); return Promise.resolve( new Response( JSON.stringify({ ok: true, channel: { id: channelId, name: channelId?.toLowerCase(), is_member: true, is_archived: false, }, }), { status: 200, headers: { "content-type": "application/json" } }, ), ); } if (url === "https://slack.com/api/agents.sessions.setStatus") { return Promise.resolve(Response.json({ ok: true })); } throw new Error(`Unexpected provider request: ${url}`); }; } function fakeDiscordFetch( applicationId: string, guildId = "1457808928258658549", ) { return async (input: string | URL | Request) => { const path = new URL(String(input)).pathname; const responses: Record = { "/api/v10/users/@me": { id: applicationId, username: `maya-${applicationId.slice(-8)}`, global_name: "Maya", bot: true, }, "/api/v10/oauth2/applications/@me": { id: applicationId, name: "Maya", flags: 1 << 18, }, [`/api/v10/guilds/${guildId}`]: { id: guildId, name: "Clawd" }, [`/api/v10/guilds/${guildId}/members/${applicationId}`]: { roles: ["222222222222222222"], user: { id: applicationId }, }, [`/api/v10/guilds/${guildId}/roles`]: [ { id: "222222222222222222", permissions: "309237763136" }, ], [`/api/v10/guilds/${guildId}/channels`]: [ { id: "333333333333333333", name: "agent-lab", position: 1, type: 0, }, ], }; return new Response(JSON.stringify(responses[path]), { status: path in responses ? 200 : 404, headers: { "content-type": "application/json" }, }); }; } function fakeTelegramFetch( botId = Number.parseInt(randomUUID().replaceAll("-", "").slice(0, 12), 16), ) { return async (input: string | URL | Request) => { const url = String(input); if (url.endsWith("/getMe")) { return new Response( JSON.stringify({ ok: true, result: { id: botId, username: `paperclip_${botId}_bot`, first_name: "Paperclip Test", }, }), { status: 200, headers: { "content-type": "application/json" } }, ); } if (url.endsWith("/getWebhookInfo")) { return new Response(JSON.stringify({ ok: true, result: { url: "" } }), { status: 200, headers: { "content-type": "application/json" }, }); } if ( url.endsWith("/setWebhook") || url.endsWith("/setMyCommands") || url.endsWith("/deleteWebhook") || url.endsWith("/deleteMyCommands") ) { return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } throw new Error(`Unexpected provider request: ${url}`); }; } function makeThread(input: { channelId: string; id: string; isDM?: boolean; name?: string; }) { const addReaction = vi.fn(async () => undefined); const startTyping = vi.fn(async () => undefined); const subscribe = vi.fn(async () => undefined); const postEphemeral = vi.fn(async () => ({ id: `thread-ephemeral-${randomUUID()}`, threadId: input.id, usedFallback: false, })); const post = vi.fn(async () => ({ id: `thread-post-${randomUUID()}`, threadId: input.id, })); const thread = { id: input.id, channelId: input.channelId, isDM: input.isDM ?? false, channel: { id: input.channelId, name: input.name ?? input.channelId }, adapter: { addReaction }, startTyping, subscribe, post, postEphemeral, } as unknown as Thread; return { thread, addReaction, startTyping, subscribe, post, postEphemeral, }; } function makeMessage(input: { attachments?: Attachment[]; id: string; raw?: unknown; text: string; mentioned?: boolean; userId?: string; userName?: string; }) { return { id: input.id, raw: input.raw, text: input.text, isMention: input.mentioned ?? false, attachments: input.attachments ?? [], metadata: { dateSent: new Date(), edited: false }, author: { userId: input.userId ?? "U-EXTERNAL", userName: input.userName ?? "alex", fullName: "Alex External", isBot: false, isMe: false, isSystem: false, } satisfies Author, } as unknown as Message; } function boardActor(companyId: string, userId = "owner-user") { return { type: "board" as const, source: "session" as const, userId, isInstanceAdmin: false, companyIds: [companyId], memberships: [{ companyId, status: "active", membershipRole: "operator" }], }; } function routesApp( db: TestDb, companyId: string, service: ChatChannelService, userId = "owner-user", ) { const app = express(); app.use(express.json()); app.use((req, _res, next) => { req.actor = boardActor(companyId, userId); next(); }); app.use( "/api", chatChannelRoutes(db, { heartbeat: { wakeup: async () => undefined }, service, }), ); app.use(errorHandler); return app; } function webhookApp( service: ChatChannelService, diagnostics?: (event: ChatWebhookDiagnosticEvent) => void, ) { const app = express(); if (diagnostics) app.use(createChatWebhookDiagnostics({ emit: diagnostics })); app.use(express.raw({ type: "*/*" })); app.use(chatWebhookRoutes(service)); app.use(errorHandler); return app; } describeEmbeddedPostgres("chat channel control-plane integration", () => { let db!: TestDb; let tempDb: Awaited< ReturnType > | null = null; const previousKeyFile = process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE; const secretsTmpDir = path.join( os.tmpdir(), `paperclip-chat-channels-${randomUUID()}`, ); beforeAll(async () => { mkdirSync(secretsTmpDir, { recursive: true }); process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = path.join( secretsTmpDir, "master.key", ); if (externalTestDatabaseUrl) { db = createDb(externalTestDatabaseUrl); } else { tempDb = await startEmbeddedPostgresTestDatabase( "paperclip-chat-channels-", ); db = createDb(tempDb.connectionString); } }, 30_000); afterAll(async () => { await tempDb?.cleanup(); if (previousKeyFile === undefined) delete process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE; else process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = previousKeyFile; rmSync(secretsTmpDir, { recursive: true, force: true }); }); // Services scan this file's shared database. Retire each case's fixtures // after its assertions so another case (or shard order) cannot claim them. const fixtureCompanies = new Set(); const fixtureServices = new Set(); afterEach(async () => { try { await Promise.all([...fixtureServices].map((service) => service.shutdown())); } finally { if (fixtureCompanies.size > 0) { await db.update(chatEndpoints).set({ status: "paused" }) .where(and(inArray(chatEndpoints.companyId, [...fixtureCompanies]), eq(chatEndpoints.status, "active"))); // The milestone scanner also considers paused endpoints while their // conversations are active. Retire those bindings after assertions. await db.update(chatConversations).set({ state: "completed" }) .where(and(inArray(chatConversations.companyId, [...fixtureCompanies]), inArray(chatConversations.state, ["active", "waiting"]))); } fixtureServices.clear(); fixtureCompanies.clear(); } }); async function seedCompany() { const companyId = randomUUID(); fixtureCompanies.add(companyId); const assignedAgentId = randomUUID(); const replacementAgentId = randomUUID(); await db.insert(companies).values({ id: companyId, name: `Chat Test ${companyId.slice(0, 8)}`, issuePrefix: `C${companyId.replaceAll("-", "").slice(0, 7).toUpperCase()}`, requireBoardApprovalForNewAgents: false, }); const now = new Date(); await db .insert(authUsers) .values({ id: "owner-user", name: "Owner User", email: "owner-user@example.com", emailVerified: true, createdAt: now, updatedAt: now, }) .onConflictDoNothing(); await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "owner-user", status: "active", membershipRole: "operator", }); await db.insert(principalPermissionGrants).values({ companyId, principalType: "user", principalId: "owner-user", permissionKey: "tools:manage_connections", scope: null, grantedByUserId: "owner-user", }); await db.insert(agents).values([ { id: assignedAgentId, companyId, name: "Maya", role: "engineer", status: "idle", adapterType: "paperclip_runner", adapterConfig: {}, runtimeConfig: {}, permissions: {}, }, { id: replacementAgentId, companyId, name: "Linus", role: "engineer", status: "idle", adapterType: "paperclip_runner", adapterConfig: {}, runtimeConfig: {}, permissions: {}, }, ]); return { companyId, assignedAgentId, replacementAgentId }; } // A truthy return is not a durable scheduler receipt. These transport tests // record the same exact receipt identity; real scheduling/coalescing is // separately exercised by durable-chat-wakeup.test.ts against heartbeat. function receiptBackedWakeup( wakeup: ChatChannelServiceOptions["heartbeat"]["wakeup"], ): ChatChannelServiceOptions["heartbeat"]["wakeup"] { return async (agentId, opts) => { const result = await wakeup(agentId, opts); const request = opts.durableChatRequest; if (request && result !== null && result !== undefined) { const [existing] = await db .select({ id: agentWakeupRequests.id }) .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, request.id)); if (existing) return result; await db.transaction(async (tx) => { await request.authorize( tx as unknown as Parameters[0], ); await tx .insert(agentWakeupRequests) .values({ id: request.id, companyId: request.companyId, agentId, source: opts.source ?? "assignment", triggerDetail: opts.triggerDetail, reason: opts.reason, payload: opts.payload, requestedByActorType: opts.requestedByActorType, requestedByActorId: opts.requestedByActorId, idempotencyKey: request.idempotencyKey, requestedAt: request.requestedAt, status: "queued", }) .onConflictDoNothing(); }); } return result; }; } function createService( runtime = new FakeChatSdkRuntime(), providerFetch: typeof globalThis.fetch = fakeSlackFetch() as typeof globalThis.fetch, overrides: Partial< Pick< ChatChannelServiceOptions, | "credentialMutationLeaseRenewalIntervalMs" | "deferWebhookProcessing" | "discordGatewayEventBarrier" | "discordGatewayMessageAdmissionBarrier" | "discordRootThreadTransportBarrier" | "discordGatewayLeaseRenewalIntervalMs" | "discordGatewayLeaseTtlMs" | "discordGatewayLeaseWaitMs" | "githubWebhookAuthenticationBarrier" | "githubWebhookReplayBarrier" | "githubWebhookResponseBudgetMs" | "publicBaseUrl" | "webhookPublicBaseUrl" | "nativeBotIdentityClaimBarrier" | "confirmationResolutionPersistBarrier" | "conversationLeaseRenewalIntervalMs" | "questionFormOpenAuthorizationBarrier" | "questionResolutionPersistBarrier" | "reactionLinkPreflightBarrier" | "reactionReplayConversationLockBarrier" | "reactionReplayEndpointLockBarrier" | "receiptReactionTransportBarrier" | "resolveNativeQuestion" | "renewCredentialMutationLease" | "renewConversationDeliveryLease" | "renewDiscordGatewayLease" | "scheduleDeferredWork" | "slackTaskAdmissionClaimBarrier" | "slackSessionSyncSelectionBarrier" | "setupSecretActivityLogger" | "setupSecretCredentialPersistBarrier" | "setupSecretFinalOwnershipBarrier" | "setupTestActivationBarrier" | "storage" | "reachAuthorizationBarrier" > > & { cancelRun?: NonNullable< ChatChannelServiceOptions["heartbeat"]["cancelRun"] >; wakeup?: ChatChannelServiceOptions["heartbeat"]["wakeup"]; } = {}, ) { const { cancelRun: cancelRunOverride, wakeup: wakeupOverride, ...serviceOverrides } = overrides; const wakeup = vi.fn(wakeupOverride ?? (async () => ({ accepted: true }))); const cancelRun = vi.fn( cancelRunOverride ?? (async () => ({ status: "cancelled" })), ); const service = chatChannelService(db, { fetch: providerFetch, heartbeat: { cancelRun, wakeup: receiptBackedWakeup(wakeup) }, publicBaseUrl: "https://paperclip.example", runtime: runtime as unknown as ChatSdkRuntime, ...serviceOverrides, }); fixtureServices.add(service); return { cancelRun, runtime, service, wakeup }; } function createStorageService() { const objects = new Map(); const putFile = vi.fn(async (input) => { const objectKey = `${input.namespace}/${randomUUID()}-${input.originalFilename ?? "attachment"}`; objects.set(objectKey, input.body); return { provider: "local_disk", objectKey, contentType: input.contentType, byteSize: input.body.length, sha256: createHash("sha256").update(input.body).digest("hex"), originalFilename: input.originalFilename, }; }); const storage: StorageService = { provider: "local_disk", putFile, getObject: vi.fn(async (_companyId, objectKey) => { const body = objects.get(objectKey); if (!body) throw new Error(`Missing test object ${objectKey}`); return { stream: Readable.from([body]), contentLength: body.length, }; }), headObject: vi.fn(async (_companyId, objectKey) => ({ exists: objects.has(objectKey), contentLength: objects.get(objectKey)?.length, })), deleteObject: vi.fn(async (_companyId, objectKey) => { objects.delete(objectKey); }), }; return { objects, putFile, storage }; } async function retirePublicationFixture( service: ChatChannelService, endpointId: string, ) { try { await service.shutdown(); } finally { // Workers scan the shared fixture database. Local shutdown alone leaves // future retries/receipts eligible for the next test's service. Preserve // their asserted audit state, but retire this endpoint after assertions. await db .update(chatEndpoints) .set({ status: "paused" }) .where( and( eq(chatEndpoints.id, endpointId), eq(chatEndpoints.status, "active"), ), ); } } async function recordSlackUrlVerification( service: ChatChannelService, publicId: string, ) { await service.handleWebhook( publicId, "slack", new Request( `https://paperclip.example/api/chat-webhooks/${publicId}/slack`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ type: "url_verification", challenge: "verified-challenge", }), }, ), ); } function signedSlackWebhookRequest(input: { url: string; body: string; contentType: string; signingSecret?: string; }) { const timestamp = String(Math.floor(Date.now() / 1000)); const signature = createHmac( "sha256", input.signingSecret ?? "test-signing-secret", ) .update(`v0:${timestamp}:${input.body}`) .digest("hex"); return new Request(input.url, { method: "POST", headers: { "content-type": input.contentType, "x-slack-request-timestamp": timestamp, "x-slack-signature": `v0=${signature}`, }, body: input.body, }); } function githubWebhookVerificationResponse( service: ChatChannelService, publicId: string, webhookSecret: string, providerDeliveryId = `github-setup-ping-${randomUUID()}`, ) { const body = JSON.stringify({ zen: "Keep it logically awesome." }); const signature = createHmac("sha256", webhookSecret) .update(body) .digest("hex"); return service.handleWebhook( publicId, "github", new Request( `https://paperclip.example/api/chat-webhooks/${publicId}/github`, { method: "POST", headers: { "content-type": "application/json", "x-github-delivery": providerDeliveryId, "x-github-event": "ping", "x-hub-signature-256": `sha256=${signature}`, }, body, }, ), ); } async function recordGitHubWebhookVerification( service: ChatChannelService, publicId: string, webhookSecret: string, ) { const providerDeliveryId = `github-setup-ping-${randomUUID()}`; const response = await githubWebhookVerificationResponse( service, publicId, webhookSecret, providerDeliveryId, ); expect([200, 202]).toContain(response.status); const action = await db .select({ id: chatActions.id }) .from(chatActions) .innerJoin(chatEndpoints, eq(chatEndpoints.id, chatActions.endpointId)) .where( and( eq(chatEndpoints.publicId, publicId), eq( chatActions.providerActionId, `github_webhook_ingress:${providerDeliveryId}`, ), ), ) .then((rows) => rows[0] ?? null); if (action) { await service.processPendingGitHubWebhookIngress(1, action.id); } await vi.waitFor(async () => { const setup = await db .select({ setup: chatEndpoints.setup }) .from(chatEndpoints) .where(eq(chatEndpoints.publicId, publicId)) .then((rows) => rows[0]?.setup); expect(setup).toMatchObject({ webhookVerifiedAt: expect.any(String) }); }); } function signedGitHubWebhookRequest(input: { delivery: string; event: string; payload: unknown; webhookSecret: string; url?: string; }) { const body = JSON.stringify(input.payload); const signature = createHmac("sha256", input.webhookSecret) .update(body) .digest("hex"); return new Request(input.url ?? "https://paperclip.example/github", { method: "POST", headers: { "content-type": "application/json", "x-github-event": input.event, "x-github-delivery": input.delivery, "x-hub-signature-256": `sha256=${signature}`, }, body, }); } async function chatWakeContext(input: { endpointId: string; issueId: string; provider: ChatProvider; providerMessageId: string; }) { const wakeCommentId = await db .select({ commentId: chatMessageLinks.commentId }) .from(chatMessageLinks) .where( and( eq(chatMessageLinks.endpointId, input.endpointId), eq(chatMessageLinks.providerMessageId, input.providerMessageId), eq(chatMessageLinks.direction, "inbound"), ), ) .then((rows) => rows[0]?.commentId ?? null); if (!wakeCommentId) { throw new Error( `Expected inbound comment link ${input.providerMessageId}`, ); } return { issueId: input.issueId, source: `chat:${input.provider}`, wakeCommentId, wakeCommentIds: [wakeCommentId], }; } async function addSelectedChatFinal(input: { agentId: string; body: string; companyId: string; issueId: string; runId: string; }) { const authorizationReason = await resolveChatRunPresentationAuthorizationReason(db, { companyId: input.companyId, issueId: input.issueId, runId: input.runId, }); if (authorizationReason !== "allow_chat_run_presentation") { throw new Error("Expected chat run presentation authorization"); } return issueService(db).addComment( input.issueId, input.body, { agentId: input.agentId, runId: input.runId }, { authorType: "agent", authorizationReason }, ); } async function qualifySetupRoundTrip( service: ChatChannelService, endpointId: string, userId = "U-EXTERNAL", ) { const endpoint = await service.get(endpointId); let conversation: typeof chatConversations.$inferSelect | undefined; await vi.waitFor(async () => { conversation = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpointId)) .then((rows) => rows.at(-1)); expect(conversation).toBeDefined(); }); if (!conversation) throw new Error("Expected setup conversation"); const fakeRuntime = service.runtime as unknown as FakeChatSdkRuntime; const callbacks = fakeRuntime.configurations.get(endpointId)?.callbacks; if (!callbacks) throw new Error("Expected setup callbacks"); const { thread } = makeThread({ id: conversation.externalThreadId, channelId: conversation.externalConversationId, isDM: conversation.isDirectMessage, name: conversation.externalLabel, }); const setupFollowUpMessageId = `setup-follow-up-${randomUUID()}`; await callbacks.onMessage({ endpointId, provider: endpoint.provider, thread, message: makeMessage({ id: setupFollowUpMessageId, text: "Setup follow-up", userId, }), trigger: endpoint.provider === "telegram" ? "direct_message" : "subscribed_message", }); const contextSnapshot = await chatWakeContext({ endpointId, issueId: conversation.issueId, provider: endpoint.provider, providerMessageId: setupFollowUpMessageId, }); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: endpoint.companyId, agentId: endpoint.assignedAgentId, status: "succeeded", contextSnapshot, }); await addSelectedChatFinal({ agentId: endpoint.assignedAgentId, body: "Setup round trip complete", companyId: endpoint.companyId, issueId: conversation.issueId, runId, }); await service.processPendingPublications(); const providerRuntime = fakeRuntime.endpoints.get(endpointId); if (providerRuntime) providerRuntime.posts.length = 0; } async function configuredSlackEndpoint( fixture: Awaited>, overrides: { allowUnlinkedPeople?: boolean } & Partial< Pick< ChatChannelServiceOptions, | "credentialMutationLeaseRenewalIntervalMs" | "conversationLeaseRenewalIntervalMs" | "deferWebhookProcessing" | "fetch" | "questionFormOpenAuthorizationBarrier" | "questionResolutionPersistBarrier" | "reactionReplayEndpointLockBarrier" | "reachAuthorizationBarrier" | "resolveNativeQuestion" | "renewCredentialMutationLease" | "renewConversationDeliveryLease" | "scheduleDeferredWork" | "slackTaskAdmissionClaimBarrier" | "storage" > > & { cancelRun?: NonNullable< ChatChannelServiceOptions["heartbeat"]["cancelRun"] >; wakeup?: ChatChannelServiceOptions["heartbeat"]["wakeup"]; } = {}, ) { const context = createService( new FakeChatSdkRuntime(), overrides.fetch ?? (fakeSlackFetch() as typeof globalThis.fetch), overrides, ); const endpoint = await context.service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId, name: "Maya in Slack", }, "owner-user", ); if (overrides?.allowUnlinkedPeople !== undefined) { await context.service.update( endpoint.id, { allowUnlinkedPeople: overrides.allowUnlinkedPeople, }, "owner-user", ); } await context.service.configure( endpoint.id, { action: "configure", credentials: { botToken: "xoxb-test-token", signingSecret: "test-signing-secret", }, }, "owner-user", ); await recordSlackUrlVerification(context.service, endpoint.publicId); await context.service.configure( endpoint.id, { action: "verify" }, "owner-user", ); const callbacks = context.runtime.configurations.get( endpoint.id, )?.callbacks; if (!callbacks) throw new Error("Fake runtime did not receive endpoint callbacks"); return { ...context, endpoint, callbacks }; } async function acceptedUnknownSlackFileReceipt(label: string) { const fixture = await seedCompany(); const storage = createStorageService(); const configured = await configuredSlackEndpoint(fixture, { storage: storage.storage, }); const { callbacks, endpoint, runtime, service } = configured; const channel = makeThread({ channelId: `C-RECEIPT-${label.toUpperCase()}`, id: `slack:C-RECEIPT-${label.toUpperCase()}:${Date.now()}.1`, name: `receipt-${label}`, }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: `${Date.now()}.1`, text: "@maya start a receipt authorization test", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Slack receipt conversation"); const comment = await issueService(db).addComment( conversation.issueId, "Accepted Slack authorization fixture", { userId: "owner-user" }, ); const stored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${conversation.issueId}`, originalFilename: `${label}.txt`, contentType: "text/plain", body: Buffer.from(`accepted ${label}`, "utf8"), }); const attachment = await issueService(db).createAttachment({ issueId: conversation.issueId, issueCommentId: comment.id, provider: stored.provider, objectKey: stored.objectKey, contentType: stored.contentType, byteSize: stored.byteSize, sha256: stored.sha256, originalFilename: stored.originalFilename, createdByUserId: "owner-user", }); const [publication] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, commentId: comment.id, idempotencyKey: `slack-file-receipt-${label}:${randomUUID()}`, payload: { text: "", attachmentIds: [attachment.id] }, state: "pending", }) .returning(); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack provider runtime"); providerRuntime.slackFilePostAcceptanceError = new Error( "connection closed after Slack accepted the upload", ); await service.processPendingPublications(); providerRuntime.slackFilePostAcceptanceError = null; const [receipt] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slack_file_upload_receipt"), eq( chatActions.providerActionId, `slack-file-receipt:${publication!.id}:1`, ), ), ); if (!publication || !receipt) { throw new Error("Expected accepted Slack receipt fixture"); } return { ...configured, channel, conversation, fixture, providerRuntime, publication, receipt, }; } async function configuredTeamsEndpoint( fixture: Awaited>, overrides: Partial< Pick< ChatChannelServiceOptions, | "deferWebhookProcessing" | "githubWebhookAuthenticationBarrier" | "githubWebhookResponseBudgetMs" | "scheduleDeferredWork" | "storage" | "reachAuthorizationBarrier" > > = {}, ) { const context = createService( new FakeChatSdkRuntime(), (async () => new Response(JSON.stringify({ access_token: "teams-test-access" }), { status: 200, headers: { "content-type": "application/json" }, })) as typeof globalThis.fetch, overrides, ); const endpoint = await context.service.create( fixture.companyId, { provider: "microsoft-teams", assignedAgentId: fixture.assignedAgentId, name: "Maya in Teams", }, "owner-user", ); await context.service.configure( endpoint.id, { action: "configure", credentials: { clientId: randomUUID(), tenantId: randomUUID(), clientSecret: "teams-test-secret", }, }, "owner-user", ); const callbacks = context.runtime.configurations.get( endpoint.id, )?.callbacks; if (!callbacks) throw new Error("Fake runtime did not receive Teams callbacks"); return { ...context, endpoint, callbacks }; } async function configuredGitHubEndpoint( fixture: Awaited>, overrides: Partial< Pick< ChatChannelServiceOptions, | "deferWebhookProcessing" | "githubWebhookReplayBarrier" | "githubWebhookResponseBudgetMs" | "publicBaseUrl" | "scheduleDeferredWork" | "setupSecretActivityLogger" | "setupSecretCredentialPersistBarrier" | "receiptReactionTransportBarrier" | "storage" > > & { wakeup?: ChatChannelServiceOptions["heartbeat"]["wakeup"] } = {}, useVerifiedAppId = false, ) { let setupComplete = false; const deferredSchedule = overrides.scheduleDeferredWork; let installationId = 2468; let additionalInstallationIds: number[] = []; let installationAvailable = true; let repositories = [ { id: 97531, full_name: "paperclipai/paperclip", html_url: "https://github.com/paperclipai/paperclip", owner: { id: 1357, login: "paperclipai" }, private: false, }, ]; let appPermissions: Record = { issues: "write", metadata: "read", pull_requests: "write", }; let installationPermissions: Record = { issues: "write", metadata: "read", pull_requests: "write", }; let appEvents = [ "github_app_authorization", "installation", "installation_repositories", "issue_comment", "pull_request_review_comment", ]; const webhookSyncRequests: Array> = []; let webhookSyncResponse: (() => Promise) | null = null; let supplementalProviderFetch: | (( input: string | URL | Request, init?: RequestInit, ) => Promise) | null = null; const appRegistrationId = Number.parseInt( fixture.companyId.replaceAll("-", "").slice(0, 8), 16, ); const privateKey = generateKeyPairSync("rsa", { modulusLength: 2048 }) .privateKey.export({ type: "pkcs8", format: "pem" }) .toString(); const providerFetch = (async ( input: string | URL | Request, init?: RequestInit, ) => { const url = String(input); const supplemental = await supplementalProviderFetch?.(input, init); if (supplemental) return supplemental; if (url === "https://api.github.com/app/hook/config") { expect(init?.method).toBe("PATCH"); const config = JSON.parse(String(init?.body)) as Record< string, unknown >; webhookSyncRequests.push(config); if (webhookSyncResponse) return webhookSyncResponse(); return new Response( JSON.stringify({ url: config.url, content_type: config.content_type, insecure_ssl: config.insecure_ssl, }), { status: 200 }, ); } if (url === "https://api.github.com/app") { return new Response( JSON.stringify({ id: appRegistrationId, slug: `maya-${fixture.companyId.slice(0, 8)}`, name: "Maya Paperclip", owner: { login: "paperclipai" }, permissions: appPermissions, events: appEvents, }), { status: 200, headers: { "content-type": "application/json" } }, ); } if (url === "https://api.github.com/app/installations?per_page=100") { return new Response( JSON.stringify( installationAvailable ? [ { id: installationId, account: { id: 1357, login: "paperclipai", type: "Organization", }, permissions: installationPermissions, suspended_at: null, }, ...additionalInstallationIds.map((id) => ({ id, account: { id: id + 10_000, login: `additional-${id}`, type: "Organization", }, permissions: installationPermissions, suspended_at: null, })), ] : [], ), { status: 200, headers: { "content-type": "application/json" } }, ); } if ( url === `https://api.github.com/app/installations/${installationId}/access_tokens` ) { return new Response(JSON.stringify({ token: "installation-token" }), { status: 201, headers: { "content-type": "application/json" }, }); } if ( url === "https://api.github.com/installation/repositories?per_page=100&page=1" ) { return new Response(JSON.stringify({ repositories }), { status: 200, headers: { "content-type": "application/json" }, }); } throw new Error(`Unexpected provider request: ${url}`); }) as typeof globalThis.fetch; const context = createService( new FakeChatSdkRuntime(), providerFetch, deferredSchedule ? { ...overrides, scheduleDeferredWork: (task) => { if (setupComplete) deferredSchedule(task); else setImmediate(task); }, } : overrides, ); const endpoint = await context.service.create( fixture.companyId, { provider: "github", assignedAgentId: fixture.assignedAgentId, name: "Maya in GitHub", }, "owner-user", ); const { webhookSecret } = await context.service.generateSetupSecret( endpoint.id, "owner-user", ); await recordGitHubWebhookVerification( context.service, endpoint.publicId, webhookSecret, ); await context.service.configure( endpoint.id, { action: "configure", credentials: { appId: useVerifiedAppId ? String(appRegistrationId) : "123456", privateKey, }, }, "owner-user", ); const resources = await context.service.listResources(endpoint.id); await context.service.replaceResources(endpoint.id, [ { id: resources[0]!.id, enabled: true }, ]); setupComplete = true; const callbacks = context.runtime.configurations.get( endpoint.id, )?.callbacks; if (!callbacks) throw new Error("Fake runtime did not receive GitHub callbacks"); return { ...context, endpoint, callbacks, webhookSecret, providerFetch, webhookSyncRequests, setSupplementalProviderFetch(value: typeof supplementalProviderFetch) { supplementalProviderFetch = value; }, setWebhookSyncResponse(value: (() => Promise) | null) { webhookSyncResponse = value; }, setInstallationId(value: number) { installationId = value; }, setAdditionalInstallationIds(value: number[]) { additionalInstallationIds = value; }, setInstallationAvailable(value: boolean) { installationAvailable = value; }, setRepositories( value: Array<{ id: number; full_name: string; html_url: string; owner: { id: number; login: string }; private: boolean; }>, ) { repositories = value; }, setAppAccess(input: { permissions?: Record; events?: string[]; }) { if (input.permissions) appPermissions = input.permissions; if (input.events) appEvents = input.events; }, setInstallationAccess(permissions: Record) { installationPermissions = permissions; }, }; } async function configuredTelegramEndpoint( fixture: Awaited>, overrides: Parameters[2] = {}, ) { const context = createService( new FakeChatSdkRuntime(), fakeTelegramFetch() as typeof globalThis.fetch, overrides, ); const endpoint = await context.service.create( fixture.companyId, { provider: "telegram", assignedAgentId: fixture.assignedAgentId, name: "Maya in Telegram", }, "owner-user", ); await context.service.configure( endpoint.id, { action: "configure", credentials: { botToken: "123456:telegram-interaction-test" }, }, "owner-user", ); const callbacks = context.runtime.configurations.get( endpoint.id, )?.callbacks; if (!callbacks) throw new Error("Fake runtime did not receive Telegram callbacks"); return { ...context, endpoint, callbacks }; } it("corrects Telegram DM conversation links without rewriting retained provider receipts", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service } = await configuredTelegramEndpoint(fixture); let server: Server | undefined; try { const thread = makeThread({ channelId: "77117711", id: "telegram:77117711", isDM: true, name: "Telegram link user", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: thread.thread, message: makeMessage({ id: "88", text: "Check the DM task link", userId: "77117711", raw: { message_id: 88, chat: { id: 77117711, type: "private", username: "human_user" }, }, }), trigger: "direct_message", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); expect(conversation).toBeDefined(); const configured = await service.get(endpoint.id); const expectedUrl = `https://t.me/${configured!.botUsername}`; expect(conversation!.providerUrl).toBe(expectedUrl); // A retained pre-fix URL must be corrected at read time, without mutating // historical receipts or requiring a new message from the external user. await db .update(chatConversations) .set({ providerUrl: "https://t.me/human_user/88" }) .where(eq(chatConversations.id, conversation!.id)); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ id: conversation!.id, isDirectMessage: true, externalUrl: expectedUrl, }), ]); // The task banner uses a different route from the conversation list; // issue-detail responses also consume this exact company-scoped helper. server = createServer(routesApp(db, fixture.companyId, service)); await new Promise((resolve, reject) => { server!.once("error", reject); server!.listen(0, "127.0.0.1", () => { server!.off("error", reject); resolve(); }); }); const binding = await service.getIssueBinding(conversation!.issueId); const response = await request(server) .get(`/api/issues/${conversation!.issueId}/chat-binding`) .expect(200); expect({ binding: binding?.externalUrl, route: response.body.externalUrl, }).toEqual({ binding: expectedUrl, route: expectedUrl }); await expect( getExternalChannelBindingSummary(db, randomUUID(), conversation!.issueId), ).resolves.toBeNull(); await expect( db .select({ providerUrl: chatConversations.providerUrl }) .from(chatConversations) .where(eq(chatConversations.id, conversation!.id)), ).resolves.toEqual([{ providerUrl: "https://t.me/human_user/88" }]); await db .update(chatEndpoints) .set({ botUsername: null }) .where(eq(chatEndpoints.id, endpoint.id)); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ id: conversation!.id, externalUrl: null }), ]); await expect( service.getIssueBinding(conversation!.issueId), ).resolves.toMatchObject({ externalUrl: null }); const missingBot = await request(server) .get(`/api/issues/${conversation!.issueId}/chat-binding`) .expect(200); expect(missingBot.body.externalUrl).toBeNull(); // A legitimate group message link remains a message link, not a bot DM. await db .update(chatConversations) .set({ isDirectMessage: false, externalThreadId: "telegram:-10077117711", providerUrl: "https://t.me/qa_group/88", }) .where(eq(chatConversations.id, conversation!.id)); await expect( service.getIssueBinding(conversation!.issueId), ).resolves.toMatchObject({ externalUrl: "https://t.me/qa_group/88" }); } finally { try { if (server?.listening) await new Promise((resolve, reject) => { server!.close((error) => (error ? reject(error) : resolve())); server!.closeAllConnections(); }); } finally { await service.shutdown(); } } }); async function configuredDiscordEndpoint( fixture: Awaited>, overrides: Parameters[2] = {}, ) { const applicationId = uniqueDiscordApplicationId(); const context = createService( new FakeChatSdkRuntime(), fakeDiscordFetch(applicationId) as typeof globalThis.fetch, overrides, ); const endpoint = await context.service.create( fixture.companyId, { provider: "discord", assignedAgentId: fixture.assignedAgentId, name: "Maya in Discord", }, "owner-user", ); await context.service.configure( endpoint.id, { action: "configure", credentials: { applicationId, botToken: "discord-secret", guildId: "1457808928258658549", }, }, "owner-user", ); const callbacks = context.runtime.configurations.get( endpoint.id, )?.callbacks; if (!callbacks) throw new Error("Fake runtime did not receive Discord callbacks"); return { ...context, endpoint, callbacks }; } describe("Discord automatic command registration", () => { async function retireRegistrationFixture( service: ChatChannelService, endpointId: string, ) { try { const owners = await db .select() .from(chatDiscordCommandOwners) .where(eq(chatDiscordCommandOwners.endpointId, endpointId)); await service.configure(endpointId, { action: "remove" }, "owner-user"); expect((await service.get(endpointId))?.status).toBe("archived"); // Removal retires only this fixture's runtime; immutable application // ownership stays available to the no-adoption/tombstone checks. expect( await db .select() .from(chatDiscordCommandOwners) .where(eq(chatDiscordCommandOwners.endpointId, endpointId)), ).toEqual(owners); } finally { await service.shutdown(); } } async function registrationFixture( mode: "success" | "unavailable" | "unknown" = "success", ) { const fixture = await seedCompany(); const applicationId = uniqueDiscordApplicationId(); const guildId = "1457808928258658549"; const baseFetch = fakeDiscordFetch(applicationId); const commandUrl = `https://discord.com/api/v10/applications/${applicationId}/commands`; let commands: Record[] = [ { id: "888888888888888881", application_id: applicationId, version: "888888888888888882", type: 1, name: "other", description: "Existing application command", }, ]; let unavailable = mode === "unavailable"; let endpointId: string; const calls: string[] = []; const providerFetch = vi.fn( async (input: string | URL | Request, init?: RequestInit) => { if (String(input) !== commandUrl) return baseFetch(input); const method = init?.method ?? "GET"; calls.push(method); if (method === "GET") return Response.json(commands, { status: unavailable ? 503 : 200 }); expect(method).toBe("POST"); const [intent] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpointId), eq(chatActions.kind, "discord_command_registration"), ), ); expect(intent?.payload.registration).toMatchObject({ phase: "attempted", scope: { applicationId, guildId }, }); expect(intent?.status).not.toBe("processed"); const definition = JSON.parse(String(init?.body)) as Record< string, unknown >; const command = { ...definition, id: "888888888888888883", application_id: applicationId, version: "888888888888888884", }; commands = [...commands, command]; if (mode === "unknown") throw new Error("private registration transport failure"); return Response.json(command); }, ) as unknown as typeof globalThis.fetch; const context = createService(new FakeChatSdkRuntime(), providerFetch); const endpoint = await context.service.create( fixture.companyId, { provider: "discord", assignedAgentId: fixture.assignedAgentId, name: "Discord command registration", }, "owner-user", ); endpointId = endpoint.id; const credentials = { applicationId, guildId, botToken: "discord-registration-private-token", }; const configure = () => context.service.configure( endpointId, { action: "configure", credentials }, "owner-user", ); const makeDue = () => db .update(chatActions) .set({ result: sql`${chatActions.result} || '{"retryAt":"2000-01-01T00:00:00.000Z"}'::jsonb`, }) .where( and( eq(chatActions.endpointId, endpointId), eq(chatActions.kind, "discord_command_registration"), ), ); return { ...context, fixture, endpoint, credentials, calls, providerFetch, configure, makeDue, available: (value = true) => { unavailable = !value; }, commandRows: () => commands, replaceCommands: (next: Record[]) => { commands = next; }, }; } it("persists registration before enabling the runtime and preserves unrelated commands", async () => { const f = await registrationFixture(); try { await f.configure(); const connected = await f.service.get(f.endpoint.id); expect(connected?.capabilities).toMatchObject({ slashCommands: true, ephemeralMessages: true, }); expect( f.runtime.configurations.get(f.endpoint.id)?.callbacks.onSlashCommand, ).toBeTypeOf("function"); expect(f.calls).toEqual(["GET", "POST"]); expect(f.commandRows().map((row) => row.name)).toEqual([ "other", "paperclip", ]); const [registration] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, f.endpoint.id), eq(chatActions.kind, "discord_command_registration"), ), ); expect(registration?.status).toBe("processed"); expect(registration?.payload.registration).toMatchObject({ phase: "registered", }); expect(JSON.stringify(registration)).not.toContain( f.credentials.botToken, ); await f.service.reconcileProviderRuntimes(); expect(f.calls).toEqual(["GET", "POST"]); } finally { await retireRegistrationFixture(f.service, f.endpoint.id); } }); it("keeps mention setup usable on registration failure and upgrades automatically when due", async () => { const f = await registrationFixture("unavailable"); try { await f.configure(); expect((await f.service.get(f.endpoint.id))?.status).toBe("verifying"); expect( (await f.service.get(f.endpoint.id))?.capabilities.slashCommands, ).toBe(false); expect( f.runtime.configurations.get(f.endpoint.id)?.callbacks.onSlashCommand, ).toBeTypeOf("function"); f.available(); await f.makeDue(); await f.service.reconcileProviderRuntimes(); expect( (await f.service.get(f.endpoint.id))?.capabilities.slashCommands, ).toBe(true); expect( f.runtime.configurations.get(f.endpoint.id)?.callbacks.onSlashCommand, ).toBeTypeOf("function"); expect(f.calls).toEqual(["GET", "GET", "POST"]); } finally { await retireRegistrationFixture(f.service, f.endpoint.id); } }); it("reconstructs an uncertain registration by GET without repeating the provider write", async () => { const f = await registrationFixture("unknown"); let restarted: ReturnType | undefined; try { await f.configure(); expect( (await f.service.get(f.endpoint.id))?.capabilities.slashCommands, ).toBe(false); expect( f.runtime.configurations.get(f.endpoint.id)?.callbacks.onSlashCommand, ).toBeTypeOf("function"); await f.service.shutdown(); await f.makeDue(); restarted = createService(new FakeChatSdkRuntime(), f.providerFetch); await restarted.service.reconcileProviderRuntimes(); expect( (await restarted.service.get(f.endpoint.id))?.capabilities .slashCommands, ).toBe(true); expect( restarted.runtime.configurations.get(f.endpoint.id)?.callbacks .onSlashCommand, ).toBeTypeOf("function"); expect(f.calls).toEqual(["GET", "POST", "GET"]); } finally { try { await retireRegistrationFixture( restarted?.service ?? f.service, f.endpoint.id, ); } finally { await f.service.shutdown(); } } }); it("preserves a healthy Discord Gateway when a due command refresh returns 503", async () => { const f = await registrationFixture(); try { await f.configure(); const original = f.runtime.endpoints.get(f.endpoint.id); expect(original).toBeDefined(); const originalCallbacks = f.runtime.configurations.get( f.endpoint.id, )!.callbacks; expect(originalCallbacks.onMessage).toBeTypeOf("function"); expect(originalCallbacks.onSlashCommand).toBeTypeOf("function"); const shutdown = vi.spyOn(original!, "shutdown"); const targetRestart = vi.fn(async () => { throw new Error( "Healthy Gateway must not restart for command maintenance", ); }); f.runtime.initializeHook = async (endpointId) => { if (endpointId === f.endpoint.id) await targetRestart(); }; f.available(false); await f.makeDue(); await f.service.reconcileProviderRuntimes(); expect(f.runtime.endpoints.get(f.endpoint.id)).toBe(original); expect(f.runtime.configurations.get(f.endpoint.id)?.callbacks).toBe( originalCallbacks, ); expect(shutdown).not.toHaveBeenCalled(); expect(targetRestart).not.toHaveBeenCalled(); expect( (await f.service.get(f.endpoint.id))?.capabilities, ).toMatchObject({ slashCommands: false, ephemeralMessages: false }); expect(f.calls).toEqual(["GET", "POST", "GET"]); await f.service.reconcileProviderRuntimes(); expect(f.calls).toEqual(["GET", "POST", "GET"]); expect(f.runtime.endpoints.get(f.endpoint.id)).toBe(original); expect(shutdown).not.toHaveBeenCalled(); expect(targetRestart).not.toHaveBeenCalled(); } finally { await retireRegistrationFixture(f.service, f.endpoint.id); } }); it("retains the Gateway after an external command namespace conflict without overwriting it", async () => { const f = await registrationFixture(); try { await f.configure(); const original = f.runtime.endpoints.get(f.endpoint.id); expect(original).toBeDefined(); const callbacks = f.runtime.configurations.get( f.endpoint.id, )!.callbacks; const shutdown = vi.spyOn(original!, "shutdown"); f.replaceCommands( f .commandRows() .map((command) => command.name === "paperclip" ? { ...command, description: "Owned by another integration" } : command, ), ); await f.makeDue(); await f.service.reconcileProviderRuntimes(); expect( (await f.service.get(f.endpoint.id))?.capabilities.slashCommands, ).toBe(false); expect(f.runtime.endpoints.get(f.endpoint.id)).toBe(original); expect(f.runtime.configurations.get(f.endpoint.id)?.callbacks).toBe( callbacks, ); expect(callbacks.onMessage).toBeTypeOf("function"); expect(callbacks.onSlashCommand).toBeTypeOf("function"); expect(shutdown).not.toHaveBeenCalled(); expect(f.calls).toEqual(["GET", "POST", "GET"]); expect(f.commandRows()[1]?.description).toBe( "Owned by another integration", ); } finally { await retireRegistrationFixture(f.service, f.endpoint.id); } }); }); async function deliverMessage(input: { callbacks: CreateChatSdkEndpointRuntimeOptions["callbacks"]; endpointId: string; message: Message; provider?: ChatProvider; providerUpdateId?: number; thread: Thread; trigger: ChatSdkMessageTrigger; }) { await input.callbacks.onMessage({ endpointId: input.endpointId, provider: input.provider ?? "slack", providerUpdateId: input.providerUpdateId, thread: input.thread, message: input.message, trigger: input.trigger, }); } function attachFakeGitHubIssueCommentWebhook(input: { botUsername: string; callbacks: CreateChatSdkEndpointRuntimeOptions["callbacks"]; endpointId: string; runtime: FakeEndpointRuntime; }) { input.runtime.webhookHook = async (request) => { const payload = (await request.clone().json()) as { action?: unknown; comment?: { body?: unknown; created_at?: unknown; id?: unknown; updated_at?: unknown; user?: { id?: unknown; login?: unknown }; }; issue?: { number?: unknown; pull_request?: unknown }; repository?: { full_name?: unknown; name?: unknown; owner?: { login?: unknown }; }; sender?: { id?: unknown; login?: unknown }; }; if (payload.action !== "created") return; const fullName = typeof payload.repository?.full_name === "string" ? payload.repository.full_name : typeof payload.repository?.owner?.login === "string" && typeof payload.repository.name === "string" ? `${payload.repository.owner.login}/${payload.repository.name}` : null; const issueNumber = payload.issue?.number; const messageId = payload.comment?.id; const text = payload.comment?.body; if ( !fullName || typeof issueNumber !== "number" || (typeof messageId !== "number" && typeof messageId !== "string") || typeof text !== "string" ) { return; } const mentionNames = new Set([ input.botUsername, input.botUsername.replace(/\[bot\]$/i, ""), ]); const mentioned = [...mentionNames].some((identity) => new RegExp( `(? void> = []; const configured = await configuredGitHubEndpoint( fixture, { deferWebhookProcessing: true, githubWebhookReplayBarrier: replayBarrier, scheduleDeferredWork: (task) => deferred.push(task), }, true, ); const { service, endpoint, runtime, callbacks } = configured; // Setup admission is covered separately. Recovery is intentionally only // eligible for an active endpoint with an explicitly enabled repository. await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, endpoint.id)); const current = await service.get(endpoint.id); const providerRuntime = runtime.endpoints.get(endpoint.id)!; attachFakeGitHubIssueCommentWebhook({ botUsername: current.botUsername!, callbacks, endpointId: endpoint.id, runtime: providerRuntime, }); const guid = randomUUID(); const deliveryId = "9007199254740993"; const webhookUrl = `https://paperclip.example/api/chat-webhooks/${endpoint.publicId}/github`; const createdAt = new Date(Date.now() - 30_000).toISOString(); const body = `@${current.botUsername} recover the original request ${randomUUID()}`; const comment = { id: 9191, body, created_at: createdAt, updated_at: createdAt, user: { id: 42, login: "octocat", type: "User" }, ...(event === "issue_comment" ? { issue_url: "https://api.github.com/repos/paperclipai/paperclip/issues/91", } : { pull_request_url: "https://api.github.com/repos/paperclipai/paperclip/pulls/91", }), }; const payload = { action: "created", installation: { id: 2468 }, repository: { id: 97531, full_name: "paperclipai/paperclip", name: "paperclip", owner: { id: 1357, login: "paperclipai" }, }, ...(event === "issue_comment" ? { issue: { id: 9190, number: 91 } } : { pull_request: { id: 9190, number: 91 } }), comment, sender: { id: 42, login: "octocat", type: "User" }, }; const original = { id: deliveryId, guid, delivered_at: createdAt, redelivery: false, status_code: 502 as number | null, event, action: "created", installation_id: 2468, repository_id: 97531, throttled_at: null as string | null, }; let deliveries = [original]; const extraDetails = new Map< string, { summary: typeof original; payload: unknown } >(); let historyHasMore = false; let historyPage = 0; let canonicalComment = { ...comment }; let beforeList: (() => Promise) | undefined; let beforePost: (() => Promise) | undefined; const requests: Array<{ method: string; pathname: string }> = []; const posts: string[] = []; const commentPath = `/repos/paperclipai/paperclip/${event === "issue_comment" ? "issues" : "pulls"}/comments/9191`; // Emit the actual unsafe JSON numeric literal, not a string-only mock. const json = (value: unknown) => new Response( JSON.stringify(value).replaceAll(`"${deliveryId}"`, deliveryId), { status: 200, headers: { "content-type": "application/json" }, }, ); configured.setSupplementalProviderFetch(async (input, init) => { const url = new URL(String(input)); const method = init?.method ?? "GET"; const isRecovery = url.pathname.startsWith("/app/hook/") || url.pathname === commentPath || url.pathname === "/installation/token"; if (!isRecovery) return undefined; requests.push({ method, pathname: url.pathname }); if (url.pathname === "/app/hook/config" && method === "GET") { return json({ url: webhookUrl, content_type: "json", insecure_ssl: "0", }); } if (url.pathname === "/app/hook/deliveries") { await beforeList?.(); const response = json(deliveries); if (historyHasMore) response.headers.set( "link", `; rel="next"`, ); return response; } if (url.pathname === `/app/hook/deliveries/${deliveryId}`) { return json({ ...original, url: webhookUrl, request: { headers: { authorization: "private-provider-header-canary" }, payload, }, response: { payload: "private-proxy-body-canary" }, }); } const extra = extraDetails.get( url.pathname.replace(/^\/app\/hook\/deliveries\//, ""), ); if (extra) return json({ ...extra.summary, url: webhookUrl, request: { payload: extra.payload }, }); if ( url.pathname === `/app/hook/deliveries/${deliveryId}/attempts` && method === "POST" ) { posts.push(url.pathname); await beforePost?.(); return new Response(null, { status: 202 }); } if (url.pathname === commentPath) return json(canonicalComment); if (url.pathname === "/installation/token" && method === "DELETE") return new Response(null, { status: 204 }); return undefined; }); const checkpoint = () => db .select() .from(chatSdkState) .where( and( eq(chatSdkState.endpointId, endpoint.id), eq(chatSdkState.stateKey, "paperclip:github-webhook-recovery:v1"), ), ) .then((rows) => rows[0]!); const makeScanDue = async (includeOriginal = true) => { const state = await checkpoint(); await db .update(chatSdkState) .set({ value: { ...(state.value as Record), ...(includeOriginal ? { floor: new Date(Date.now() - 120_000).toISOString() } : {}), nextScanAt: new Date(Date.now() - 1_000).toISOString(), }, }) .where(eq(chatSdkState.id, state.id)); }; const ingress = () => db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.providerActionId, `github_webhook_ingress:${guid}`), ), ) .then((rows) => rows[0]); const receipt = () => db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.providerActionId, `github_webhook_recovery:${guid}`), ), ) .then((rows) => rows[0]); const callback = ( webhookSecret = configured.webhookSecret, incomingPayload: unknown = payload, ) => service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: guid, event, payload: incomingPayload, webhookSecret, url: webhookUrl, }), ); const initializedAt = Date.now(); await service.processFailedGitHubWebhookDeliveries(5, endpoint.id); expect(requests).toEqual([]); const initial = (await checkpoint()).value as Record; expect(initial).toMatchObject({ appId: current.botExternalId, webhookUrl, generation: expect.any(Number), }); expect(Date.parse(String(initial.floor))).toBeGreaterThanOrEqual( initializedAt, ); expect(Date.parse(String(initial.nextScanAt))).toBeGreaterThanOrEqual( initializedAt + 60_000, ); return { ...configured, fixture, body, guid, deliveryId, original, payload, providerRuntime, requests, posts, checkpoint, makeScanDue, ingress, receipt, callback, deferred, setDeliveries(value: typeof deliveries) { deliveries = value; }, setDetail(summary: typeof original, incomingPayload: unknown) { extraDetails.set(summary.id, { summary, payload: incomingPayload }); }, setHistoryHasMore(value: boolean) { historyHasMore = value; }, setCanonicalComment(value: Partial) { canonicalComment = { ...canonicalComment, ...value }; }, setBeforeList(value: typeof beforeList) { beforeList = value; }, setBeforePost(value: typeof beforePost) { beforePost = value; }, }; } it("creates a channel-purpose connection and rejects attempts to change its assigned agent", async () => { const fixture = await seedCompany(); const { service } = createService(); const app = routesApp(db, fixture.companyId, service); const createResponse = await request(app) .post(`/api/companies/${fixture.companyId}/chat-endpoints`) .send({ provider: "slack", assignedAgentId: fixture.assignedAgentId }) .expect(201); expect(createResponse.body).toMatchObject({ provider: "slack", assignedAgentId: fixture.assignedAgentId, assignedAgentName: "Maya", status: "draft", setup: { command: expect.stringMatching(/^\/maya-[a-z0-9]{6}$/) }, }); const createdCommand = createResponse.body.setup.command as string; const [connection] = await db .select() .from(toolConnections) .where(eq(toolConnections.id, createResponse.body.connectionId)); expect(connection).toMatchObject({ connectionPurpose: "channel", transport: "chat_sdk", }); await request(app) .patch(`/api/chat-endpoints/${createResponse.body.id}`) .send({ assignedAgentId: fixture.replacementAgentId }) .expect(400); const [stored] = await db .select({ assignedAgentId: chatEndpoints.assignedAgentId, setup: chatEndpoints.setup, }) .from(chatEndpoints) .where(eq(chatEndpoints.id, createResponse.body.id)); expect(stored.assignedAgentId).toBe(fixture.assignedAgentId); expect(stored.setup.command).toBe(createdCommand); await db .update(agents) .set({ name: "Maya Renamed" }) .where(eq(agents.id, fixture.assignedAgentId)); expect((await service.get(createResponse.body.id)).setup.command).toBe( createdCommand, ); }); it("rejects chat endpoints for non-invokable agents", async () => { const fixture = await seedCompany(); const { service } = createService(); const app = routesApp(db, fixture.companyId, service); for (const status of ["paused", "terminated"] as const) { await db .update(agents) .set({ status }) .where(eq(agents.id, fixture.assignedAgentId)); const response = await request(app) .post(`/api/companies/${fixture.companyId}/chat-endpoints`) .send({ provider: "slack", assignedAgentId: fixture.assignedAgentId }) .expect(422); expect(response.body).toMatchObject({ code: "chat_agent_not_invokable", }); } const endpoints = await db .select({ id: chatEndpoints.id }) .from(chatEndpoints) .where(eq(chatEndpoints.companyId, fixture.companyId)); expect(endpoints).toEqual([]); }); describe("resource change auditing", () => { async function setupResourceAudit() { const fixture = await seedCompany(); const context = createService(); const endpoint = await context.service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId, }, "owner-user", ); const resources = await db .insert(chatEndpointResources) .values([ { companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: "C-AUDIT-FIRST", label: "private-label-not-for-audit", metadata: { credential: "private-resource-metadata" }, enabled: true, availability: "available", }, { companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: "C-AUDIT-SECOND", label: "second", enabled: false, availability: "available", }, ]) .returning(); const audits = () => db .select() .from(activityLog) .where( and( eq(activityLog.companyId, fixture.companyId), eq(activityLog.action, "chat_endpoint.resources_updated"), ), ) .orderBy(asc(activityLog.createdAt)); return { ...context, ...fixture, endpoint, resources, audits, app: routesApp(db, fixture.companyId, context.service), }; } it("records exact actual changes and authenticated route actor, without resource content", async () => { const context = await setupResourceAudit(); const { app, audits, companyId, endpoint, resources, service, wakeup } = context; const events: unknown[] = []; const unsubscribe = subscribeCompanyLiveEvents(companyId, (event) => { if ( event.type === "activity.logged" && event.payload.action === "chat_endpoint.resources_updated" ) events.push(event.payload); }); try { await request(app) .put(`/api/chat-endpoints/${endpoint.id}/resources`) .send({ resources: resources.map((resource) => ({ id: resource.id, enabled: false, })), }) .expect(200); const rows = await audits(); expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ companyId, actorType: "user", actorId: "owner-user", responsibleUserId: "owner-user", entityType: "tool_connection", entityId: endpoint.connectionId, details: { endpointId: endpoint.id, provider: "slack", changes: [ { resourceId: resources[0]!.id, before: { enabled: true }, after: { enabled: false }, }, ], }, }); expect(rows[0]!.details).toEqual({ endpointId: endpoint.id, provider: "slack", changes: [ { resourceId: resources[0]!.id, before: { enabled: true }, after: { enabled: false }, }, ], }); expect(events).toHaveLength(1); expect(JSON.stringify(events)).not.toContain("private-"); await request(app) .put(`/api/chat-endpoints/${endpoint.id}/resources`) .send({ resources: [{ id: resources[0]!.id, enabled: true }], actorUserId: "spoofed", }) .expect(400); await request(app) .put(`/api/chat-endpoints/${endpoint.id}/resources`) .send({ resources: resources.map((resource) => ({ id: resource.id, enabled: false, })), }) .expect(200); await request(app) .put(`/api/chat-endpoints/${endpoint.id}/resources`) .send({ resources: [] }) .expect(200); expect(await audits()).toHaveLength(1); expect(events).toHaveLength(1); expect(wakeup).not.toHaveBeenCalled(); } finally { unsubscribe(); await service.shutdown(); } }); }); describe("resource change audit atomicity", () => { async function fixtureForAudit( overrides: Parameters[2] = {}, ) { const fixture = await seedCompany(); const context = createService(undefined, undefined, overrides); const endpoint = await context.service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId, }, "owner-user", ); const [resource] = await db .insert(chatEndpointResources) .values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: "C-RESOURCE-AUDIT", label: "audit", availability: "available", enabled: true, }) .returning(); const audits = () => db .select() .from(activityLog) .where( and( eq(activityLog.companyId, fixture.companyId), eq(activityLog.action, "chat_endpoint.resources_updated"), ), ) .orderBy(asc(activityLog.createdAt)); const current = () => db .select({ enabled: chatEndpointResources.enabled }) .from(chatEndpointResources) .where(eq(chatEndpointResources.id, resource!.id)); const events: unknown[] = []; const unsubscribe = subscribeCompanyLiveEvents( fixture.companyId, (event) => { if ( event.type === "activity.logged" && event.payload.action === "chat_endpoint.resources_updated" ) events.push(event.payload); }, ); return { ...context, ...fixture, endpoint, resource: resource!, audits, current, events, async cleanup() { unsubscribe(); await context.service.shutdown(); }, }; } it("keeps duplicate last-write semantics but audits only net changes", async () => { const context = await fixtureForAudit(); const { service, endpoint, resource, audits, events } = context; try { await service.replaceResources( endpoint.id, [ { id: resource.id, enabled: false }, { id: resource.id, enabled: true }, ], "owner-user", ); expect(await audits()).toEqual([]); expect(events).toEqual([]); await service.replaceResources( endpoint.id, [ { id: resource.id, enabled: true }, { id: resource.id, enabled: false }, ], "owner-user", ); expect(await context.current()).toEqual([{ enabled: false }]); expect(await audits()).toMatchObject([ { details: { changes: [ { resourceId: resource.id, before: { enabled: true }, after: { enabled: false }, }, ], }, }, ]); expect(events).toHaveLength(1); } finally { await context.cleanup(); } }); it.each(["foreign", "unavailable"] as const)( "rolls back the entire %s resource batch without audit or events", async (kind) => { const context = await fixtureForAudit(); const foreign = await fixtureForAudit(); try { const [other] = await db .insert(chatEndpointResources) .values({ companyId: context.companyId, endpointId: context.endpoint.id, type: "channel", providerResourceId: "C-UNAVAILABLE", label: "unavailable", availability: "unavailable", enabled: false, }) .returning(); await expect( context.service.replaceResources( context.endpoint.id, [ { id: context.resource.id, enabled: false }, ...(kind === "foreign" ? [{ id: foreign.resource.id, enabled: false }] : [ { id: other!.id, enabled: true }, { id: other!.id, enabled: false }, ]), ], "owner-user", ), ).rejects.toMatchObject({ status: kind === "foreign" ? 422 : 409 }); expect(await context.current()).toEqual([{ enabled: true }]); expect(await foreign.current()).toEqual([{ enabled: true }]); expect(await context.audits()).toEqual([]); expect(context.events).toEqual([]); expect(foreign.events).toEqual([]); } finally { await context.cleanup(); await foreign.cleanup(); } }, ); it.each(["audit_insert", "after_audit"] as const)( "rolls back resource writes and suppresses events on %s failure", async (failureAt) => { const context = await fixtureForAudit(); const failure = new Error(`resource-audit-${failureAt}`); const transaction = db.transaction.bind(db); let sawAudit = false; const transactionSpy = vi .spyOn(db, "transaction") .mockImplementation((callback, config) => transaction(async (tx) => { const insert = tx.insert.bind(tx); const insertSpy = vi .spyOn(tx, "insert") .mockImplementation((table) => { if (table === activityLog && failureAt === "audit_insert") { sawAudit = true; throw failure; } return insert(table); }); try { const result = await callback(tx); if (failureAt === "after_audit") { const rows = await tx .select() .from(activityLog) .where( and( eq(activityLog.companyId, context.companyId), eq( activityLog.action, "chat_endpoint.resources_updated", ), ), ); expect(rows).toHaveLength(1); sawAudit = true; expect(context.events).toEqual([]); throw failure; } return result; } finally { insertSpy.mockRestore(); } }, config), ); try { await expect( context.service.replaceResources( context.endpoint.id, [{ id: context.resource.id, enabled: false }], "owner-user", ), ).rejects.toBe(failure); expect(sawAudit).toBe(true); expect(await context.current()).toEqual([{ enabled: true }]); expect(await context.audits()).toEqual([]); expect(context.events).toEqual([]); } finally { transactionSpy.mockRestore(); await context.cleanup(); } }, ); it.each(["inside_transaction", "after_commit"] as const)( "keeps the audit aligned with commit on lease loss %s", async (failureAt) => { let renewals = 0; const context = await fixtureForAudit({ credentialMutationLeaseRenewalIntervalMs: 60_000, renewCredentialMutationLease: async () => ++renewals !== (failureAt === "inside_transaction" ? 2 : 3), }); try { await expect( context.service.replaceResources( context.endpoint.id, [{ id: context.resource.id, enabled: false }], "owner-user", ), ).rejects.toMatchObject({ code: "CHAT_CREDENTIAL_LEASE_LOST" }); const committed = failureAt === "after_commit"; expect(await context.current()).toEqual([{ enabled: !committed }]); expect(await context.audits()).toHaveLength(committed ? 1 : 0); expect(context.events).toHaveLength(committed ? 1 : 0); await context.service.replaceResources( context.endpoint.id, [{ id: context.resource.id, enabled: false }], "owner-user", ); expect(await context.audits()).toHaveLength(1); expect(context.events).toHaveLength(1); } finally { await context.cleanup(); } }, ); it("reads the actual resource state after the row lock, not a pre-lock snapshot", async () => { const context = await fixtureForAudit(); let ready!: (pid: number) => void; const held = new Promise((resolve) => { ready = resolve; }); let release!: () => void; const gate = new Promise((resolve) => { release = resolve; }); const transaction = db.transaction(async (tx) => { await tx .update(chatEndpointResources) .set({ enabled: false }) .where(eq(chatEndpointResources.id, context.resource.id)); const [row] = (await tx.execute( sql`select pg_backend_pid() as pid`, )) as unknown as Array<{ pid: number }>; ready(row!.pid); await gate; }); let mutation: Promise | undefined; try { const pid = await held; mutation = context.service.replaceResources( context.endpoint.id, [{ id: context.resource.id, enabled: false }], "owner-user", ); await vi.waitFor(async () => { const [row] = (await db.execute(sql`select exists ( select 1 from pg_stat_activity where datname = current_database() and ${pid} = any(pg_blocking_pids(pid)) ) as blocked`)) as unknown as Array<{ blocked: boolean }>; expect(row!.blocked).toBe(true); }); release(); await Promise.all([transaction, mutation]); expect(await context.current()).toEqual([{ enabled: false }]); expect(await context.audits()).toEqual([]); expect(context.events).toEqual([]); await context.service.replaceResources( context.endpoint.id, [{ id: context.resource.id, enabled: true }], "owner-user", ); expect(await context.audits()).toMatchObject([ { details: { changes: [ { resourceId: context.resource.id, before: { enabled: false }, after: { enabled: true }, }, ], }, }, ]); } finally { release(); await Promise.allSettled([ transaction, ...(mutation ? [mutation] : []), ]); await context.cleanup(); } }); it("joins concurrent identical and opposite changes into actual before/after history", async () => { const context = await fixtureForAudit(); try { const change = (enabled: boolean) => context.service.replaceResources( context.endpoint.id, [{ id: context.resource.id, enabled }], "owner-user", ); await Promise.all([change(false), change(false)]); expect(await context.audits()).toHaveLength(1); await Promise.all([change(true), change(false)]); const rows = await context.audits(); expect(rows.length).toBeGreaterThanOrEqual(2); let enabled = true; for (const row of rows) { const changes = row.details!.changes as Array<{ resourceId: string; before: { enabled: boolean }; after: { enabled: boolean }; }>; expect(changes).toHaveLength(1); expect(changes[0]!.resourceId).toBe(context.resource.id); expect(changes[0]!.before.enabled).toBe(enabled); expect(changes[0]!.after.enabled).toBe(!enabled); enabled = !enabled; } expect(await context.current()).toEqual([{ enabled }]); expect(context.events).toHaveLength(rows.length); expect(context.wakeup).not.toHaveBeenCalled(); } finally { await context.cleanup(); } }); }); it("requires connection-manager authority for chat connector administration", async () => { const fixture = await seedCompany(); const { service } = createService(); const endpoint = await service.create( fixture.companyId, { provider: "github", assignedAgentId: fixture.assignedAgentId, }, "owner-user", ); const memberUserId = `member-${randomUUID()}`; const now = new Date(); await db.insert(authUsers).values({ id: memberUserId, name: "Ordinary Member", email: `${memberUserId}@example.com`, emailVerified: true, createdAt: now, updatedAt: now, }); await db.insert(companyMemberships).values({ companyId: fixture.companyId, principalType: "user", principalId: memberUserId, status: "active", membershipRole: "member", }); const memberApp = routesApp(db, fixture.companyId, service, memberUserId); await request(memberApp) .get(`/api/chat-endpoints/${endpoint.id}`) .expect(200); await request(memberApp) .get(`/api/companies/${fixture.companyId}/chat-endpoints`) .expect(200); const deniedMutations = [ request(memberApp) .post(`/api/companies/${fixture.companyId}/chat-endpoints`) .send({ provider: "slack", assignedAgentId: fixture.assignedAgentId }), request(memberApp) .patch(`/api/chat-endpoints/${endpoint.id}`) .send({ allowDirectMessages: true }), request(memberApp) .post(`/api/chat-endpoints/${endpoint.id}/setup`) .send({ action: "pause" }), request(memberApp).post( `/api/chat-endpoints/${endpoint.id}/setup-secret`, ), request(memberApp).post(`/api/chat-endpoints/${endpoint.id}/test`), request(memberApp) .put(`/api/chat-endpoints/${endpoint.id}/resources`) .send({ resources: [] }), request(memberApp) .post( `/api/chat-endpoints/${endpoint.id}/principals/${randomUUID()}/link-intent`, ) .send({}), request(memberApp).delete( `/api/chat-endpoints/${endpoint.id}/principals/${randomUUID()}/link`, ), request(memberApp).post( `/api/chat-endpoints/${endpoint.id}/deliveries/${randomUUID()}/replay`, ), request(memberApp).post( `/api/chat-endpoints/${endpoint.id}/publications/${randomUUID()}/replay`, ), request(memberApp) .post( `/api/chat-endpoints/${endpoint.id}/publications/${randomUUID()}/resolve`, ) .send({ action: "cancel" }), request(memberApp) .post( `/api/chat-endpoints/${endpoint.id}/actions/${randomUUID()}/resolve`, ) .send({ action: "cancel" }), ]; for (const mutation of deniedMutations) { const response = await mutation.expect(403); expect(response.body.error).toBe( "Missing permission: tools:manage_connections", ); } const managerApp = routesApp(db, fixture.companyId, service); await request(managerApp) .patch(`/api/chat-endpoints/${endpoint.id}`) .send({ allowDirectMessages: true }) .expect(200) .expect(({ body }) => { expect(body.allowDirectMessages).toBe(true); }); }); it("returns not found rather than revealing another company's chat endpoint", async () => { const viewerCompany = await seedCompany(); const ownerCompany = await seedCompany(); const { service } = createService(); const endpoint = await service.create( ownerCompany.companyId, { provider: "telegram", assignedAgentId: ownerCompany.assignedAgentId, }, "owner-user", ); const app = routesApp(db, viewerCompany.companyId, service); await request(app).get(`/api/chat-endpoints/${endpoint.id}`).expect(404); await request(app) .patch(`/api/chat-endpoints/${endpoint.id}`) .send({ allowDirectMessages: true }) .expect(404); }); it("does not let two live agent connections claim the same native bot or spoof its verified identity", async () => { const fixture = await seedCompany(); const { service } = createService( new FakeChatSdkRuntime(), fakeSlackFetch("U-ONE-NATIVE-BOT") as typeof globalThis.fetch, ); const first = await service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId, }, "owner-user", ); const second = await service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.replacementAgentId, }, "owner-user", ); await service.configure( first.id, { action: "configure", credentials: { botToken: "xoxb-first-agent", signingSecret: "first-signing-secret", }, }, "owner-user", ); await expect( service.configure( second.id, { action: "configure", credentials: { botToken: "xoxb-second-agent", signingSecret: "second-signing-secret", }, }, "owner-user", ), ).rejects.toMatchObject({ status: 409, details: { code: "chat_bot_identity_in_use", endpointId: first.id, assignedAgentId: fixture.assignedAgentId, }, }); const app = routesApp(db, fixture.companyId, service); await request(app) .post(`/api/chat-endpoints/${second.id}/setup`) .send({ action: "configure", providerAccountId: "T-SPOOFED", botExternalId: "U-SPOOFED", credentials: { botToken: "xoxb-spoof", signingSecret: "spoof-signing-secret", }, }) .expect(400); }); it.each([ { provider: "slack" as const, claimedIdentity: { providerAccountId: "T-PAPERCLIP", botExternalId: "U-CLAIMED-BOT", botUsername: "maya-claimed", }, providerFetch: fakeSlackFetch( "U-DIFFERENT-BOT", ) as typeof globalThis.fetch, credentials: { botToken: "xoxb-different-bot", signingSecret: "different-signing-secret", }, }, { provider: "microsoft-teams" as const, claimedIdentity: { providerAccountId: "00000000-0000-4000-8000-000000000191", botExternalId: "00000000-0000-4000-8000-000000000192", botUsername: null, }, providerFetch: (async () => new Response(JSON.stringify({ access_token: "teams-access" }), { status: 200, headers: { "content-type": "application/json" }, })) as typeof globalThis.fetch, credentials: { clientId: "00000000-0000-4000-8000-000000000193", tenantId: "00000000-0000-4000-8000-000000000191", clientSecret: "different-teams-secret", }, }, ])( "keeps a claimed $provider identity immutable when first setup resumes from attention", async ({ provider, claimedIdentity, providerFetch, credentials }) => { const fixture = await seedCompany(); const { service } = createService( new FakeChatSdkRuntime(), providerFetch, ); const endpoint = await service.create( fixture.companyId, { provider, assignedAgentId: fixture.assignedAgentId }, "owner-user", ); // Model a process interruption after the provider identity claim and // before credential refs were committed. Retrying the original // `configure` request must not be able to replace that identity. await db .update(chatEndpoints) .set({ status: "attention", ...claimedIdentity, healthMessage: "Provider setup must be completed", }) .where(eq(chatEndpoints.id, endpoint.id)); await expect( service.configure( endpoint.id, { action: "configure", credentials }, "owner-user", ), ).rejects.toMatchObject({ status: 409, details: { code: "chat_bot_identity_changed" }, }); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "attention", providerAccountId: claimedIdentity.providerAccountId, botExternalId: claimedIdentity.botExternalId, }); await expect( db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)) .then((rows) => rows[0]?.refs ?? []), ).resolves.toEqual([]); await service.shutdown(); }, ); it("globally fences GitHub App identity across owner transfers without leaking another company", async () => { const firstCompany = await seedCompany(); const secondCompany = await seedCompany(); const appRegistrationId = 987654321; const privateKey = generateKeyPairSync("rsa", { modulusLength: 2048 }) .privateKey.export({ type: "pkcs8", format: "pem" }) .toString(); let inventoryRequestCount = 0; let releaseInventory!: () => void; const bothInventoriesStarted = new Promise((resolve) => { releaseInventory = resolve; }); const providerFetch = (owner: string, installationId: number) => (async (input: string | URL | Request) => { const url = String(input); if (url === "https://api.github.com/app") { return new Response( JSON.stringify({ id: appRegistrationId, slug: "shared-paperclip-app", name: "Shared Paperclip App", owner: { login: owner }, permissions: { issues: "write", metadata: "read", pull_requests: "write", }, events: [ "github_app_authorization", "installation", "installation_repositories", "issue_comment", "pull_request_review_comment", ], }), { status: 200, headers: { "content-type": "application/json" } }, ); } if (url === "https://api.github.com/app/installations?per_page=100") { inventoryRequestCount += 1; if (inventoryRequestCount === 2) releaseInventory(); await bothInventoriesStarted; return new Response( JSON.stringify([ { id: installationId, account: { id: installationId, login: owner }, permissions: { issues: "write", metadata: "read", pull_requests: "write", }, suspended_at: null, }, ]), { status: 200, headers: { "content-type": "application/json" } }, ); } if ( url === `https://api.github.com/app/installations/${installationId}/access_tokens` ) { return new Response(JSON.stringify({ token: "installation-token" }), { status: 201, headers: { "content-type": "application/json" }, }); } if ( url === "https://api.github.com/installation/repositories?per_page=100&page=1" ) { return new Response(JSON.stringify({ repositories: [] }), { status: 200, headers: { "content-type": "application/json" }, }); } throw new Error(`Unexpected provider request: ${url}`); }) as typeof globalThis.fetch; const first = createService( new FakeChatSdkRuntime(), providerFetch("original-owner", 8101), ); const second = createService( new FakeChatSdkRuntime(), providerFetch("transferred-owner", 8102), ); const firstEndpoint = await first.service.create( firstCompany.companyId, { provider: "github", assignedAgentId: firstCompany.assignedAgentId, }, "owner-user", ); const secondEndpoint = await second.service.create( secondCompany.companyId, { provider: "github", assignedAgentId: secondCompany.assignedAgentId, }, "owner-user", ); for (const [service, endpoint] of [ [first.service, firstEndpoint], [second.service, secondEndpoint], ] as const) { const { webhookSecret } = await service.generateSetupSecret( endpoint.id, "owner-user", ); await recordGitHubWebhookVerification( service, endpoint.publicId, webhookSecret, ); } const endpoints = [firstEndpoint, secondEndpoint] as const; const services = [first.service, second.service] as const; const outcomes = await Promise.allSettled( services.map((service, index) => service.configure( endpoints[index]!.id, { action: "configure", credentials: { appId: "123456", privateKey }, }, "owner-user", ), ), ); expect( outcomes.filter((outcome) => outcome.status === "fulfilled"), ).toHaveLength(1); const rejectedIndex = outcomes.findIndex( (outcome) => outcome.status === "rejected", ); expect(rejectedIndex).toBeGreaterThanOrEqual(0); const conflict = (outcomes[rejectedIndex] as PromiseRejectedResult).reason; expect(conflict).toMatchObject({ status: 409, details: { code: "chat_bot_identity_in_use", }, }); expect(conflict).not.toMatchObject({ details: { endpointId: expect.any(String), assignedAgentId: expect.any(String), }, }); await expect( db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoints[rejectedIndex]!.connectionId)) .then( (rows) => rows[0]?.refs.map((ref) => ref.configPath).sort() ?? [], ), ).resolves.toEqual(["credentials.webhookSecret"]); await first.service.shutdown(); await second.service.shutdown(); }); it("configures a customer-owned GitHub App and only auto-enables the first addressed setup repository", async () => { const fixture = await seedCompany(); const appId = "123456"; const privateKey = generateKeyPairSync("rsa", { modulusLength: 2048, }) .privateKey.export({ type: "pkcs8", format: "pem" }) .toString(); let observedIssuer: string | null = null; let appRegistrationId = 789; const providerFetch = vi.fn( async (input: string | URL | Request, init?: RequestInit) => { const url = String(input); if (url === "https://api.github.com/app") { const authorization = new Headers(init?.headers).get("authorization"); const token = authorization?.replace(/^Bearer\s+/i, ""); const payload = token?.split(".")[1]; if (!payload) throw new Error("GitHub App JWT was not sent"); observedIssuer = String( ( JSON.parse( Buffer.from(payload, "base64url").toString("utf8"), ) as { iss?: unknown } ).iss, ); return new Response( JSON.stringify({ id: appRegistrationId, slug: "maya-paperclip", name: "Maya Paperclip", owner: { login: "paperclipai" }, permissions: { issues: "write", metadata: "read", pull_requests: "write", }, events: [ "github_app_authorization", "installation", "installation_repositories", "issue_comment", "pull_request_review_comment", ], }), { status: 200, headers: { "content-type": "application/json" } }, ); } if (url === "https://api.github.com/app/installations?per_page=100") { return new Response( JSON.stringify([ { id: 2468, account: { id: 1357, login: "paperclipai", type: "Organization", }, permissions: { issues: "write", metadata: "read", pull_requests: "write", }, suspended_at: null, }, ]), { status: 200, headers: { "content-type": "application/json" } }, ); } if ( url === "https://api.github.com/app/installations/2468/access_tokens" ) { return new Response(JSON.stringify({ token: "installation-token" }), { status: 201, headers: { "content-type": "application/json" }, }); } if ( url === "https://api.github.com/installation/repositories?per_page=100&page=1" ) { return new Response( JSON.stringify({ repositories: [ { id: 97531, full_name: "paperclipai/paperclip", html_url: "https://github.com/paperclipai/paperclip", owner: { id: 1357, login: "paperclipai" }, private: false, }, { id: 97532, full_name: "paperclipai/paperclip-disabled", html_url: "https://github.com/paperclipai/paperclip-disabled", owner: { id: 1357, login: "paperclipai" }, private: false, }, ], }), { status: 200, headers: { "content-type": "application/json" } }, ); } throw new Error(`Unexpected provider request: ${url}`); }, ) as unknown as typeof globalThis.fetch; const { runtime, service } = createService( new FakeChatSdkRuntime(), providerFetch, { scheduleDeferredWork: () => undefined }, ); const endpoint = await service.create( fixture.companyId, { provider: "github", assignedAgentId: fixture.assignedAgentId, }, "owner-user", ); const app = routesApp(db, fixture.companyId, service); const generatedSecretResponse = await request(app) .post(`/api/chat-endpoints/${endpoint.id}/setup-secret`) .send({}) .expect(201); const generatedWebhookSecret = generatedSecretResponse.body .webhookSecret as string; expect(generatedSecretResponse.headers["cache-control"]).toBe("no-store"); expect(generatedWebhookSecret).toMatch(/^[a-f0-9]{64}$/); const endpointAfterGeneration = await request(app) .get(`/api/chat-endpoints/${endpoint.id}`) .expect(200); expect(endpointAfterGeneration.body.setup.webhookSecretConfigured).toBe( true, ); expect(JSON.stringify(endpointAfterGeneration.body)).not.toContain( generatedWebhookSecret, ); await expect( service.configure( endpoint.id, { action: "configure", credentials: { appId, privateKey }, }, "owner-user", ), ).rejects.toMatchObject({ status: 409, details: { code: "chat_webhook_not_verified" }, }); await recordGitHubWebhookVerification( service, endpoint.publicId, generatedWebhookSecret, ); const configured = await service.configure( endpoint.id, { action: "configure", credentials: { appId, privateKey, }, }, "owner-user", ); expect(observedIssuer).toBe(appId); expect(configured).toMatchObject({ status: "verifying", providerAccountId: "paperclipai", botExternalId: "789", botUsername: "maya-paperclip[bot]", setup: { step: "test", webhookVerifiedAt: expect.any(String) }, }); expect( runtime.configurations.get(endpoint.id)?.providerConfig, ).toMatchObject({ provider: "github", credentials: { appId, privateKey, installationId: 2468, webhookSecret: generatedWebhookSecret, }, }); expect( runtime.configurations.get(endpoint.id)?.providerConfig.provider === "github" && runtime.configurations.get(endpoint.id)?.providerConfig.credentials .botUserId, ).toBeUndefined(); appRegistrationId = 790; await expect( service.configure( endpoint.id, { action: "reconnect", credentials: { appId, privateKey }, }, "owner-user", ), ).rejects.toMatchObject({ status: 409, details: { code: "chat_bot_identity_changed" }, }); appRegistrationId = 789; const [connection] = await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(connection.refs.map((ref) => ref.configPath).sort()).toEqual([ "credentials.appId", "credentials.installationId", "credentials.privateKey", "credentials.webhookSecret", ]); const githubResources = await service.listResources(endpoint.id); expect( githubResources.map((resource) => ({ providerResourceId: resource.providerResourceId, availability: resource.availability, enabled: resource.enabled, })), ).toEqual([ { providerResourceId: "paperclipai/paperclip", availability: "available", enabled: false, }, { providerResourceId: "paperclipai/paperclip-disabled", availability: "available", enabled: false, }, ]); const githubCallbacks = runtime.configurations.get(endpoint.id)?.callbacks; if (!githubCallbacks) throw new Error("Expected GitHub callbacks"); await deliverMessage({ callbacks: githubCallbacks, endpointId: endpoint.id, provider: "github", thread: makeThread({ channelId: "PaperclipAI/Paperclip", id: "github:PaperclipAI/Paperclip:issue:17", name: "paperclipai/paperclip", }).thread, message: makeMessage({ id: "github-root-17", text: "@maya triage issue 17", mentioned: true, }), trigger: "mention", }); await expect( db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)), ).resolves.toHaveLength(1); expect( (await service.listResources(endpoint.id)).map((resource) => ({ providerResourceId: resource.providerResourceId, enabled: resource.enabled, })), ).toEqual([ { providerResourceId: "paperclipai/paperclip", enabled: true }, { providerResourceId: "paperclipai/paperclip-disabled", enabled: false, }, ]); await service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: "github-repository-removed", event: "installation_repositories", payload: { action: "removed", repositories_added: [], repositories_removed: [ { id: 97531, full_name: "paperclipai/paperclip", html_url: "https://github.com/paperclipai/paperclip", owner: { id: 1357, login: "paperclipai" }, }, ], }, webhookSecret: generatedWebhookSecret, }), ); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ providerResourceId: "paperclipai/paperclip", availability: "available", enabled: true, }), expect.objectContaining({ providerResourceId: "paperclipai/paperclip-disabled", availability: "available", enabled: false, }), ]); await service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: "github-installation-suspended", event: "installation", payload: { action: "suspend", installation: { id: 2468 }, }, webhookSecret: generatedWebhookSecret, }), ); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "verifying", healthMessage: "Waiting for a test conversation", }); expect(runtime.endpoints.has(endpoint.id)).toBe(true); }); it("admits only an addressed raw GitHub comment as the first setup repository", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, webhookSecret } = await configuredGitHubEndpoint(fixture); const [resource] = await service.listResources(endpoint.id); await service.replaceResources(endpoint.id, [ { id: resource!.id, enabled: false }, ]); const endpointRuntime = runtime.endpoints.get(endpoint.id); const currentEndpoint = await service.get(endpoint.id); if (!endpointRuntime || !currentEndpoint.botUsername) throw new Error("Expected configured GitHub runtime identity"); attachFakeGitHubIssueCommentWebhook({ botUsername: currentEndpoint.botUsername, callbacks, endpointId: endpoint.id, runtime: endpointRuntime, }); const payload = (body: string, commentId: number, issueNumber: number) => ({ action: "created", installation: { id: 2468 }, repository: { id: 97531, full_name: "paperclipai/paperclip", name: "paperclip", owner: { id: 1357, login: "paperclipai" }, }, issue: { number: issueNumber }, comment: { id: commentId, body, created_at: "2026-09-06T12:00:00Z", updated_at: "2026-09-06T12:00:00Z", user: { id: 42, login: "octocat" }, }, sender: { id: 42, login: "octocat" }, }); const unaddressedMarker = `private-unaddressed-${randomUUID()}`; const ignored = await service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: "github-unaddressed-first-setup", event: "issue_comment", payload: payload(unaddressedMarker, 17001, 17), webhookSecret, }), ); expect(ignored.status).toBe(200); expect(endpointRuntime.webhookRequest).toBeNull(); await expect( db.execute( sql`select id from chat_actions where endpoint_id = ${endpoint.id} and payload::text like ${`%${unaddressedMarker}%`}`, ), ).resolves.toHaveLength(0); const botMention = currentEndpoint.botUsername.replace(/\[bot\]$/i, ""); const accepted = await service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: "github-addressed-first-setup", event: "issue_comment", payload: payload(`@${botMention} triage issue 18`, 18001, 18), webhookSecret, }), ); expect(accepted.status).toBe(202); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ providerResourceId: "paperclipai/paperclip", enabled: true, }), ]); await expect(service.listConversations(endpoint.id)).resolves.toHaveLength( 1, ); }); it("atomically admits only one first GitHub repository under concurrent root mentions", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, webhookSecret } = await configuredGitHubEndpoint(fixture); const endpointRuntime = runtime.endpoints.get(endpoint.id); const currentEndpoint = await service.get(endpoint.id); if (!endpointRuntime || !currentEndpoint.botUsername) throw new Error("Expected configured GitHub runtime identity"); attachFakeGitHubIssueCommentWebhook({ botUsername: currentEndpoint.botUsername, callbacks, endpointId: endpoint.id, runtime: endpointRuntime, }); const [firstResource] = await service.listResources(endpoint.id); await service.replaceResources(endpoint.id, [ { id: firstResource!.id, enabled: false }, ]); await db.insert(chatEndpointResources).values({ companyId: endpoint.companyId, endpointId: endpoint.id, type: "repository", providerResourceId: "paperclipai/paperclip-second", label: "paperclipai/paperclip-second", availability: "available", enabled: false, }); const roots = [ { channelId: "paperclipai/paperclip", issueNumber: 901, messageId: 901001, repositoryId: 97531, }, { channelId: "paperclipai/paperclip-second", issueNumber: 902, messageId: 902001, repositoryId: 97532, }, ]; await Promise.all( roots.map((root) => { const [owner, name] = root.channelId.split("/"); return service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: `github-concurrent-root-${root.issueNumber}`, event: "issue_comment", payload: { action: "created", installation: { id: 2468 }, repository: { id: root.repositoryId, full_name: root.channelId, name, owner: { id: 1357, login: owner }, }, issue: { number: root.issueNumber }, comment: { id: root.messageId, body: `@${currentEndpoint.botUsername} investigate this race`, created_at: "2026-09-06T12:00:00Z", updated_at: "2026-09-06T12:00:00Z", user: { id: 42, login: "octocat" }, }, sender: { id: 42, login: "octocat" }, }, webhookSecret, }), ); }), ); const resources = await service.listResources(endpoint.id); expect(resources.filter((resource) => resource.enabled)).toHaveLength(1); const conversations = await db .select({ issueId: chatConversations.issueId }) .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); expect(conversations).toHaveLength(1); const [issue] = await db .select({ title: issues.title }) .from(issues) .where(eq(issues.id, conversations[0]!.issueId)); expect(issue?.title).toBe("investigate this race"); }); it("rejects caller-controlled GitHub setup secrets and arbitrary credential fields", async () => { const fixture = await seedCompany(); const providerFetch = vi.fn(async () => { throw new Error("Credential validation must run before provider access"); }) as unknown as typeof globalThis.fetch; const { service } = createService(new FakeChatSdkRuntime(), providerFetch); const endpoint = await service.create( fixture.companyId, { provider: "github", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); const { webhookSecret } = await service.generateSetupSecret( endpoint.id, "owner-user", ); await recordGitHubWebhookVerification( service, endpoint.publicId, webhookSecret, ); const app = routesApp(db, fixture.companyId, service); await request(app) .post(`/api/chat-endpoints/${endpoint.id}/setup`) .send({ action: "configure", credentials: { appId: "123456", privateKey: "private-key", webhookSecret: "caller-controlled-secret", }, }) .expect(422); await expect( service.configure( endpoint.id, { action: "configure", credentials: { appId: "123456", privateKey: "private-key", installationId: "2468", unexpected: "value", }, }, "owner-user", ), ).rejects.toMatchObject({ status: 422, details: { code: "chat_endpoint_credentials_invalid", provider: "github", action: "configure", unsupportedKeys: ["installationId", "unexpected"], }, }); expect(providerFetch).not.toHaveBeenCalled(); const [connection] = await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(connection!.refs).toEqual([ expect.objectContaining({ configPath: "credentials.webhookSecret" }), ]); }); it("accepts a manually created GitHub App when /app lists only selectable webhook events", async () => { const fixture = await seedCompany(); const privateKey = generateKeyPairSync("rsa", { modulusLength: 2048 }) .privateKey.export({ type: "pkcs8", format: "pem" }) .toString(); const providerFetch = vi.fn(async (input: string | URL | Request) => { const url = String(input); if (url === "https://api.github.com/app") { return new Response( JSON.stringify({ id: 991124, slug: "maya-selectable-events", name: "Maya Selectable Events", owner: { login: "paperclipai" }, permissions: { issues: "write", metadata: "read", pull_requests: "write", }, events: ["issue_comment", "pull_request_review_comment"], }), { status: 200, headers: { "content-type": "application/json" } }, ); } if (url === "https://api.github.com/app/installations?per_page=100") { return new Response( JSON.stringify([ { id: 2468, account: { id: 1357, login: "paperclipai", type: "Organization", }, permissions: { issues: "write", metadata: "read", pull_requests: "write", }, suspended_at: null, }, ]), { status: 200, headers: { "content-type": "application/json" } }, ); } if ( url === "https://api.github.com/app/installations/2468/access_tokens" ) { return new Response( JSON.stringify({ token: "selectable-events-installation-token" }), { status: 201, headers: { "content-type": "application/json" } }, ); } if ( url === "https://api.github.com/installation/repositories?per_page=100&page=1" ) { return new Response( JSON.stringify({ repositories: [ { id: 97531, full_name: "paperclipai/paperclip", html_url: "https://github.com/paperclipai/paperclip", owner: { id: 1357, login: "paperclipai" }, private: false, }, ], }), { status: 200, headers: { "content-type": "application/json" } }, ); } throw new Error(`Unexpected provider request: ${url}`); }) as typeof globalThis.fetch; const { service } = createService(new FakeChatSdkRuntime(), providerFetch); const endpoint = await service.create( fixture.companyId, { provider: "github", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); const { webhookSecret } = await service.generateSetupSecret( endpoint.id, "owner-user", ); await recordGitHubWebhookVerification( service, endpoint.publicId, webhookSecret, ); await expect( service.configure( endpoint.id, { action: "configure", credentials: { appId: "991124", privateKey } }, "owner-user", ), ).resolves.toMatchObject({ status: "verifying", providerAccountId: "paperclipai", botExternalId: "991124", botUsername: "maya-selectable-events[bot]", }); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ providerResourceId: "paperclipai/paperclip", availability: "available", }), ]); await service.shutdown(); }); it("rejects over-scoped GitHub Apps while tolerating unavoidable lifecycle events", async () => { const fixture = await seedCompany(); const privateKey = generateKeyPairSync("rsa", { modulusLength: 2048 }) .privateKey.export({ type: "pkcs8", format: "pem" }) .toString(); const { service } = createService(new FakeChatSdkRuntime(), (async ( input: string | URL | Request, ) => { if (String(input) !== "https://api.github.com/app") { throw new Error(`Unexpected provider request: ${String(input)}`); } return new Response( JSON.stringify({ id: 991123, slug: "maya-over-scoped", name: "Maya Over-scoped", owner: { login: "paperclipai" }, permissions: { contents: "read", issues: "write", metadata: "read", pull_requests: "write", }, events: [ "github_app_authorization", "installation", "installation_repositories", "issue_comment", "pull_request_review_comment", "push", ], }), { status: 200, headers: { "content-type": "application/json" } }, ); }) as typeof globalThis.fetch); const endpoint = await service.create( fixture.companyId, { provider: "github", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); const { webhookSecret } = await service.generateSetupSecret( endpoint.id, "owner-user", ); await recordGitHubWebhookVerification( service, endpoint.publicId, webhookSecret, ); await expect( service.configure( endpoint.id, { action: "configure", credentials: { appId: "991123", privateKey } }, "owner-user", ), ).rejects.toMatchObject({ status: 422, details: { code: "chat_provider_permissions_missing", provider: "github", missingPermissions: [], excessivePermissions: ["contents"], missingEvents: [], excessiveEvents: ["push"], }, }); const [connection] = await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(connection!.refs).toEqual([ expect.objectContaining({ configPath: "credentials.webhookSecret" }), ]); }); it("rejects under-scoped Slack and GitHub apps before saving provider credentials", async () => { const fixture = await seedCompany(); const slack = createService(new FakeChatSdkRuntime(), (async ( input: string | URL | Request, ) => { if (String(input) !== "https://slack.com/api/auth.test") { throw new Error(`Unexpected provider request: ${String(input)}`); } return new Response( JSON.stringify({ ok: true, team_id: "T-UNDER-SCOPED", team: "Under-scoped", user_id: "U-UNDER-SCOPED", user: "maya-under-scoped", }), { status: 200, headers: { "content-type": "application/json", "x-oauth-scopes": "chat:write", }, }, ); }) as typeof globalThis.fetch); const slackEndpoint = await slack.service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await expect( slack.service.configure( slackEndpoint.id, { action: "configure", credentials: { botToken: "xoxb-under-scoped", signingSecret: "signing-secret", }, }, "owner-user", ), ).rejects.toMatchObject({ status: 422, details: { code: "chat_provider_permissions_missing", provider: "slack", }, }); const appId = "991122"; const privateKey = generateKeyPairSync("rsa", { modulusLength: 2048 }) .privateKey.export({ type: "pkcs8", format: "pem" }) .toString(); const github = createService(new FakeChatSdkRuntime(), (async ( input: string | URL | Request, ) => { if (String(input) !== "https://api.github.com/app") { throw new Error(`Unexpected provider request: ${String(input)}`); } return new Response( JSON.stringify({ id: 991122, slug: "maya-under-scoped", name: "Maya Under-scoped", owner: { login: "paperclipai" }, permissions: { issues: "read", metadata: "read", pull_requests: "write", }, events: ["issue_comment"], }), { status: 200, headers: { "content-type": "application/json" } }, ); }) as typeof globalThis.fetch); const githubEndpoint = await github.service.create( fixture.companyId, { provider: "github", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); const { webhookSecret } = await github.service.generateSetupSecret( githubEndpoint.id, "owner-user", ); await recordGitHubWebhookVerification( github.service, githubEndpoint.publicId, webhookSecret, ); await expect( github.service.configure( githubEndpoint.id, { action: "configure", credentials: { appId, privateKey, }, }, "owner-user", ), ).rejects.toMatchObject({ status: 422, details: { code: "chat_provider_permissions_missing", provider: "github", missingPermissions: ["issues"], }, }); const connections = await db .select({ id: toolConnections.id, refs: toolConnections.credentialSecretRefs, }) .from(toolConnections) .where( inArray(toolConnections.id, [ slackEndpoint.connectionId, githubEndpoint.connectionId, ]), ); expect( connections.find( (connection) => connection.id === slackEndpoint.connectionId, )?.refs, ).toEqual([]); expect( connections.find( (connection) => connection.id === githubEndpoint.connectionId, )?.refs, ).toEqual([ expect.objectContaining({ configPath: "credentials.webhookSecret" }), ]); }); it("rotates a live GitHub webhook secret fail-closed and reconnects with stored credentials", async () => { const fixture = await seedCompany(); const { endpoint, runtime, service } = await configuredGitHubEndpoint(fixture); await db .update(chatEndpoints) .set({ status: "active", setup: { step: "complete" } }) .where(eq(chatEndpoints.id, endpoint.id)); const { webhookSecret } = await service.generateSetupSecret( endpoint.id, "owner-user", ); expect(webhookSecret).toMatch(/^[a-f0-9]{64}$/); expect(runtime.endpoints.has(endpoint.id)).toBe(false); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "attention", healthMessage: "Update the GitHub webhook secret, then reconnect this App", setup: { step: "provider_setup", webhookSecretConfigured: true }, }); const [disabledConnection] = await db .select({ status: toolConnections.status, enabled: toolConnections.enabled, healthStatus: toolConnections.healthStatus, }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(disabledConnection).toEqual({ status: "disabled", enabled: false, healthStatus: "degraded", }); await recordGitHubWebhookVerification( service, endpoint.publicId, webhookSecret, ); const reconnected = await service.configure( endpoint.id, { action: "reconnect" }, "owner-user", ); expect(reconnected).toMatchObject({ status: "verifying", setup: { step: "test", webhookSecretConfigured: true }, }); expect( runtime.configurations.get(endpoint.id)?.providerConfig, ).toMatchObject({ provider: "github", credentials: { appId: "123456", webhookSecret, }, }); expect(JSON.stringify(reconnected)).not.toContain(webhookSecret); }); it("reveals a rotated GitHub secret despite a later completion-audit failure and makes retry a new rotation", async () => { const fixture = await seedCompany(); let failedAuditAction: string | null = null; let failCredentialPersistence = false; const setupSecretActivityLogger: typeof logActivity = async ( database, input, publications, ) => { if (failedAuditAction === input.action) { failedAuditAction = null; throw new Error("simulated setup-secret audit outage"); } return logActivity(database, input, publications); }; const setupSecretCredentialPersistBarrier = async () => { if (!failCredentialPersistence) return; failCredentialPersistence = false; throw new Error("simulated setup-secret credential persistence outage"); }; const { endpoint, runtime, service, webhookSecret: originalSecret, } = await configuredGitHubEndpoint(fixture, { setupSecretActivityLogger, setupSecretCredentialPersistBarrier, }); failedAuditAction = "chat_endpoint.setup_secret_rotation_started"; await expect( service.generateSetupSecret(endpoint.id, "owner-user"), ).rejects.toThrow("simulated setup-secret audit outage"); expect(runtime.endpoints.has(endpoint.id)).toBe(true); await expect( githubWebhookVerificationResponse( service, endpoint.publicId, originalSecret, ), ).resolves.toMatchObject({ status: 200, }); const [refsBeforePersistenceFailure] = await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); failCredentialPersistence = true; await expect( service.generateSetupSecret(endpoint.id, "owner-user"), ).rejects.toThrow("simulated setup-secret credential persistence outage"); const [refsAfterPersistenceFailure] = await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(refsAfterPersistenceFailure!.refs).toEqual( refsBeforePersistenceFailure!.refs, ); expect(runtime.endpoints.has(endpoint.id)).toBe(false); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "attention", healthMessage: "Update the GitHub webhook secret, then reconnect this App", setup: { step: "provider_setup", webhookSecretConfigured: true }, }); await expect( githubWebhookVerificationResponse( service, endpoint.publicId, originalSecret, ), ).resolves.toMatchObject({ status: 200, }); failedAuditAction = "chat_endpoint.setup_secret_generated"; const first = await service.generateSetupSecret(endpoint.id, "owner-user"); expect(first.webhookSecret).toMatch(/^[a-f0-9]{64}$/); expect(first.webhookSecret).not.toBe(originalSecret); expect(runtime.endpoints.has(endpoint.id)).toBe(false); await expect( githubWebhookVerificationResponse( service, endpoint.publicId, originalSecret, ), ).resolves.toMatchObject({ status: 401, }); await expect( githubWebhookVerificationResponse( service, endpoint.publicId, first.webhookSecret, ), ).resolves.toMatchObject({ status: 200 }); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "attention", setup: { step: "provider_setup", webhookSecretConfigured: true, }, }); // A caller that lost the successful response may explicitly retry. That // retry is another serialized rotation: only the newly returned value is // valid, so the caller can converge without reusing hidden plaintext. const second = await service.generateSetupSecret(endpoint.id, "owner-user"); expect(second.webhookSecret).toMatch(/^[a-f0-9]{64}$/); expect(second.webhookSecret).not.toBe(first.webhookSecret); await expect( githubWebhookVerificationResponse( service, endpoint.publicId, first.webhookSecret, ), ).resolves.toMatchObject({ status: 401 }); await expect( githubWebhookVerificationResponse( service, endpoint.publicId, second.webhookSecret, ), ).resolves.toMatchObject({ status: 200 }); const auditRows = await db .select({ action: activityLog.action, details: activityLog.details }) .from(activityLog) .where(eq(activityLog.entityId, endpoint.connectionId)); const started = auditRows.filter( (row) => row.action === "chat_endpoint.setup_secret_rotation_started", ); const completed = auditRows.filter( (row) => row.action === "chat_endpoint.setup_secret_generated", ); const failed = auditRows.filter( (row) => row.action === "chat_endpoint.setup_secret_rotation_failed", ); expect(started).toHaveLength(4); expect(completed).toHaveLength(2); expect(failed).toHaveLength(1); const startedRotationIds = started.map((row) => row.details.rotationId); const terminalRotationIds = new Set( [...completed, ...failed].map((row) => row.details.rotationId), ); expect( startedRotationIds.filter( (rotationId) => !terminalRotationIds.has(rotationId), ), ).toHaveLength(1); expect(JSON.stringify(auditRows)).not.toContain(first.webhookSecret); expect(JSON.stringify(auditRows)).not.toContain(second.webhookSecret); }); it("replaces a pre-connect GitHub webhook secret without inventing a broken existing App", async () => { const fixture = await seedCompany(); const { service } = createService(); const endpoint = await service.create( fixture.companyId, { provider: "github", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); const first = await service.generateSetupSecret(endpoint.id, "owner-user"); await recordGitHubWebhookVerification( service, endpoint.publicId, first.webhookSecret, ); const second = await service.generateSetupSecret(endpoint.id, "owner-user"); expect(second.webhookSecret).not.toBe(first.webhookSecret); await expect( githubWebhookVerificationResponse( service, endpoint.publicId, first.webhookSecret, ), ).resolves.toMatchObject({ status: 401 }); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "draft", providerAccountId: null, botExternalId: null, healthMessage: null, setup: { step: "provider_setup", webhookSecretConfigured: true, webhookVerifiedAt: null, }, }); const [connection] = await db .select({ status: toolConnections.status, enabled: toolConnections.enabled, healthStatus: toolConnections.healthStatus, refs: toolConnections.credentialSecretRefs, }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(connection).toMatchObject({ status: "draft", enabled: false, healthStatus: "unchecked", refs: [ expect.objectContaining({ configPath: "credentials.webhookSecret", }), ], }); await recordGitHubWebhookVerification( service, endpoint.publicId, second.webhookSecret, ); const setupActions = await db .select({ action: activityLog.action, details: activityLog.details }) .from(activityLog) .where(eq(activityLog.entityId, endpoint.connectionId)); const generatedActions = setupActions.filter( (item) => item.action === "chat_endpoint.setup_secret_generated", ); expect(generatedActions).toHaveLength(2); expect(generatedActions).toEqual( expect.arrayContaining([ expect.objectContaining({ details: expect.objectContaining({ rotated: false, replacedPrevious: false, }), }), expect.objectContaining({ details: expect.objectContaining({ rotated: false, replacedPrevious: true, }), }), ]), ); expect( setupActions.filter( (item) => item.action === "chat_endpoint.webhook_verified", ), ).toHaveLength(2); }); it("preserves GitHub webhook verification through setup completion and reconnect", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service } = await configuredGitHubEndpoint(fixture); const setupThread = makeThread({ channelId: "paperclipai/paperclip", id: "github:paperclipai/paperclip:issue:811", name: "paperclipai/paperclip", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: setupThread.thread, message: makeMessage({ id: "github-setup-root-811", text: "@maya-paperclip[bot] verify setup state", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); const activated = await service.test(endpoint.id); expect(activated).toMatchObject({ status: "active", setup: { step: "complete", webhookVerifiedAt: expect.any(String), }, }); const verifiedAt = activated.setup.webhookVerifiedAt; const reconnected = await service.configure( endpoint.id, { action: "reconnect" }, "owner-user", ); expect(reconnected).toMatchObject({ status: "verifying", setup: { step: "test", webhookVerifiedAt: verifiedAt, }, }); const [stored] = await db .select({ setup: chatEndpoints.setup }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)); expect(stored!.setup).toMatchObject({ step: "test", webhookVerifiedAt: verifiedAt, runtimeGeneration: expect.any(Number), }); }); it("repairs the same GitHub App webhook on reconnect without claiming a fresh signed ping or chat round trip", async () => { const fixture = await seedCompany(); const { endpoint, service, webhookSecret, webhookSyncRequests } = await configuredGitHubEndpoint(fixture); const before = await service.get(endpoint.id); expect(webhookSyncRequests).toHaveLength(0); const [storedBefore] = await db .select({ setup: chatEndpoints.setup }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)); const refsBefore = await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); const repaired = await service.configure( endpoint.id, { action: "reconnect" }, "owner-user", ); expect(webhookSyncRequests).toEqual([ { url: before.setup.webhookUrl, content_type: "json", insecure_ssl: "0", secret: webhookSecret, }, ]); expect(repaired).toMatchObject({ id: endpoint.id, assignedAgentId: fixture.assignedAgentId, botExternalId: before.botExternalId, status: "verifying", setup: { step: "test", webhookVerifiedAt: before.setup.webhookVerifiedAt, }, }); const [storedRepair] = await db .select({ lastEventAt: chatEndpoints.lastEventAt, setup: chatEndpoints.setup, }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)); expect(storedRepair?.lastEventAt).toBeNull(); expect(storedRepair?.setup?.testStartedAt).not.toBe( storedBefore?.setup?.testStartedAt, ); expect(storedRepair?.setup?.runtimeGeneration).toBe( Number(storedBefore?.setup?.runtimeGeneration) + 1, ); await expect(service.test(endpoint.id)).rejects.toMatchObject({ details: { code: "chat_test_message_missing" }, }); expect( await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)), ).toEqual(refsBefore); const audits = await db .select({ action: activityLog.action, details: activityLog.details }) .from(activityLog) .where(eq(activityLog.entityId, endpoint.connectionId)); expect(audits).toContainEqual({ action: "chat_endpoint.webhook_sync_started", details: { endpointId: endpoint.id, provider: "github" }, }); expect(audits).toContainEqual({ action: "chat_endpoint.webhook_synced", details: { endpointId: endpoint.id, provider: "github" }, }); expect(JSON.stringify({ repaired, audits })).not.toContain(webhookSecret); await service.configure(endpoint.id, { action: "reconnect" }, "owner-user"); expect(webhookSyncRequests).toHaveLength(2); expect(webhookSyncRequests[1]).toEqual(webhookSyncRequests[0]); }); it.each(["http", "transport", "malformed", "mismatch", "oversized"])( "leaves GitHub reconnect unqualified after a %s webhook repair failure", async (failure) => { const fixture = await seedCompany(); const { endpoint, service, runtime, webhookSecret, setWebhookSyncResponse, } = await configuredGitHubEndpoint(fixture); setWebhookSyncResponse(async () => { if (failure === "transport") throw new Error(`private provider echo ${webhookSecret}`); if (failure === "http") return new Response(webhookSecret, { status: 403 }); if (failure === "oversized") return new Response(" ".repeat(32_769)); if (failure === "malformed") return new Response(webhookSecret); return new Response( JSON.stringify({ url: `https://bad.example/${webhookSecret}`, content_type: "json", insecure_ssl: "0", }), ); }); await expect( service.configure(endpoint.id, { action: "reconnect" }, "owner-user"), ).rejects.toMatchObject({ details: { code: "chat_provider_setup_failed" }, }); expect(runtime.endpoints.has(endpoint.id)).toBe(false); const failed = await service.get(endpoint.id); expect(failed).toMatchObject({ status: "attention" }); const audits = await db .select({ action: activityLog.action, details: activityLog.details }) .from(activityLog) .where(eq(activityLog.entityId, endpoint.connectionId)); expect(audits.map((row) => row.action)).toContain( "chat_endpoint.webhook_sync_started", ); expect(audits.map((row) => row.action)).not.toContain( "chat_endpoint.webhook_synced", ); expect(audits.map((row) => row.action)).not.toContain( "chat_endpoint.reconnected", ); expect(JSON.stringify({ failed, audits })).not.toContain(webhookSecret); await expect(service.test(endpoint.id)).rejects.toMatchObject({ details: { code: "chat_endpoint_not_testing" }, }); }, ); it("does not complete GitHub webhook repair or overwrite a new credential lease owner after remote success", async () => { const fixture = await seedCompany(); const { endpoint, service, runtime, setWebhookSyncResponse, webhookSyncRequests, } = await configuredGitHubEndpoint(fixture); setWebhookSyncResponse(async () => { await db .update(chatEndpointLeases) .set({ token: randomUUID() }) .where( and( eq(chatEndpointLeases.endpointId, endpoint.id), eq(chatEndpointLeases.leaseKey, "credentials"), ), ); await db .update(chatEndpoints) .set({ status: "attention", healthMessage: "Winning lease state" }) .where(eq(chatEndpoints.id, endpoint.id)); const config = webhookSyncRequests.at(-1)!; return new Response( JSON.stringify({ url: config.url, content_type: "json", insecure_ssl: "0", }), ); }); await expect( service.configure(endpoint.id, { action: "reconnect" }, "owner-user"), ).rejects.toMatchObject({ code: "CHAT_CREDENTIAL_LEASE_LOST" }); expect(runtime.endpoints.has(endpoint.id)).toBe(false); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "attention", healthMessage: "Winning lease state", }); const audits = await db .select({ action: activityLog.action }) .from(activityLog) .where(eq(activityLog.entityId, endpoint.connectionId)); expect(audits.map((row) => row.action)).not.toContain( "chat_endpoint.webhook_synced", ); expect(audits.map((row) => row.action)).not.toContain( "chat_endpoint.reconnected", ); }); it("serializes concurrent GitHub setup-secret requests without losing stored credentials", async () => { const fixture = await seedCompany(); const { endpoint, runtime, service } = await configuredGitHubEndpoint(fixture); const app = routesApp(db, fixture.companyId, service); const [first, second] = await Promise.all([ request(app) .post(`/api/chat-endpoints/${endpoint.id}/setup-secret`) .send({}) .expect(201), request(app) .post(`/api/chat-endpoints/${endpoint.id}/setup-secret`) .send({}) .expect(201), ]); const rotatedSecrets = [ first.body.webhookSecret as string, second.body.webhookSecret as string, ]; expect(rotatedSecrets[0]).toMatch(/^[a-f0-9]{64}$/); expect(rotatedSecrets[1]).toMatch(/^[a-f0-9]{64}$/); expect(rotatedSecrets[0]).not.toBe(rotatedSecrets[1]); const [connection] = await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(connection!.refs.map((ref) => ref.configPath).sort()).toEqual([ "credentials.appId", "credentials.installationId", "credentials.privateKey", "credentials.webhookSecret", ]); await expect( db .select() .from(activityLog) .where( and( eq(activityLog.entityId, endpoint.connectionId), eq(activityLog.action, "chat_endpoint.setup_secret_generated"), ), ), ).resolves.toHaveLength(3); await expect( db .select() .from(chatEndpointLeases) .where(eq(chatEndpointLeases.endpointId, endpoint.id)), ).resolves.toHaveLength(0); let currentWebhookSecret: string | null = null; for (const candidate of rotatedSecrets) { const response = await githubWebhookVerificationResponse( service, endpoint.publicId, candidate, ); if (response.status === 200) currentWebhookSecret = candidate; } expect(currentWebhookSecret).not.toBeNull(); await service.configure(endpoint.id, { action: "reconnect" }, "owner-user"); const providerConfig = runtime.configurations.get( endpoint.id, )?.providerConfig; expect(providerConfig).toMatchObject({ provider: "github", credentials: { appId: "123456", installationId: 2468, privateKey: expect.stringContaining("BEGIN PRIVATE KEY"), }, }); if (providerConfig?.provider !== "github") throw new Error("Expected GitHub provider configuration"); expect(providerConfig.credentials.webhookSecret).toBe(currentWebhookSecret); }); it("serializes a GitHub secret rotation racing reconnect and preserves the rotated secret", async () => { const fixture = await seedCompany(); const { endpoint, runtime, service } = await configuredGitHubEndpoint(fixture); const app = routesApp(db, fixture.companyId, service); const [rotation, reconnect] = await Promise.all([ request(app) .post(`/api/chat-endpoints/${endpoint.id}/setup-secret`) .send({}) .expect(201), request(app) .post(`/api/chat-endpoints/${endpoint.id}/setup`) .send({ action: "reconnect" }), ]); expect([200, 409]).toContain(reconnect.status); const rotatedSecret = rotation.body.webhookSecret as string; expect(rotatedSecret).toMatch(/^[a-f0-9]{64}$/); // The final state depends on which request acquired the lease first. A // final reconnect must consume the complete, most recent credential set // in either ordering. await recordGitHubWebhookVerification( service, endpoint.publicId, rotatedSecret, ); await service.configure(endpoint.id, { action: "reconnect" }, "owner-user"); const providerConfig = runtime.configurations.get( endpoint.id, )?.providerConfig; expect(providerConfig).toMatchObject({ provider: "github", credentials: { appId: "123456", installationId: 2468, privateKey: expect.stringContaining("BEGIN PRIVATE KEY"), webhookSecret: rotatedSecret, }, }); const [connection] = await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(connection!.refs.map((ref) => ref.configPath).sort()).toEqual([ "credentials.appId", "credentials.installationId", "credentials.privateKey", "credentials.webhookSecret", ]); await expect( db .select() .from(chatEndpointLeases) .where(eq(chatEndpointLeases.endpointId, endpoint.id)), ).resolves.toHaveLength(0); }); it("never lets a GitHub receipt reaction mutate the repository after its runtime credentials are superseded", async () => { const fixture = await seedCompany(); let releaseReaction!: () => void; let markReactionReady!: () => void; const reactionReady = new Promise((resolve) => { markReactionReady = resolve; }); const reactionRelease = new Promise((resolve) => { releaseReaction = resolve; }); const { callbacks, endpoint, runtime, service } = await configuredGitHubEndpoint(fixture, { receiptReactionTransportBarrier: async () => { markReactionReady(); await reactionRelease; }, }); const obsoleteRuntime = runtime.endpoints.get(endpoint.id); if (!obsoleteRuntime) throw new Error("Expected initial GitHub runtime"); const thread = makeThread({ channelId: "github:paperclipai/paperclip", id: "github:paperclipai/paperclip:issue:909", name: "paperclipai/paperclip", }); const inbound = deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id: "90901", text: "@maya rotate credentials before acknowledging this", mentioned: true, }), trigger: "mention", }); await reactionReady; const { webhookSecret } = await service.generateSetupSecret( endpoint.id, "owner-user", ); await recordGitHubWebhookVerification( service, endpoint.publicId, webhookSecret, ); await service.configure(endpoint.id, { action: "reconnect" }, "owner-user"); const currentRuntime = runtime.endpoints.get(endpoint.id); if (!currentRuntime) throw new Error("Expected replacement GitHub runtime"); expect(currentRuntime).not.toBe(obsoleteRuntime); releaseReaction(); await inbound; expect(obsoleteRuntime.reactions).toEqual([]); expect(currentRuntime.reactions).toEqual([]); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "receipt_reaction"), ), ), ).resolves.toEqual([ { status: "cancelled", result: { attempts: 1, code: "receipt_reaction_runtime_superseded", }, }, ]); }); it("filters a durably admitted GitHub callback when rotation and reconnect supersede its runtime before deferred processing", async () => { const fixture = await seedCompany(); const deferred: Array<() => void> = []; const { callbacks, endpoint, runtime, service } = await configuredGitHubEndpoint(fixture, { deferWebhookProcessing: true, scheduleDeferredWork: (task) => deferred.push(task), }); const obsoleteRuntime = runtime.endpoints.get(endpoint.id); if (!obsoleteRuntime) throw new Error("Expected initial GitHub runtime"); const thread = makeThread({ channelId: "github:paperclipai/paperclip", id: "github:paperclipai/paperclip:issue:910", name: "paperclipai/paperclip", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id: "91001", text: "@maya this old callback must not cross the reconnect fence", mentioned: true, }), trigger: "mention", }); expect(deferred).toHaveLength(1); const obsoleteDeliveryDrain = deferred[0]; const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(delivery.normalizedEvent).toMatchObject({ runtimeContext: { credentialFingerprint: expect.any(String), generation: expect.any(Number), }, }); const { webhookSecret } = await service.generateSetupSecret( endpoint.id, "owner-user", ); await recordGitHubWebhookVerification( service, endpoint.publicId, webhookSecret, ); await service.configure(endpoint.id, { action: "reconnect" }, "owner-user"); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date() }) .where(eq(chatDeliveries.id, delivery.id)); await service.processPendingDeliveries(25, delivery.id); await expect( db .select({ state: chatDeliveries.state, error: chatDeliveries.redactedError, }) .from(chatDeliveries) .where(eq(chatDeliveries.id, delivery.id)), ).resolves.toEqual([ { state: "filtered", error: "Connection activation changed before admission", }, ]); await expect( db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)), ).resolves.toHaveLength(0); expect(obsoleteRuntime.reactions).toEqual([]); expect(runtime.endpoints.get(endpoint.id)?.reactions).toEqual([]); // The original delivery drain is still queued. Durable ping admission also // schedules a worker, even though the scoped setup helper has already // processed that exact receipt; neither callback is executed in this race. expect(deferred).toHaveLength(2); expect(deferred[0]).toBe(obsoleteDeliveryDrain); }); it("does not rotate a GitHub webhook secret when stored credentials cannot be resolved", async () => { const fixture = await seedCompany(); const { endpoint, runtime, service } = await configuredGitHubEndpoint(fixture); await db .update(chatEndpoints) .set({ status: "active", setup: { step: "complete" } }) .where(eq(chatEndpoints.id, endpoint.id)); const [connectionBefore] = await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); const appIdRef = connectionBefore!.refs.find( (ref) => ref.configPath === "credentials.appId", ); if (!appIdRef) throw new Error("Expected stored GitHub App ID"); await db .update(companySecrets) .set({ status: "disabled" }) .where(eq(companySecrets.id, appIdRef.secretId)); await expect( service.generateSetupSecret(endpoint.id, "owner-user"), ).rejects.toMatchObject({ status: 422, details: { code: "secret_inactive" }, }); const [connectionAfter] = await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(connectionAfter!.refs).toEqual(connectionBefore!.refs); expect(runtime.endpoints.has(endpoint.id)).toBe(true); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "active", setup: { step: "complete", webhookSecretConfigured: true }, }); }); it("keeps GitHub issues, PR conversations, and inline review threads on distinct tasks", async () => { const fixture = await seedCompany(); const { callbacks, endpoint } = await configuredGitHubEndpoint(fixture); const cases = [ { id: "github:paperclipai/paperclip:issue:51", rootId: "51001", }, { id: "github:paperclipai/paperclip:52", rootId: "52001" }, { id: "github:paperclipai/paperclip:52:rc:88001", rootId: "88001", }, ]; for (const item of cases) { const thread = makeThread({ channelId: "github:paperclipai/paperclip", id: item.id, name: "paperclipai/paperclip", }); const rootDelivery = { callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id: item.rootId, text: `@maya handle ${item.id}`, mentioned: true, }), trigger: "mention", } as const; await deliverMessage(rootDelivery); if (item === cases[0]) await deliverMessage(rootDelivery); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id: `${item.rootId}-reply`, text: "Unmentioned follow-up", }), trigger: "subscribed_message", }); } const conversations = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); expect(conversations).toHaveLength(3); expect(new Set(conversations.map((row) => row.issueId)).size).toBe(3); expect(conversations.map((row) => row.externalThreadId).sort()).toEqual( cases.map((item) => item.id).sort(), ); const deliveries = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(deliveries).toHaveLength(6); expect(deliveries.every((row) => row.state === "processed")).toBe(true); expect( deliveries.find((row) => row.providerEventId.endsWith(":51001")), ).toMatchObject({ normalizedEvent: { deduplication: { duplicateCount: 1 } }, }); expect( deliveries.filter( (row) => row.normalizedEvent.trigger === "subscribed_message", ), ).toHaveLength(3); }); it("uses GitHub numeric user ids for linked authority and rechecks membership on every follow-up", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service, wakeup } = await configuredGitHubEndpoint(fixture); await service.update( endpoint.id, { allowUnlinkedPeople: false }, "owner-user", ); const githubUserId = "18446744073709551"; const thread = makeThread({ channelId: "paperclipai/paperclip", id: "github:paperclipai/paperclip:issue:59", name: "paperclipai/paperclip", }); const send = ( id: string, text: string, trigger: "mention" | "subscribed_message", ) => deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id, text, mentioned: trigger === "mention", userId: githubUserId, userName: "octocat-renamable", }), trigger, }); await send("59001", "@maya attempt before identity linking", "mention"); await send("59001", "@maya attempt before identity linking", "mention"); expect(await service.listConversations(endpoint.id)).toEqual([]); const [principal] = await db .select() .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, fixture.companyId), eq(chatExternalPrincipals.provider, "github"), eq(chatExternalPrincipals.externalId, githubUserId), ), ); expect(principal).toMatchObject({ externalId: githubUserId }); const [filtered] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(filtered).toMatchObject({ state: "filtered", redactedError: "External identity must be linked to a Paperclip account", normalizedEvent: { deduplication: { duplicateCount: 1 } }, }); const linkedUserId = `github-user-${randomUUID()}`; const now = new Date(); await db.insert(authUsers).values({ id: linkedUserId, name: "GitHub Linked User", email: `${linkedUserId}@example.com`, emailVerified: true, createdAt: now, updatedAt: now, }); await db.insert(companyMemberships).values({ companyId: fixture.companyId, principalType: "user", principalId: linkedUserId, status: "active", membershipRole: "operator", }); const intent = await service.createLinkIntent( endpoint.id, principal!.id, 1_800, ); const token = new URL(intent.confirmationUrl).searchParams.get("token"); if (!token) throw new Error("GitHub identity-link confirmation token was absent"); await service.confirmIdentityLink(token, linkedUserId); await send("59002", "@maya authorized GitHub task", "mention"); const [conversation] = await service.listConversations(endpoint.id); expect(conversation).toBeDefined(); await expect( db .select({ authorUserId: issueComments.authorUserId, body: issueComments.body, }) .from(issueComments) .where(eq(issueComments.issueId, conversation!.issueId)), ).resolves.toEqual([ { authorUserId: linkedUserId, body: "@maya authorized GitHub task", }, ]); expect(wakeup).toHaveBeenCalledTimes(1); await db .update(companyMemberships) .set({ status: "suspended" }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalId, linkedUserId), ), ); await send( "59003", "This follow-up must be rejected now", "subscribed_message", ); await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, conversation!.issueId)), ).resolves.toEqual([{ body: "@maya authorized GitHub task" }]); expect(wakeup).toHaveBeenCalledTimes(1); const suspendedDelivery = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.providerEventId, `${thread.thread.id}:59003`), ), ) .then((rows) => rows[0]); expect(suspendedDelivery).toMatchObject({ state: "filtered", redactedError: "Linked Paperclip account is not currently permitted", }); }); it("reorders same-second GitHub callbacks by comment id before starting the task", async () => { const fixture = await seedCompany(); const deferred: Array<() => void> = []; const { callbacks, endpoint, service, wakeup } = await configuredGitHubEndpoint(fixture, { deferWebhookProcessing: true, scheduleDeferredWork: (task) => deferred.push(task), }); const thread = makeThread({ channelId: "github:paperclipai/paperclip", id: "github:paperclipai/paperclip:issue:71", name: "paperclipai/paperclip", }); const providerSentAt = new Date("2026-09-05T18:00:00.000Z"); const laterReply = makeMessage({ id: "71002", text: "unmentioned follow-up delivered first", }); laterReply.metadata.dateSent = providerSentAt; await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: laterReply, trigger: "unaddressed_message", }); const earlierMention = makeMessage({ id: "71001", text: "@maya start the GitHub task", mentioned: true, }); earlierMention.metadata.dateSent = providerSentAt; await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: earlierMention, trigger: "mention", }); const durable = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(durable).toHaveLength(2); expect(durable.every((delivery) => delivery.nextAttemptAt !== null)).toBe( true, ); expect( new Set( durable.map((delivery) => delivery.nextAttemptAt?.getTime() ?? null), ).size, ).toBe(1); expect(deferred).toHaveLength(1); deferred.shift()?.(); await vi.waitFor(async () => { const rows = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); expect(rows).toHaveLength(1); }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); await vi.waitFor(async () => { const rows = await db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, conversation!.issueId)) .orderBy(asc(issueComments.createdAt), asc(issueComments.id)); expect(rows.map((row) => row.body)).toEqual([ "@maya start the GitHub task", "unmentioned follow-up delivered first", ]); expect(wakeup).toHaveBeenCalledTimes(2); }); await service.shutdown(); }); it("holds a GitHub follow-up that arrives after the reorder window until its older root mention arrives", async () => { const fixture = await seedCompany(); const deferred: Array<() => void> = []; const { callbacks, endpoint, service, wakeup } = await configuredGitHubEndpoint(fixture, { deferWebhookProcessing: true, scheduleDeferredWork: (task) => deferred.push(task), }); const thread = makeThread({ channelId: "github:paperclipai/paperclip", id: "github:paperclipai/paperclip:issue:72", name: "paperclipai/paperclip", }); const laterReply = makeMessage({ id: "72002", text: "follow-up delivered well before its root callback", }); laterReply.metadata.dateSent = new Date("2026-09-05T18:00:02.000Z"); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: laterReply, trigger: "unaddressed_message", }); const [replyDelivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date() }) .where(eq(chatDeliveries.id, replyDelivery!.id)); await service.processPendingDeliveries(25, replyDelivery!.id); await expect( db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, replyDelivery!.id)), ).resolves.toEqual([ expect.objectContaining({ state: "retry", attempts: 1, redactedError: "Waiting briefly for an earlier root mention", }), ]); await expect( db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)), ).resolves.toHaveLength(0); expect(wakeup).not.toHaveBeenCalled(); const earlierMention = makeMessage({ id: "72001", text: "@maya start the delayed GitHub task", mentioned: true, }); earlierMention.metadata.dateSent = new Date("2026-09-05T18:00:01.000Z"); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: earlierMention, trigger: "mention", }); const mentionDelivery = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.providerEventId, `${thread.thread.id}:72001`), ), ) .then((rows) => rows[0]); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date() }) .where(eq(chatDeliveries.id, mentionDelivery!.id)); await service.processPendingDeliveries(25, mentionDelivery!.id); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); expect(conversation).toBeDefined(); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date() }) .where(eq(chatDeliveries.id, replyDelivery!.id)); await service.processPendingDeliveries(25, replyDelivery!.id); const comments = await db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, conversation!.issueId)) .orderBy(asc(issueComments.createdAt), asc(issueComments.id)); expect(comments.map((comment) => comment.body)).toEqual([ "@maya start the delayed GitHub task", "follow-up delivered well before its root callback", ]); expect(wakeup).toHaveBeenCalledTimes(2); expect(deferred).toHaveLength(1); await service.shutdown(); }); it("filters a standalone unaddressed GitHub comment after bounded orphan retention", async () => { const fixture = await seedCompany(); const deferred: Array<() => void> = []; const { callbacks, endpoint, service, wakeup } = await configuredGitHubEndpoint(fixture, { deferWebhookProcessing: true, scheduleDeferredWork: (task) => deferred.push(task), }); const thread = makeThread({ channelId: "github:paperclipai/paperclip", id: "github:paperclipai/paperclip:issue:73", name: "paperclipai/paperclip", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id: "73001", text: "ordinary comment that never mentions the agent", }), trigger: "unaddressed_message", }); const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); for (let expectedAttempt = 1; expectedAttempt <= 13; expectedAttempt += 1) { await db .update(chatDeliveries) .set({ nextAttemptAt: new Date() }) .where(eq(chatDeliveries.id, delivery!.id)); await service.processPendingDeliveries(25, delivery!.id); const current = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, delivery!.id)) .then((rows) => rows[0]); expect(current).toMatchObject({ state: expectedAttempt <= 12 ? "retry" : "filtered", attempts: expectedAttempt, }); } await expect( db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)), ).resolves.toHaveLength(0); expect(wakeup).not.toHaveBeenCalled(); expect(deferred).toHaveLength(1); await service.shutdown(); }); it("revokes a deleted canonical GitHub installation and recovers the same endpoint after reinstall", async () => { const fixture = await seedCompany(); const context = await configuredGitHubEndpoint(fixture); const { callbacks, endpoint, runtime, service, webhookSecret } = context; const thread = makeThread({ channelId: "github:paperclipai/paperclip", id: "github:paperclipai/paperclip:issue:61", name: "paperclipai/paperclip", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id: "61001", text: "@maya establish the recoverable thread", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const credentialRefsBeforeDeletion = await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)) .then((rows) => rows[0]!.refs); context.setInstallationAvailable(false); const deleted = await service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: "github-installation-deleted", event: "installation", payload: { action: "deleted", installation: { id: 2468 }, }, webhookSecret, }), ); expect(deleted.status).toBe(202); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "revoked", healthMessage: "GitHub App installation was removed", }); await expect( db .select({ status: toolConnections.status, enabled: toolConnections.enabled, healthStatus: toolConnections.healthStatus, refs: toolConnections.credentialSecretRefs, }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)), ).resolves.toEqual([ { status: "disabled", enabled: false, healthStatus: "failed", refs: credentialRefsBeforeDeletion, }, ]); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ availability: "unavailable", enabled: true }), ]); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ id: conversation!.id, state: "unavailable" }), ]); await expect( db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, "lifecycle:github-installation-deleted", ), ), ), ).resolves.toEqual([{ state: "processed" }]); expect(runtime.endpoints.has(endpoint.id)).toBe(false); context.setInstallationId(8642); context.setInstallationAvailable(true); await service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: "github-installation-recreated", event: "installation", payload: { action: "created", installation: { id: 8642 }, }, webhookSecret, }), ); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "active", healthMessage: "Connected", }); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ providerResourceId: "paperclipai/paperclip", availability: "available", enabled: true, }), ]); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ id: conversation.id, state: "active" }), ]); expect(runtime.endpoints.has(endpoint.id)).toBe(false); const publication = await service.publishBoardMessage( endpoint.id, conversation.id, "Recovered GitHub response", "recovered-github-response", "owner-user", ); expect( runtime.configurations.get(endpoint.id)?.providerConfig, ).toMatchObject({ provider: "github", credentials: { installationId: 8642 }, }); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.id, publication!.id)), ).resolves.toEqual([expect.objectContaining({ state: "published" })]); }); it("fails closed when GitHub App permissions drift before installation recovery", async () => { const fixture = await seedCompany(); const context = await configuredGitHubEndpoint(fixture); const { callbacks, endpoint, runtime, service, webhookSecret } = context; const thread = makeThread({ channelId: "github:paperclipai/paperclip", id: "github:paperclipai/paperclip:issue:62", name: "paperclipai/paperclip", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id: "62001", text: "@maya establish the permission drift thread", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); await service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: "github-installation-suspended-before-drift", event: "installation", payload: { action: "suspend", installation: { id: 2468 }, }, webhookSecret, }), ); context.setAppAccess({ permissions: { issues: "read", metadata: "read", pull_requests: "write", }, }); await expect( service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: "github-installation-unsuspended-with-drift", event: "installation", payload: { action: "unsuspend", installation: { id: 2468 }, }, webhookSecret, }), ), ).rejects.toMatchObject({ status: 422, details: { code: "chat_provider_permissions_missing", provider: "github", missingPermissions: ["issues"], }, }); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "attention", healthMessage: "GitHub App credentials, permissions, events, or identity need attention", lastError: expect.stringContaining("issues"), }); const [connection] = await db .select({ status: toolConnections.status, enabled: toolConnections.enabled, healthStatus: toolConnections.healthStatus, }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(connection).toEqual({ status: "disabled", enabled: false, healthStatus: "degraded", }); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ availability: "unavailable", enabled: true }), ]); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ id: conversation!.id, state: "unavailable" }), ]); expect(runtime.endpoints.has(endpoint.id)).toBe(false); const lifecycleDelivery = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, "lifecycle:github-installation-unsuspended-with-drift", ), ), ) .then((rows) => rows[0]); expect(lifecycleDelivery).toMatchObject({ state: "retry", redactedError: expect.stringContaining("issues"), }); }); it("fails closed when the GitHub installation has not approved the App permissions", async () => { const fixture = await seedCompany(); const context = await configuredGitHubEndpoint(fixture); const { callbacks, endpoint, service, webhookSecret } = context; const thread = makeThread({ channelId: "github:paperclipai/paperclip", id: "github:paperclipai/paperclip:issue:63", name: "paperclipai/paperclip", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id: "63001", text: "@maya establish the installation permission thread", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); context.setInstallationAccess({ issues: "read", metadata: "read", pull_requests: "write", }); await expect( service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: "github-installation-permission-approval-pending", event: "installation", payload: { action: "new_permissions_accepted", installation: { id: 2468 }, }, webhookSecret, }), ), ).rejects.toMatchObject({ status: 422, details: { code: "chat_provider_inventory_failed" }, }); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "attention", healthMessage: "GitHub App credentials, permissions, events, or identity need attention", lastError: expect.stringContaining( "active installation has not granted the required access for: issues", ), }); const lifecycleDelivery = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, "lifecycle:github-installation-permission-approval-pending", ), ), ) .then((rows) => rows[0]); expect(lifecycleDelivery).toMatchObject({ state: "retry", redactedError: expect.stringContaining("issues"), }); }); it("reconciles concurrent opposite GitHub lifecycle callbacks from canonical App state", async () => { const fixture = await seedCompany(); const context = await configuredGitHubEndpoint(fixture); const { endpoint, runtime, service, webhookSecret } = context; const repository = { id: 97531, full_name: "paperclipai/paperclip", html_url: "https://github.com/paperclipai/paperclip", owner: { id: 1357, login: "paperclipai" }, private: false, }; const send = (delivery: string, event: string, payload: unknown) => service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery, event, payload, webhookSecret, }), ); context.setRepositories([]); await send( "github-repositories-canonical-empty", "installation_repositories", { action: "removed", repositories_added: [], repositories_removed: [repository], }, ); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ providerResourceId: "paperclipai/paperclip", availability: "removed", enabled: true, }), ]); context.setRepositories([repository]); await Promise.all([ send("github-installation-late-suspend", "installation", { action: "suspend", installation: { id: 2468 }, }), send("github-installation-current-unsuspend", "installation", { action: "unsuspend", installation: { id: 2468 }, }), send("github-repositories-late-remove", "installation_repositories", { action: "removed", repositories_added: [], repositories_removed: [repository], }), send("github-repositories-current-add", "installation_repositories", { action: "added", repositories_added: [repository], repositories_removed: [], }), ]); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "verifying", healthMessage: "Waiting for a test conversation", }); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ providerResourceId: "paperclipai/paperclip", availability: "available", enabled: true, }), ]); expect(runtime.endpoints.has(endpoint.id)).toBe(true); }); it("does not let GitHub lifecycle recovery undo a webhook-secret rotation", async () => { const fixture = await seedCompany(); const { endpoint, runtime, service, webhookSecret: oldWebhookSecret, } = await configuredGitHubEndpoint(fixture); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected GitHub runtime"); let releaseWebhook!: () => void; let markWebhookEntered!: () => void; const webhookEntered = new Promise((resolve) => { markWebhookEntered = resolve; }); const webhookRelease = new Promise((resolve) => { releaseWebhook = resolve; }); providerRuntime.webhookHook = async () => { markWebhookEntered(); await webhookRelease; }; const body = JSON.stringify({ action: "unsuspend", installation: { id: 2468 }, }); const signature = createHmac("sha256", oldWebhookSecret) .update(body) .digest("hex"); const staleCallback = service.handleWebhook( endpoint.publicId, "github", new Request("https://paperclip.example/github", { method: "POST", headers: { "content-type": "application/json", "x-github-event": "installation", "x-github-delivery": "github-available-during-secret-rotation", "x-hub-signature-256": `sha256=${signature}`, }, body, }), ); await webhookEntered; await service.generateSetupSecret(endpoint.id, "owner-user"); releaseWebhook(); await expect(staleCallback).resolves.toMatchObject({ status: 202 }); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "attention", setup: { step: "provider_setup", webhookVerifiedAt: null }, }); const [connection] = await db .select() .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(connection).toMatchObject({ status: "disabled", enabled: false, healthStatus: "degraded", }); expect(runtime.endpoints.has(endpoint.id)).toBe(false); const [delivery] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, "lifecycle:github-available-during-secret-rotation", ), ), ); expect(delivery).toMatchObject({ state: "filtered", redactedError: "Provider lifecycle callback belonged to a superseded runtime", }); }); it("does not let an obsolete GitHub webhook secret reopen a repository during rotation", async () => { const fixture = await seedCompany(); let blockWebhookAuthentication = false; let blockCredentialPersistence = false; let markWebhookReady!: () => void; let releaseWebhook!: () => void; let markRotationReady!: () => void; let releaseRotation!: () => void; const webhookReady = new Promise((resolve) => { markWebhookReady = resolve; }); const webhookRelease = new Promise((resolve) => { releaseWebhook = resolve; }); const rotationReady = new Promise((resolve) => { markRotationReady = resolve; }); const rotationRelease = new Promise((resolve) => { releaseRotation = resolve; }); const { endpoint, service, webhookSecret: oldWebhookSecret, } = await configuredGitHubEndpoint(fixture, { githubWebhookAuthenticationBarrier: async () => { if (!blockWebhookAuthentication) return; markWebhookReady(); await webhookRelease; }, setupSecretCredentialPersistBarrier: async () => { if (!blockCredentialPersistence) return; markRotationReady(); await rotationRelease; }, }); blockWebhookAuthentication = true; const staleCallback = service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: "github-repository-during-secret-rotation", event: "issue_comment", payload: { action: "created", installation: { id: 2468 }, repository: { id: 86420, full_name: "paperclipai/stale-secret-repository", html_url: "https://github.com/paperclipai/stale-secret-repository", owner: { id: 1357, login: "paperclipai" }, private: false, }, issue: { number: 7 }, comment: { id: 7001, body: "obsolete credential callback" }, sender: { id: 42, login: "octocat" }, }, webhookSecret: oldWebhookSecret, }), ); await webhookReady; blockCredentialPersistence = true; const rotation = service.generateSetupSecret(endpoint.id, "owner-user"); await rotationReady; releaseWebhook(); releaseRotation(); await rotation; // The callback authenticated before rotation won the mutation lease, but // its repository was never enabled. It may be rejected as obsolete or // acknowledged as out of scope; either way it must not mutate inventory. await expect(staleCallback).resolves.toMatchObject({ status: 200 }); await expect(service.listResources(endpoint.id)).resolves.not.toEqual( expect.arrayContaining([ expect.objectContaining({ providerResourceId: "paperclipai/stale-secret-repository", }), ]), ); }); it("acknowledges GitHub lifecycle callbacks without changing a paused endpoint", async () => { const fixture = await seedCompany(); const { endpoint, service } = await configuredGitHubEndpoint(fixture); const current = await db .select({ setup: chatEndpoints.setup }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)) .then((rows) => rows[0]!); await db .update(chatEndpoints) .set({ status: "active", setup: { ...current.setup, step: "complete" } }) .where(eq(chatEndpoints.id, endpoint.id)); await service.configure(endpoint.id, { action: "pause" }, "owner-user"); const before = await db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)) .then((rows) => rows.length); const response = await service.handleWebhook( endpoint.publicId, "github", new Request("https://paperclip.example/github", { method: "POST", headers: { "content-type": "application/json", "x-github-event": "installation", "x-github-delivery": "github-available-while-paused", }, body: JSON.stringify({ action: "unsuspend", installation: { id: 2468 }, }), }), ); expect(response.status).toBe(200); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "paused", }); await expect( db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)) .then((rows) => rows.length), ).resolves.toBe(before); }); it("reconciles a foreign GitHub installation lifecycle event against canonical App inventory", async () => { const fixture = await seedCompany(); const { endpoint, runtime, service, webhookSecret } = await configuredGitHubEndpoint(fixture); const setup = await db .select({ setup: chatEndpoints.setup }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)) .then((rows) => rows[0]!.setup); await db .update(chatEndpoints) .set({ status: "active", setup: { ...setup, step: "complete" } }) .where(eq(chatEndpoints.id, endpoint.id)); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected GitHub runtime"); providerRuntime.webhookRequest = null; const payload = JSON.stringify({ action: "deleted", installation: { id: 9999 }, }); const signature = createHmac("sha256", webhookSecret) .update(payload) .digest("hex"); const response = await service.handleWebhook( endpoint.publicId, "github", new Request("https://paperclip.example/github", { method: "POST", headers: { "content-type": "application/json", "x-github-event": "installation", "x-github-delivery": "foreign-installation-deleted", "x-hub-signature-256": `sha256=${signature}`, }, body: payload, }), ); expect(response.status).toBe(202); expect(providerRuntime.webhookRequest).not.toBeNull(); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "active", setup: { step: "complete" }, }); await expect( db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)), ).resolves.toEqual([ expect.objectContaining({ providerEventId: "lifecycle:foreign-installation-deleted", state: "processed", }), ]); }); it("quarantines an active GitHub endpoint when its dedicated App drifts to multiple installations", async () => { const fixture = await seedCompany(); const context = await configuredGitHubEndpoint(fixture); const { endpoint, runtime, service, webhookSecret } = context; const setup = await db .select({ setup: chatEndpoints.setup }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)) .then((rows) => rows[0]!.setup); await db .update(chatEndpoints) .set({ status: "active", setup: { ...setup, step: "complete" } }) .where(eq(chatEndpoints.id, endpoint.id)); context.setAdditionalInstallationIds([9999]); await expect( service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: "github-second-installation-created", event: "installation", payload: { action: "created", installation: { id: 9999 }, }, webhookSecret, }), ), ).rejects.toMatchObject({ status: 422, details: { code: "chat_provider_inventory_failed" }, }); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "attention", healthMessage: "GitHub App credentials, permissions, events, or identity need attention", lastError: expect.stringContaining("exactly one active installation"), }); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ providerResourceId: "paperclipai/paperclip", availability: "unavailable", }), ]); expect(runtime.endpoints.has(endpoint.id)).toBe(false); const [failedLifecycle] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, "lifecycle:github-second-installation-created", ), ), ); expect(failedLifecycle).toMatchObject({ state: "retry", redactedError: expect.stringContaining("exactly one active installation"), }); context.setAdditionalInstallationIds([]); const recovered = await service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: "github-second-installation-removed", event: "installation", payload: { action: "deleted", installation: { id: 9999 }, }, webhookSecret, }), ); expect(recovered.status).toBe(202); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "active", healthMessage: "Connected", }); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ providerResourceId: "paperclipai/paperclip", availability: "available", }), ]); }); it("accepts only signed GitHub setup pings before App credentials exist", async () => { const fixture = await seedCompany(); const runtime = new FakeChatSdkRuntime(); const { service } = createService(runtime); const endpoint = await service.create( fixture.companyId, { provider: "github", assignedAgentId: fixture.assignedAgentId, name: "Maya GitHub setup ping", }, "owner-user", ); const { webhookSecret } = await service.generateSetupSecret( endpoint.id, "owner-user", ); const body = JSON.stringify({ zen: "Keep it logically awesome." }); const signature = createHmac("sha256", webhookSecret) .update(body) .digest("hex"); const setupPing = (signatureHeader?: string) => service.handleWebhook( endpoint.publicId, "github", new Request("https://paperclip.example/github", { method: "POST", headers: { "content-type": "application/json", "x-github-event": "ping", ...(signatureHeader ? { "x-hub-signature-256": signatureHeader } : {}), }, body, }), ); await expect(setupPing()).resolves.toMatchObject({ status: 401 }); await expect(setupPing("sha256=invalid")).resolves.toMatchObject({ status: 401, }); const accepted = await setupPing(`sha256=${signature}`); expect(accepted.status).toBe(200); await expect(accepted.text()).resolves.toBe("pong"); expect(runtime.configurations.has(endpoint.id)).toBe(false); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "draft", healthMessage: "GitHub webhook verified", setup: { webhookSecretConfigured: true, webhookVerifiedAt: expect.any(String), }, }); }); it("ignores a signed GitHub installation before App credentials exist", async () => { const fixture = await seedCompany(); const runtime = new FakeChatSdkRuntime(); const { service } = createService(runtime); const endpoint = await service.create( fixture.companyId, { provider: "github", assignedAgentId: fixture.assignedAgentId, name: "Maya GitHub pre-key installation", }, "owner-user", ); const { webhookSecret } = await service.generateSetupSecret( endpoint.id, "owner-user", ); await recordGitHubWebhookVerification( service, endpoint.publicId, webhookSecret, ); const endpointBeforeInstallation = await service.get(endpoint.id); const webhookVerifiedAt = endpointBeforeInstallation.setup?.webhookVerifiedAt; expect(webhookVerifiedAt).toEqual(expect.any(String)); const [connectionBeforeInstallation] = await db .select({ enabled: toolConnections.enabled, refs: toolConnections.credentialSecretRefs, status: toolConnections.status, }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); const deliveryId = `github-pre-key-installation-${randomUUID()}`; const response = await service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: deliveryId, event: "installation", payload: { action: "created", installation: { id: 987_654_321, account: { id: 1, login: "paperclip-e2e", type: "User" }, permissions: { issues: "write", metadata: "read", pull_requests: "write", }, suspended_at: null, }, }, webhookSecret, }), ); expect(response.status).toBe(200); await expect(response.text()).resolves.toBe("ignored"); expect(runtime.configurations.has(endpoint.id)).toBe(false); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "draft", providerAccountId: null, botExternalId: null, healthMessage: "GitHub webhook verified", setup: { step: "provider_setup", webhookSecretConfigured: true, webhookVerifiedAt, }, }); await expect(service.listResources(endpoint.id)).resolves.toEqual([]); await expect(service.listConversations(endpoint.id)).resolves.toEqual([]); await expect( db .select({ id: chatActions.id }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "github_webhook_ingress"), eq( chatActions.providerActionId, `github_webhook_ingress:${deliveryId}`, ), ), ), ).resolves.toEqual([]); const [connectionAfterInstallation] = await db .select({ enabled: toolConnections.enabled, refs: toolConnections.credentialSecretRefs, status: toolConnections.status, }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(connectionAfterInstallation).toEqual(connectionBeforeInstallation); expect(connectionAfterInstallation).toMatchObject({ enabled: false, refs: [ expect.objectContaining({ configPath: "credentials.webhookSecret", }), ], status: "draft", }); }); it("singleflights concurrent cold GitHub runtime initialization", async () => { const fixture = await seedCompany(); const { endpoint, runtime, service, webhookSecret } = await configuredGitHubEndpoint(fixture); await runtime.removeEndpoint(endpoint.id); const replacementsBeforeBurst = runtime.replaceCount; let releaseInitialization!: () => void; const initializationGate = new Promise((resolve) => { releaseInitialization = resolve; }); runtime.initializeHook = () => initializationGate; const body = JSON.stringify({ installation: { id: 2468 } }); const signature = createHmac("sha256", webhookSecret) .update(body) .digest("hex"); const send = (delivery: string) => service.handleWebhook( endpoint.publicId, "github", new Request("https://paperclip.example/github", { method: "POST", headers: { "content-type": "application/json", "x-github-event": "installation_repositories", "x-github-delivery": delivery, "x-hub-signature-256": `sha256=${signature}`, }, body, }), ); const requests = [send("github-cold-1"), send("github-cold-2")]; await vi.waitFor(() => { expect(runtime.replaceCount).toBe(replacementsBeforeBurst + 1); }); releaseInitialization(); const responses = await Promise.all(requests); expect(responses.map((response) => response.status)).toEqual([202, 202]); expect(runtime.replaceCount).toBe(replacementsBeforeBurst + 1); expect( runtime.endpoints.get(endpoint.id)?.initialize, ).toHaveBeenCalledTimes(1); }); it("durably stages GitHub webhooks within budget before cold runtime initialization", async () => { const fixture = await seedCompany(); const deferred: Array<() => void> = []; const { endpoint, runtime, service, webhookSecret } = await configuredGitHubEndpoint(fixture, { githubWebhookResponseBudgetMs: 25, scheduleDeferredWork: (task) => deferred.push(task), }); // The setup helper intentionally uses setImmediate until setup completes, // while this test replaces later work with a captured queue. Let those // setup-era callbacks join the runtime singleflight before measuring the // cold request so their replacement is not attributed to this webhook. await new Promise((resolve) => setImmediate(resolve)); await service.reconcileProviderRuntimes(); deferred.length = 0; await runtime.removeEndpoint(endpoint.id); const replacementsBeforeRequest = runtime.replaceCount; let releaseInitialization!: () => void; const initializationGate = new Promise((resolve) => { releaseInitialization = resolve; }); runtime.initializeHook = () => initializationGate; const startedAt = Date.now(); const response = await service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: "github-cold-budget", event: "installation_repositories", payload: { installation: { id: 2468 } }, webhookSecret, }), ); const elapsedMs = Date.now() - startedAt; expect(response.status).toBe(202); expect(elapsedMs).toBeLessThan(500); // The durable response budget is intentionally independent of cold // runtime startup. Under a loaded event loop the 202 can win the budget // race before the background processor reaches replaceEndpoint, so wait // for the explicit initialization boundary instead of assuming same-tick // scheduling. await vi.waitFor(() => { expect(runtime.replaceCount).toBe(replacementsBeforeRequest + 1); }); expect(deferred).toHaveLength(0); await expect( db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq( chatActions.providerActionId, "github_webhook_ingress:github-cold-budget", ), ), ) .then((rows) => rows[0]), ).resolves.toMatchObject({ status: "processing" }); releaseInitialization(); await vi.waitFor( () => { expect( runtime.endpoints.get(endpoint.id)?.webhookRequest, ).not.toBeNull(); }, { timeout: 2_000 }, ); await vi.waitFor(async () => { const action = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq( chatActions.providerActionId, "github_webhook_ingress:github-cold-budget", ), ), ) .then((rows) => rows[0]); expect(action).toMatchObject({ status: "processed" }); expect(action?.payload).not.toHaveProperty("body"); }); await service.shutdown(); }); it("durably stages GitHub webhooks while credential mutation owns the processing lease", async () => { const fixture = await seedCompany(); const deferred: Array<() => void> = []; const { endpoint, runtime, service, webhookSecret } = await configuredGitHubEndpoint(fixture, { githubWebhookResponseBudgetMs: 25, scheduleDeferredWork: (task) => deferred.push(task), }); deferred.length = 0; const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected GitHub provider runtime"); providerRuntime.webhookRequest = null; await db.insert(chatEndpointLeases).values({ companyId: endpoint.companyId, endpointId: endpoint.id, leaseKey: "credentials", token: "external-credential-mutation", expiresAt: new Date(Date.now() + 60_000), }); const startedAt = Date.now(); const response = await service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: "github-authentication-lane-budget", event: "installation_repositories", payload: { installation: { id: 2468 } }, webhookSecret, }), ); const elapsedMs = Date.now() - startedAt; expect(response.status).toBe(202); expect(elapsedMs).toBeLessThan(500); expect(providerRuntime.webhookRequest).toBeNull(); expect(deferred).toHaveLength(0); // The HTTP response budget can expire before the asynchronous worker has // claimed the durable receipt. Wait for its claim while the lease is held. await vi.waitFor( async () => { await expect( db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq( chatActions.providerActionId, "github_webhook_ingress:github-authentication-lane-budget", ), ), ) .then((rows) => rows[0]), ).resolves.toMatchObject({ status: "processing" }); }, { timeout: 2_000 }, ); await db .delete(chatEndpointLeases) .where( and( eq(chatEndpointLeases.endpointId, endpoint.id), eq(chatEndpointLeases.leaseKey, "credentials"), ), ); await vi.waitFor( () => { expect(providerRuntime.webhookRequest).not.toBeNull(); }, { timeout: 2_000 }, ); await vi.waitFor(async () => { const action = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq( chatActions.providerActionId, "github_webhook_ingress:github-authentication-lane-budget", ), ), ) .then((rows) => rows[0]); expect(action).toMatchObject({ status: "processed" }); expect(action?.payload).not.toHaveProperty("body"); }); await service.shutdown(); }); it("recovers an authenticated GitHub webhook after shutdown before deferred work starts", async () => { const fixture = await seedCompany(); const configured = await configuredGitHubEndpoint(fixture); const deferred: Array<() => void> = []; const receiverRuntime = new FakeChatSdkRuntime(); const receiver = createService(receiverRuntime, undefined, { deferWebhookProcessing: true, scheduleDeferredWork: (task) => deferred.push(task), }).service; const delivery = `github-restart-${randomUUID()}`; const bodyMarker = `restart-body-${randomUUID()}`; const response = await receiver.handleWebhook( configured.endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery, event: "issue_comment", payload: { action: "created", installation: { id: 2468 }, repository: { id: 97531, full_name: "paperclipai/paperclip", name: "paperclip", owner: { id: 1357, login: "paperclipai" }, }, issue: { number: 91 }, comment: { id: 9191, body: bodyMarker }, sender: { id: 42, login: "octocat" }, }, webhookSecret: configured.webhookSecret, }), ); expect(response.status).toBe(202); expect(deferred).toHaveLength(1); const staged = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, configured.endpoint.id), eq( chatActions.providerActionId, `github_webhook_ingress:${delivery}`, ), ), ) .then((rows) => rows[0]); expect(staged).toMatchObject({ status: "received", payload: expect.objectContaining({ body: expect.stringContaining(bodyMarker), }), }); const conflictingMarker = `conflicting-body-${randomUUID()}`; const conflictingDuplicate = await receiver.handleWebhook( configured.endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery, event: "issue_comment", payload: { action: "created", installation: { id: 2468 }, repository: { id: 97531, full_name: "paperclipai/paperclip", name: "paperclip", owner: { id: 1357, login: "paperclipai" }, }, issue: { number: 91 }, comment: { id: 9191, body: conflictingMarker }, sender: { id: 42, login: "octocat" }, }, webhookSecret: configured.webhookSecret, }), ); expect(conflictingDuplicate.status).toBe(202); const afterConflictingDuplicate = await db .select({ payload: chatActions.payload }) .from(chatActions) .where( and( eq(chatActions.endpointId, configured.endpoint.id), eq( chatActions.providerActionId, `github_webhook_ingress:${delivery}`, ), ), ); expect(afterConflictingDuplicate).toEqual([ expect.objectContaining({ payload: expect.objectContaining({ body: expect.stringContaining(bodyMarker), }), }), ]); expect(JSON.stringify(afterConflictingDuplicate[0]?.payload)).not.toContain( conflictingMarker, ); // The scheduler never began the callback. Graceful shutdown must leave // the authenticated receipt available to the next server process. await receiver.shutdown(); const restartedRuntime = new FakeChatSdkRuntime(); const restarted = createService(restartedRuntime).service; await restarted.processPendingGitHubWebhookIngress(25, staged!.id); expect( restartedRuntime.endpoints.get(configured.endpoint.id)?.webhookRequest, ).not.toBeNull(); const processed = await db .select() .from(chatActions) .where(eq(chatActions.id, staged!.id)) .then((rows) => rows[0]); expect(processed).toMatchObject({ status: "processed", payload: expect.objectContaining({ bodySha256: expect.any(String), redacted: true, }), }); expect(processed?.payload).not.toHaveProperty("body"); const recoveredRuntime = restartedRuntime.endpoints.get( configured.endpoint.id, ); if (!recoveredRuntime) throw new Error("Expected recovered GitHub runtime"); recoveredRuntime.webhookRequest = null; const duplicate = await restarted.handleWebhook( configured.endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery, event: "issue_comment", payload: { action: "created", installation: { id: 2468 }, repository: { id: 97531, full_name: "paperclipai/paperclip", name: "paperclip", owner: { id: 1357, login: "paperclipai" }, }, issue: { number: 91 }, comment: { id: 9191, body: bodyMarker }, sender: { id: 42, login: "octocat" }, }, webhookSecret: configured.webhookSecret, }), ); expect(duplicate.status).toBe(202); expect(recoveredRuntime.webhookRequest).toBeNull(); await restarted.shutdown(); await configured.service.shutdown(); }); it("automatically recovers a pre-ingress GitHub failure only through a genuine callback and normal durable workers", async () => { const context = await githubPreIngressRecoveryFixture(); const { service, endpoint } = context; try { await context.makeScanDue(); await service.processFailedGitHubWebhookDeliveries(5, endpoint.id); expect(context.posts).toEqual([ `/app/hook/deliveries/${context.deliveryId}/attempts`, ]); expect(await context.ingress()).toBeUndefined(); expect(context.wakeup).not.toHaveBeenCalled(); const receipt = await context.receipt(); expect(receipt).toMatchObject({ status: "processed", result: { attempts: 1, latestAttemptId: context.deliveryId }, }); const serialized = JSON.stringify(receipt); expect(serialized).not.toContain(context.body); expect(serialized).not.toContain("private-provider-header-canary"); expect(serialized).not.toContain("private-proxy-body-canary"); const repairActivity = async () => (await service.listActivity(endpoint.id)).find( (entry) => entry.id === receipt!.id, ); expect(await repairActivity()).toMatchObject({ kind: "repair", status: "pending", summary: "GitHub webhook redelivery requested", detail: expect.stringContaining( "does not yet confirm receipt or a reply", ), }); // A coincident GUID in another endpoint's inbox is not this callback's // receipt, even when that other endpoint has already processed it. const other = await service.create( context.fixture.companyId, { provider: "github", assignedAgentId: context.fixture.assignedAgentId, name: "Unrelated GitHub receipt", }, "owner-user", ); await db.insert(chatActions).values({ companyId: context.fixture.companyId, endpointId: other.id, kind: "github_webhook_ingress", providerActionId: `github_webhook_ingress:${context.guid}`, status: "processed", payload: { version: 1, deliveryId: context.guid, eventType: "issue_comment", redacted: true, }, }); expect(await repairActivity()).toMatchObject({ status: "pending", summary: "GitHub webhook redelivery requested", }); // A successful recovery POST is not a delivery. Only the separately // authenticated provider callback may put content into the durable inbox. expect((await context.callback("not-the-secret")).status).toBe(401); expect(await context.ingress()).toBeUndefined(); expect((await context.callback()).status).toBe(202); const ingress = await context.ingress(); expect(ingress).toMatchObject({ status: "received" }); await service.processPendingGitHubWebhookIngress(1, ingress!.id); const deliveries = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); for (const delivery of deliveries) { // Advance only this fixture's persisted coalescing deadline; the // production batching delay is exercised by its dedicated tests. await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, delivery.id)); await service.processPendingDeliveries(1, delivery.id); } expect(context.wakeup).toHaveBeenCalledTimes(1); const tasks = await db .select() .from(issues) .where(eq(issues.companyId, context.fixture.companyId)); expect(tasks).toHaveLength(1); const comments = await db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, tasks[0]!.id)); expect(comments).toEqual([{ body: context.body }]); expect(await context.ingress()).toMatchObject({ status: "processed", payload: { redacted: true }, }); expect(await repairActivity()).toMatchObject({ kind: "repair", status: "received", summary: "GitHub webhook received after recovery request", detail: "Paperclip received this callback. Its normal access checks and processing still apply.", replayable: false, resolutionActions: [], }); await context.callback(); await context.makeScanDue(); await service.processFailedGitHubWebhookDeliveries(5, endpoint.id); expect(context.posts).toHaveLength(1); expect(context.wakeup).toHaveBeenCalledTimes(1); } finally { await retirePublicationFixture(service, endpoint.id); } }); it("does not adopt historical pre-ingress GitHub failures when initializing a recovery epoch", async () => { const context = await githubPreIngressRecoveryFixture(); try { await context.makeScanDue(false); await context.service.processFailedGitHubWebhookDeliveries( 5, context.endpoint.id, ); expect(context.posts).toEqual([]); expect(await context.receipt()).toBeUndefined(); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }); it("immediately initializes a new GitHub recovery generation despite the previous epoch's 24-hour backoff", async () => { const context = await githubPreIngressRecoveryFixture(); const baseTime = Date.now(); vi.useFakeTimers({ toFake: ["Date"] }); try { vi.setSystemTime(new Date(baseTime)); const oldState = await context.checkpoint(); const oldWindow = oldState.value as Record; const previousDeadline = new Date( baseTime + 24 * 60 * 60_000, ).toISOString(); await db .update(chatSdkState) .set({ value: { ...oldWindow, nextScanAt: previousDeadline } }) .where(eq(chatSdkState.id, oldState.id)); const oldFailure = { ...context.original, id: "91000", guid: randomUUID(), }; await context.service.configure( context.endpoint.id, { action: "pause" }, "owner-user", ); await context.service.configure( context.endpoint.id, { action: "reconnect" }, "owner-user", ); await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, context.endpoint.id)); context.requests.length = 0; await context.service.processFailedGitHubWebhookDeliveries( 5, context.endpoint.id, ); const reset = (await context.checkpoint()).value as Record< string, unknown >; expect(reset.generation).not.toBe(oldWindow.generation); expect(reset.floor).toBe(new Date(baseTime).toISOString()); expect(Date.parse(String(reset.nextScanAt))).toBe(baseTime + 60_000); expect(context.requests).toEqual([]); expect(context.posts).toEqual([]); // This message belongs to the new window; the other failure predates // reconnect and must never be adopted when the old backoff is cleared. const createdAt = new Date(baseTime + 1_000).toISOString(); context.original.delivered_at = createdAt; context.payload.comment.created_at = createdAt; context.payload.comment.updated_at = createdAt; context.setCanonicalComment({ created_at: createdAt, updated_at: createdAt, }); context.setDeliveries([context.original, oldFailure]); vi.setSystemTime(new Date(baseTime + 61_000)); await context.service.processFailedGitHubWebhookDeliveries( 5, context.endpoint.id, ); expect(context.posts).toEqual([ `/app/hook/deliveries/${context.deliveryId}/attempts`, ]); expect( context.requests.some((entry) => entry.pathname.includes(oldFailure.id), ), ).toBe(false); expect(await context.receipt()).toMatchObject({ payload: { generation: reset.generation, floor: reset.floor }, result: { attempts: 1 }, }); } finally { vi.useRealTimers(); await retirePublicationFixture(context.service, context.endpoint.id); } }); it.each(["received", "processing", "processed", "failed", "cancelled"])( "suppresses automatic pre-ingress GitHub redelivery for any existing local receipt (%s)", async (status) => { const context = await githubPreIngressRecoveryFixture(); try { await context.callback(); const ingress = await context.ingress(); await db .update(chatActions) .set({ status, result: { attempts: 5, retryable: false } }) .where(eq(chatActions.id, ingress!.id)); const before = await context.ingress(); await context.makeScanDue(); await context.service.processFailedGitHubWebhookDeliveries( 5, context.endpoint.id, ); expect(context.posts).toEqual([]); expect(await context.receipt()).toBeUndefined(); expect(await context.ingress()).toEqual(before); expect(context.wakeup).not.toHaveBeenCalled(); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }, ); it("coalesces concurrent automatic pre-ingress GitHub scans into one provider redelivery", async () => { const context = await githubPreIngressRecoveryFixture(); const competitor = createService( new FakeChatSdkRuntime(), context.providerFetch, ).service; let release!: () => void; let entered!: () => void; const parked = new Promise((resolve) => { entered = resolve; }); const barrier = new Promise((resolve) => { release = resolve; }); try { context.setBeforeList(async () => { entered(); await barrier; }); await context.makeScanDue(); const scans = Promise.all([ context.service.processFailedGitHubWebhookDeliveries( 5, context.endpoint.id, ), competitor.processFailedGitHubWebhookDeliveries(5, context.endpoint.id), ]); await parked; release(); await scans; expect( context.requests.filter( (request) => request.pathname === "/app/hook/deliveries", ), ).toHaveLength(1); expect(context.posts).toHaveLength(1); expect(await context.receipt()).toMatchObject({ result: { attempts: 1 }, }); } finally { release(); await competitor.shutdown(); await retirePublicationFixture(context.service, context.endpoint.id); } }); it("does not retry automatic pre-ingress GitHub redelivery without a distinct failed provider attempt", async () => { const context = await githubPreIngressRecoveryFixture(); try { await context.makeScanDue(); await context.service.processFailedGitHubWebhookDeliveries( 5, context.endpoint.id, ); const receipt = await context.receipt(); await db .update(chatActions) .set({ result: { ...receipt!.result, retryAt: new Date(0).toISOString() }, }) .where(eq(chatActions.id, receipt!.id)); await context.makeScanDue(); await context.service.processFailedGitHubWebhookDeliveries( 5, context.endpoint.id, ); expect(context.posts).toHaveLength(1); expect(await context.receipt()).toMatchObject({ result: { attempts: 1, latestAttemptId: context.deliveryId }, }); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }); it.each(["pending", "4xx", "throttled", "too recent"])( "does not request automatic pre-ingress GitHub recovery while the latest provider attempt is %s", async (state) => { const context = await githubPreIngressRecoveryFixture(); try { context.setDeliveries([ { ...context.original, id: "9007199254740994", redelivery: true, status_code: state === "pending" ? null : state === "4xx" ? 400 : 502, throttled_at: state === "throttled" ? new Date().toISOString() : null, delivered_at: new Date( Date.now() - (state === "too recent" ? 1_000 : 15_000), ).toISOString(), }, context.original, ]); await context.makeScanDue(); await context.service.processFailedGitHubWebhookDeliveries( 5, context.endpoint.id, ); expect(context.posts).toEqual([]); expect(await context.receipt()).toBeUndefined(); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }, ); it("does not let ineligible pre-ingress GitHub candidates exhaust every future recovery scan", async () => { const context = await githubPreIngressRecoveryFixture(); try { const bots = Array.from({ length: 5 }, (_, index) => ({ ...context.original, id: String(10_000 + index), guid: randomUUID(), })); for (const bot of bots) context.setDetail(bot, { ...context.payload, comment: { ...context.payload.comment, user: { ...context.payload.comment.user, type: "Bot" }, }, }); context.setDeliveries([...bots, context.original]); await context.makeScanDue(); await context.service.processFailedGitHubWebhookDeliveries( 5, context.endpoint.id, ); expect(context.posts).toEqual([]); const rejected = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, context.endpoint.id), eq(chatActions.kind, "github_webhook_recovery"), ), ); expect(rejected).toHaveLength(5); for (const receipt of rejected) { expect(receipt).toMatchObject({ status: "cancelled", result: { attempts: 0 }, }); expect(receipt.payload.original).toEqual({ id: expect.any(String), guid: expect.any(String), }); } expect(JSON.stringify(rejected)).not.toContain(context.body); await context.makeScanDue(); await context.service.processFailedGitHubWebhookDeliveries( 5, context.endpoint.id, ); expect(context.posts).toHaveLength(1); expect( context.requests.filter((request) => /^\/app\/hook\/deliveries\/\d+$/.test(request.pathname), ), ).toHaveLength(6); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }); it("quarantines an ambiguous automatic pre-ingress GitHub redelivery without blindly repeating the POST", async () => { const context = await githubPreIngressRecoveryFixture(); try { context.setBeforePost(async () => { throw new Error( "provider connection closed after accepting the request", ); }); await context.makeScanDue(); await context.service.processFailedGitHubWebhookDeliveries( 5, context.endpoint.id, ); expect(await context.receipt()).toMatchObject({ status: "delivery_unknown", result: { attempts: 1 }, }); const receipt = await context.receipt(); await db .update(chatActions) .set({ result: { ...receipt!.result, retryAt: new Date(0).toISOString() }, }) .where(eq(chatActions.id, receipt!.id)); await context.makeScanDue(); await context.service.processFailedGitHubWebhookDeliveries( 5, context.endpoint.id, ); expect(context.posts).toHaveLength(1); expect(context.wakeup).not.toHaveBeenCalled(); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }); it.each(["successful sibling", "edited comment", "disabled repository"])( "rejects automatic pre-ingress GitHub recovery after %s", async (change) => { const context = await githubPreIngressRecoveryFixture(); try { if (change === "successful sibling") context.setDeliveries([ { ...context.original, id: "9007199254740994", redelivery: true, status_code: 202, delivered_at: new Date(Date.now() - 15_000).toISOString(), }, context.original, ]); if (change === "edited comment") context.setCanonicalComment({ body: "edited after the original callback", }); if (change === "disabled repository") { const resources = await context.service.listResources( context.endpoint.id, ); await context.service.replaceResources( context.endpoint.id, resources.map((resource) => ({ id: resource.id, enabled: false })), ); } await context.makeScanDue(); await context.service.processFailedGitHubWebhookDeliveries( 5, context.endpoint.id, ); expect(context.posts).toEqual([]); if (change === "edited comment") expect(await context.receipt()).toMatchObject({ status: "cancelled", result: { attempts: 0, code: "source_changed_or_unavailable" }, }); else expect(await context.receipt()).toBeUndefined(); expect(context.wakeup).not.toHaveBeenCalled(); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }, ); it.each(["truncated history", "ordering drift"])( "fails closed on automatic pre-ingress GitHub recovery with %s", async (mode) => { const context = await githubPreIngressRecoveryFixture(); try { if (mode === "truncated history") context.setHistoryHasMore(true); else context.setDeliveries([ context.original, { ...context.original, id: "9007199254740994", guid: randomUUID(), delivered_at: new Date(Date.now() - 15_000).toISOString(), }, ]); await context.makeScanDue(); await context.service.processFailedGitHubWebhookDeliveries( 5, context.endpoint.id, ); expect(context.posts).toEqual([]); expect(await context.receipt()).toBeUndefined(); expect((await context.checkpoint()).value).toMatchObject({ outcome: mode === "truncated history" ? "history_limit_reached" : "scan_failed", }); expect( context.requests.filter( (request) => request.pathname === "/app/hook/deliveries", ), ).toHaveLength(mode === "truncated history" ? 3 : 1); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }, ); it("stops automatic pre-ingress GitHub pagination at the epoch floor without adopting older failures", async () => { const context = await githubPreIngressRecoveryFixture(); try { context.setHistoryHasMore(true); context.setDeliveries([ context.original, { ...context.original, id: "9007199254740994", guid: randomUUID(), delivered_at: new Date(Date.now() - 180_000).toISOString(), }, ]); await context.makeScanDue(); await context.service.processFailedGitHubWebhookDeliveries( 5, context.endpoint.id, ); expect(context.posts).toHaveLength(1); expect( context.requests.filter( (request) => request.pathname === "/app/hook/deliveries", ), ).toHaveLength(1); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }); it.each(["issue", "pull request", "sender", "action"])( "rejects a signed automatic GitHub callback whose original %s identity changed", async (field) => { const context = await githubPreIngressRecoveryFixture( field === "pull request" ? "pull_request_review_comment" : "issue_comment", ); try { await context.makeScanDue(); await context.service.processFailedGitHubWebhookDeliveries( 5, context.endpoint.id, ); expect(context.posts).toHaveLength(1); const changed = { ...context.payload, ...(field === "issue" ? { issue: { id: 9290, number: 92 } } : {}), ...(field === "pull request" ? { pull_request: { id: 9290, number: 92 } } : {}), ...(field === "sender" ? { sender: { id: 99, login: "different-user", type: "User" } } : {}), ...(field === "action" ? { action: "edited" } : {}), }; expect( (await context.callback(context.webhookSecret, changed)).status, ).toBe(200); expect(await context.ingress()).toBeUndefined(); expect(context.wakeup).not.toHaveBeenCalled(); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }, ); it("rechecks canonical content before processing an automatically recovered GitHub callback", async () => { const context = await githubPreIngressRecoveryFixture(); try { await context.makeScanDue(); await context.service.processFailedGitHubWebhookDeliveries( 5, context.endpoint.id, ); expect(context.posts).toHaveLength(1); await context.callback(); const staged = await context.ingress(); expect(staged).toMatchObject({ status: "received" }); context.setCanonicalComment({ body: "changed after successful callback staging", }); await context.service.processPendingGitHubWebhookIngress(1, staged!.id); expect(await context.ingress()).toMatchObject({ status: "cancelled", result: { code: "github_webhook_recovery_source_changed" }, payload: { redacted: true }, }); expect(context.wakeup).not.toHaveBeenCalled(); expect( await db .select({ id: issues.id }) .from(issues) .where(eq(issues.companyId, context.fixture.companyId)), ).toEqual([]); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }); it("denies a stale automatic GitHub callback after pause and reconnect even with the still-current HMAC secret", async () => { const context = await githubPreIngressRecoveryFixture(); try { await context.makeScanDue(); await context.service.processFailedGitHubWebhookDeliveries( 5, context.endpoint.id, ); const receipt = await context.receipt(); expect(receipt).toBeDefined(); await context.service.configure( context.endpoint.id, { action: "pause" }, "owner-user", ); expect((await context.callback()).status).toBe(200); expect(await context.ingress()).toBeUndefined(); await context.service.configure( context.endpoint.id, { action: "reconnect" }, "owner-user", ); expect((await context.callback()).status).toBe(200); expect(await context.ingress()).toBeUndefined(); expect(await context.receipt()).toEqual(receipt); expect(context.wakeup).not.toHaveBeenCalled(); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }); it("never re-arms a failed local GitHub receipt when an automatic redelivery races the original callback", async () => { const context = await githubPreIngressRecoveryFixture(); try { await context.makeScanDue(); await context.service.processFailedGitHubWebhookDeliveries( 5, context.endpoint.id, ); expect(context.posts).toHaveLength(1); await context.callback(); const ingress = await context.ingress(); context.providerRuntime.webhookHook = undefined; context.providerRuntime.webhookResponse = new Response( "invalid provider content", { status: 400 }, ); await context.service.processPendingGitHubWebhookIngress(1, ingress!.id); const failed = await context.ingress(); expect(failed).toMatchObject({ status: "failed", result: { attempts: 1, retryable: false }, }); context.providerRuntime.webhookResponse = new Response(null, { status: 202, }); await context.callback(); expect(await context.ingress()).toEqual(failed); await context.makeScanDue(); await context.service.processFailedGitHubWebhookDeliveries( 5, context.endpoint.id, ); expect(context.posts).toHaveLength(1); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }); it("never stores a GitHub webhook body before its signature is authenticated", async () => { const fixture = await seedCompany(); const { endpoint, service, webhookSecret } = await configuredGitHubEndpoint(fixture); const delivery = `github-invalid-${randomUUID()}`; const marker = `must-not-persist-${randomUUID()}`; const signed = signedGitHubWebhookRequest({ delivery, event: "issue_comment", payload: { action: "created", installation: { id: 2468 }, repository: { id: 97531, full_name: "paperclipai/paperclip", name: "paperclip", owner: { id: 1357, login: "paperclipai" }, }, issue: { number: 91 }, comment: { id: 9191, body: marker }, sender: { id: 42, login: "octocat" }, }, webhookSecret, }); const headers = new Headers(signed.headers); headers.set("x-hub-signature-256", "sha256=invalid"); const response = await service.handleWebhook( endpoint.publicId, "github", new Request(signed, { headers }), ); expect(response.status).toBe(401); await expect( db .select({ id: chatActions.id }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq( chatActions.providerActionId, `github_webhook_ingress:${delivery}`, ), ), ), ).resolves.toEqual([]); await expect( db.execute( sql`select id from chat_actions where endpoint_id = ${endpoint.id} and payload::text like ${`%${marker}%`}`, ), ).resolves.toHaveLength(0); await service.shutdown(); }); it("records one content-free GitHub receipt for a disabled repository without admitting work", async () => { const fixture = await seedCompany(); const { endpoint, runtime, service, webhookSecret, wakeup } = await configuredGitHubEndpoint(fixture); await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, endpoint.id)); const resources = await service.listResources(endpoint.id); await service.replaceResources(endpoint.id, [ { id: resources[0]!.id, enabled: false }, ]); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected GitHub provider runtime"); providerRuntime.webhookRequest = null; const delivery = `github-disabled-${randomUUID()}`; const marker = `private-disabled-body-${randomUUID()}`; const makeRequest = () => signedGitHubWebhookRequest({ delivery, event: "issue_comment", payload: { action: "created", installation: { id: 2468 }, repository: { id: 97531, full_name: "paperclipai/paperclip", name: "paperclip", owner: { id: 1357, login: "paperclipai" }, }, issue: { number: 91, body: marker }, comment: { id: 9191, body: marker }, sender: { id: 42, login: "private-disabled-author" }, }, webhookSecret, }); const invalid = makeRequest(); invalid.headers.set("x-hub-signature-256", "sha256=invalid"); expect( (await service.handleWebhook(endpoint.publicId, "github", invalid)) .status, ).toBe(401); await expect( db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)), ).resolves.toEqual([]); const responses = await Promise.all( [1, 2, 3].map(() => service.handleWebhook(endpoint.publicId, "github", makeRequest()), ), ); expect(responses.map((response) => response.status)).toEqual([ 200, 200, 200, ]); const rows = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ state: "filtered", eventKind: "message", conversationId: null, principalId: null, redactedError: "Destination is not enabled in Paperclip", normalizedEvent: { kind: "message", filtering: { contentRetained: false, reason: "destination_not_enabled", resourceId: resources[0]!.id, }, }, }); expect(JSON.stringify(rows)).not.toContain(marker); expect(JSON.stringify(rows)).not.toContain("private-disabled-author"); expect(Object.keys(rows[0]!.normalizedEvent).sort()).toEqual([ "filtering", "kind", "providerEventId", ]); expect(providerRuntime.webhookRequest).toBeNull(); expect(wakeup).not.toHaveBeenCalled(); await expect(service.listConversations(endpoint.id)).resolves.toEqual([]); await expect(service.listActivity(endpoint.id)).resolves.toContainEqual( expect.objectContaining({ id: rows[0]!.id, status: "filtered", summary: "message ignored", detail: "Destination is not enabled in Paperclip", replayable: false, }), ); await expect( db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq( chatActions.providerActionId, `github_webhook_ingress:${delivery}`, ), ), ), ).resolves.toEqual([]); await service.replaceResources(endpoint.id, [ { id: resources[0]!.id, enabled: true }, ]); await service.processPendingDeliveries(); expect( wakeup.mock.calls.some( ([, options]) => options?.durableChatRequest?.companyId === fixture.companyId, ), ).toBe(false); expect(providerRuntime.webhookRequest).toBeNull(); await service.shutdown(); }); it("does not retain signed GitHub events outside the configured App event set", async () => { const fixture = await seedCompany(); const { endpoint, runtime, service, webhookSecret } = await configuredGitHubEndpoint(fixture); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected GitHub provider runtime"); providerRuntime.webhookRequest = null; const delivery = `github-unsupported-${randomUUID()}`; const marker = `unsupported-body-${randomUUID()}`; const response = await service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery, event: "issues", payload: { action: "opened", installation: { id: 2468 }, issue: { body: marker }, }, webhookSecret, }), ); expect(response.status).toBe(200); expect(providerRuntime.webhookRequest).toBeNull(); await expect( db .select({ id: chatActions.id }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "github_webhook_ingress"), sql`${chatActions.payload}::text like ${`%${marker}%`}`, ), ), ).resolves.toEqual([]); await service.shutdown(); }); it("safely re-arms an exact terminal GitHub delivery after an operator redelivery", async () => { const fixture = await seedCompany(); const { endpoint, runtime, service, webhookSecret } = await configuredGitHubEndpoint(fixture); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected GitHub provider runtime"); providerRuntime.webhookResponse = new Response("bad payload", { status: 400, }); const webhookHook = vi.fn(async () => undefined); providerRuntime.webhookHook = webhookHook; const delivery = `github-terminal-${randomUUID()}`; const makeRequest = () => signedGitHubWebhookRequest({ delivery, event: "installation_repositories", payload: { action: "added", installation: { id: 2468 } }, webhookSecret, }); const first = await service.handleWebhook( endpoint.publicId, "github", makeRequest(), ); expect(first.status).toBe(400); const terminal = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq( chatActions.providerActionId, `github_webhook_ingress:${delivery}`, ), ), ) .then((rows) => rows[0]); expect(terminal).toMatchObject({ status: "failed", result: expect.objectContaining({ attempts: 1, httpStatus: 400, retryable: false, }), payload: expect.objectContaining({ redacted: true }), }); expect(webhookHook).toHaveBeenCalledTimes(1); await expect(service.listActivity(endpoint.id)).resolves.toEqual( expect.arrayContaining([ expect.objectContaining({ id: terminal!.id, kind: "action", actionType: "github_webhook_ingress", status: "failed", summary: "GitHub webhook could not be processed", detail: expect.stringContaining(delivery), replayable: false, }), ]), ); const conflictingDuplicate = await service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery, event: "installation_repositories", payload: { action: "removed", installation: { id: 2468 }, marker: "different authenticated body", }, webhookSecret, }), ); expect(conflictingDuplicate.status).toBe(202); expect(webhookHook).toHaveBeenCalledTimes(1); await expect( db .select() .from(chatActions) .where(eq(chatActions.id, terminal!.id)) .then((rows) => rows[0]), ).resolves.toMatchObject({ status: "failed", result: expect.objectContaining({ attempts: 1, httpStatus: 400, retryable: false, }), payload: expect.objectContaining({ redacted: true }), }); providerRuntime.webhookResponse = new Response("accepted", { status: 202 }); const recovered = await service.handleWebhook( endpoint.publicId, "github", makeRequest(), ); expect(recovered.status).toBe(202); expect(webhookHook).toHaveBeenCalledTimes(2); await expect( db .select() .from(chatActions) .where(eq(chatActions.id, terminal!.id)) .then((rows) => rows[0]), ).resolves.toMatchObject({ status: "processed", result: expect.objectContaining({ attempts: 2, httpStatus: 202 }), payload: expect.objectContaining({ redacted: true }), }); const processedDuplicate = await service.handleWebhook( endpoint.publicId, "github", makeRequest(), ); expect(processedDuplicate.status).toBe(202); expect(webhookHook).toHaveBeenCalledTimes(2); await service.shutdown(); }); it("cancels staged GitHub ingress when same-secret pause and resume wins at the replay barrier", async () => { let armed = false; let release!: () => void; let entered!: () => void; const reached = new Promise((resolve) => { entered = resolve; }); const barrier = new Promise((resolve) => { release = resolve; }); const context = await githubPreIngressRecoveryFixture( "issue_comment", async () => { if (armed) { entered(); await barrier; } }, ); let worker: | ReturnType | undefined; const sdkDispatch = vi.fn(); try { const [record] = await db .select() .from(chatEndpoints) .where(eq(chatEndpoints.id, context.endpoint.id)); await db .update(chatEndpoints) .set({ setup: { ...record!.setup, step: "complete" } }) .where(eq(chatEndpoints.id, context.endpoint.id)); const refs = () => db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, record!.connectionId)) .then((rows) => rows[0]!.refs); const originalRefs = await refs(); const botUsername = (await context.service.get(context.endpoint.id)) .botUsername!; const wireRuntime = async () => { const runtime = context.runtime.endpoints.get(context.endpoint.id)!; const callbacks = context.runtime.configurations.get( context.endpoint.id, )!.callbacks; attachFakeGitHubIssueCommentWebhook({ botUsername, callbacks, endpointId: context.endpoint.id, runtime, }); const dispatch = runtime.webhookHook!; runtime.webhookHook = async (request) => { sdkDispatch(); await dispatch(request); }; }; context.runtime.initializeHook = wireRuntime; await wireRuntime(); armed = true; expect((await context.callback()).status).toBe(202); const staged = await context.ingress(); expect(staged).toMatchObject({ status: "received" }); worker = context.service.processPendingGitHubWebhookIngress( 1, staged!.id, ); await reached; await context.service.configure( context.endpoint.id, { action: "pause" }, "owner-user", ); await context.service.configure( context.endpoint.id, { action: "resume" }, "owner-user", ); expect(await refs()).toEqual(originalRefs); expect(await context.service.get(context.endpoint.id)).toMatchObject({ status: "active", }); release(); await worker; expect(await context.ingress()).toMatchObject({ status: "cancelled", payload: { redacted: true }, result: { code: "github_webhook_ingress_runtime_superseded" }, }); expect((await context.ingress())!.payload).not.toHaveProperty("body"); expect(sdkDispatch).not.toHaveBeenCalled(); expect( await db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, context.endpoint.id)), ).toEqual([]); expect( await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.companyId, context.fixture.companyId)), ).toEqual([]); expect(context.wakeup).not.toHaveBeenCalled(); } finally { release(); await worker?.catch(() => undefined); await retirePublicationFixture(context.service, context.endpoint.id); } }); it("cancels staged GitHub ingress when a minimal recovery tombstone arrives before processing", async () => { const context = await githubPreIngressRecoveryFixture(); try { expect((await context.callback()).status).toBe(202); const staged = await context.ingress(); const window = (await context.checkpoint()).value as Record< string, unknown >; await db.insert(chatActions).values({ companyId: context.fixture.companyId, endpointId: context.endpoint.id, kind: "github_webhook_recovery", providerActionId: `github_webhook_recovery:${context.guid}`, status: "cancelled", payload: { ...window, original: { id: context.deliveryId, guid: context.guid }, }, result: { attempts: 0, code: "source_changed_or_unavailable" }, }); context.requests.length = 0; await context.service.processPendingGitHubWebhookIngress(1, staged!.id); const cancelled = await context.ingress(); expect(cancelled).toMatchObject({ status: "cancelled", payload: { redacted: true }, result: { code: "github_webhook_recovery_source_changed" }, }); expect(cancelled!.payload).not.toHaveProperty("body"); expect(cancelled!.result?.retryable).not.toBe(true); expect(context.requests).toEqual([]); expect( await db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, context.endpoint.id)), ).toEqual([]); expect(context.wakeup).not.toHaveBeenCalled(); expect( await context.service.processPendingGitHubWebhookIngress(1, staged!.id), ).toBe(0); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }); it("cancels and redacts GitHub ingress when rotation wins before replay authentication", async () => { const fixture = await seedCompany(); const configured = await configuredGitHubEndpoint(fixture); let releaseReplay!: () => void; let markReplayReady!: () => void; const replayReady = new Promise((resolve) => { markReplayReady = resolve; }); const replayRelease = new Promise((resolve) => { releaseReplay = resolve; }); const receiver = createService(new FakeChatSdkRuntime(), undefined, { githubWebhookReplayBarrier: async () => { markReplayReady(); await replayRelease; }, }).service; const delivery = `github-stale-generation-${randomUUID()}`; const marker = `stale-body-${randomUUID()}`; const pendingResponse = receiver.handleWebhook( configured.endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery, event: "installation_repositories", payload: { action: "opened", installation: { id: 2468 }, marker, }, webhookSecret: configured.webhookSecret, }), ); await replayReady; const action = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, configured.endpoint.id), eq( chatActions.providerActionId, `github_webhook_ingress:${delivery}`, ), ), ) .then((rows) => rows[0]!); await configured.service.generateSetupSecret( configured.endpoint.id, "owner-user", ); releaseReplay(); await expect(pendingResponse).resolves.toMatchObject({ status: 202 }); const cancelled = await db .select() .from(chatActions) .where(eq(chatActions.id, action.id)) .then((rows) => rows[0]); expect(cancelled).toMatchObject({ status: "cancelled", result: expect.objectContaining({ code: "github_webhook_ingress_runtime_superseded", }), payload: expect.objectContaining({ redacted: true }), }); expect(cancelled?.payload).not.toHaveProperty("body"); await receiver.shutdown(); await configured.service.shutdown(); }); it("preserves the GitHub resource, task, and conversation across repository rename and transfer", async () => { const fixture = await seedCompany(); const context = await configuredGitHubEndpoint(fixture); const { callbacks, endpoint, service, webhookSecret } = context; const originalResource = (await service.listResources(endpoint.id))[0]!; await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: makeThread({ channelId: "PaperclipAI/Paperclip", id: "github:PaperclipAI/Paperclip:issue:77", name: "PaperclipAI/Paperclip", }).thread, message: makeMessage({ id: "github-rename-root", text: "@maya keep this task through repository moves", mentioned: true, }), trigger: "mention", }); const originalConversation = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)) .then((rows) => rows[0]!); const renamedRepository = { id: 97531, full_name: "paperclipai/paperclip-renamed", html_url: "https://github.com/paperclipai/paperclip-renamed", owner: { id: 1357, login: "paperclipai" }, private: false, }; context.setRepositories([renamedRepository]); await service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: "github-repository-renamed-inventory", event: "installation_repositories", payload: { action: "added", installation: { id: 2468 }, repositories_added: [renamedRepository], repositories_removed: [], }, webhookSecret, }), ); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ id: originalResource.id, providerResourceId: "paperclipai/paperclip-renamed", enabled: true, metadata: expect.objectContaining({ providerRepositoryId: "97531" }), }), ]); const transferredRepository = { ...renamedRepository, full_name: "new-owner/paperclip-renamed", html_url: "https://github.com/new-owner/paperclip-renamed", owner: { id: 24680, login: "new-owner" }, }; // installation_repositories is a canonical inventory-refresh signal. Keep // the provider fixture consistent with the transfer carried by the event. context.setRepositories([transferredRepository]); await service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: "github-repository-transferred-webhook", event: "installation_repositories", payload: { action: "opened", installation: { id: 2468 }, repository: transferredRepository, }, webhookSecret, }), ); const [migratedConversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); expect(migratedConversation).toMatchObject({ id: originalConversation.id, issueId: originalConversation.issueId, resourceId: originalResource.id, externalConversationId: "new-owner/paperclip-renamed", externalThreadId: "github:new-owner/paperclip-renamed:issue:77", providerUrl: "https://github.com/new-owner/paperclip-renamed/issues/77#issuecomment-github-rename-root", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: makeThread({ channelId: "new-owner/paperclip-renamed", id: "github:new-owner/paperclip-renamed:issue:77", name: "new-owner/paperclip-renamed", }).thread, message: makeMessage({ id: "github-transfer-followup", text: "a follow-up after the transfer", }), trigger: "subscribed_message", }); await expect( db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)), ).resolves.toEqual([ expect.objectContaining({ id: originalConversation.id, issueId: originalConversation.issueId, }), ]); await expect( db .select({ id: issues.id }) .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).resolves.toHaveLength(1); }); it("configures a customer-owned Microsoft Teams bot with the entered credentials", async () => { const fixture = await seedCompany(); const clientId = "00000000-0000-4000-8000-000000000001"; const tenantId = "00000000-0000-4000-8000-000000000002"; const clientSecret = "teams-client-secret"; let observedTokenRequest: URLSearchParams | null = null; const providerFetch = vi.fn( async (input: string | URL | Request, init?: RequestInit) => { const url = String(input); expect(url).toBe( `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`, ); expect(init?.method).toBe("POST"); observedTokenRequest = new URLSearchParams(String(init?.body)); return new Response(JSON.stringify({ access_token: "teams-access" }), { status: 200, headers: { "content-type": "application/json" }, }); }, ) as unknown as typeof globalThis.fetch; const { runtime, service } = createService( new FakeChatSdkRuntime(), providerFetch, ); const endpoint = await service.create( fixture.companyId, { provider: "microsoft-teams", assignedAgentId: fixture.assignedAgentId, }, "owner-user", ); const configured = await service.configure( endpoint.id, { action: "configure", credentials: { clientId, tenantId, clientSecret }, }, "owner-user", ); expect(Object.fromEntries(observedTokenRequest ?? [])).toEqual({ client_id: clientId, client_secret: clientSecret, grant_type: "client_credentials", scope: "https://api.botframework.com/.default", }); expect(configured).toMatchObject({ status: "verifying", providerAccountId: tenantId, botExternalId: clientId, allowGroupChats: false, capabilities: { nativeStreaming: false, messageEdits: true, messageDeletes: true, }, setup: { step: "test" }, }); expect( runtime.configurations.get(endpoint.id)?.providerConfig, ).toMatchObject({ provider: "microsoft-teams", credentials: { appId: clientId, appPassword: clientSecret, appTenantId: tenantId, appType: "SingleTenant", }, }); }); it("globally fences one Microsoft Bot application across tenants", async () => { const firstCompany = await seedCompany(); const secondCompany = await seedCompany(); const clientId = "00000000-0000-4000-8000-000000000091"; const providerFetch = (async () => new Response(JSON.stringify({ access_token: "teams-access" }), { status: 200, headers: { "content-type": "application/json" }, })) as typeof globalThis.fetch; let claimCount = 0; let releaseClaims!: () => void; const bothClaimsReady = new Promise((resolve) => { releaseClaims = resolve; }); const claimBarrier = async () => { claimCount += 1; if (claimCount === 2) releaseClaims(); await bothClaimsReady; }; const first = createService(new FakeChatSdkRuntime(), providerFetch, { nativeBotIdentityClaimBarrier: claimBarrier, }); const second = createService(new FakeChatSdkRuntime(), providerFetch, { nativeBotIdentityClaimBarrier: claimBarrier, }); const firstEndpoint = await first.service.create( firstCompany.companyId, { provider: "microsoft-teams", assignedAgentId: firstCompany.assignedAgentId, }, "owner-user", ); const secondEndpoint = await second.service.create( secondCompany.companyId, { provider: "microsoft-teams", assignedAgentId: secondCompany.assignedAgentId, }, "owner-user", ); const endpoints = [firstEndpoint, secondEndpoint] as const; const services = [first.service, second.service] as const; const tenantIds = [ "00000000-0000-4000-8000-000000000092", "00000000-0000-4000-8000-000000000093", ] as const; const outcomes = await Promise.allSettled( services.map((service, index) => service.configure( endpoints[index]!.id, { action: "configure", credentials: { clientId, tenantId: tenantIds[index]!, clientSecret: `teams-secret-${index}`, }, }, "owner-user", ), ), ); expect( outcomes.filter((outcome) => outcome.status === "fulfilled"), ).toHaveLength(1); const rejectedIndex = outcomes.findIndex( (outcome) => outcome.status === "rejected", ); expect(rejectedIndex).toBeGreaterThanOrEqual(0); const conflict = (outcomes[rejectedIndex] as PromiseRejectedResult).reason; expect(conflict).toMatchObject({ status: 409, details: { code: "chat_bot_identity_in_use" }, }); expect(conflict).not.toMatchObject({ details: { endpointId: expect.any(String), assignedAgentId: expect.any(String), }, }); await expect( db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoints[rejectedIndex]!.connectionId)) .then((rows) => rows[0]?.refs ?? []), ).resolves.toEqual([]); await first.service.shutdown(); await second.service.shutdown(); }); it("rehydrates one Discord Gateway owner on startup and hands off cleanly across replicas", async () => { const fixture = await seedCompany(); const applicationId = uniqueDiscordApplicationId(); const guildId = "1457808928258658549"; const providerFetch = fakeDiscordFetch( applicationId, guildId, ) as typeof globalThis.fetch; const ownerRuntime = new FakeChatSdkRuntime(); const owner = createService(ownerRuntime, providerFetch); const endpoint = await owner.service.create( fixture.companyId, { provider: "discord", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await owner.service.configure( endpoint.id, { action: "configure", credentials: { applicationId, botToken: "discord-secret", guildId, }, }, "owner-user", ); expect( ownerRuntime.configurations.get(endpoint.id)?.enableDiscordGateway, ).toBe(true); const standbyRuntime = new FakeChatSdkRuntime(); const standby = createService(standbyRuntime, providerFetch); await expect(standby.service.reconcileProviderRuntimes()).resolves.toEqual({ eligible: 1, local: 0, ownedElsewhere: 1, failed: 0, }); expect(standbyRuntime.endpoints.has(endpoint.id)).toBe(false); await expect( db .select({ token: chatEndpointLeases.token }) .from(chatEndpointLeases) .where( and( eq(chatEndpointLeases.endpointId, endpoint.id), eq(chatEndpointLeases.leaseKey, "discord_gateway_runtime"), ), ), ).resolves.toHaveLength(1); const firstRuntime = ownerRuntime.endpoints.get(endpoint.id); await owner.service.shutdown(); expect(firstRuntime?.shutdown).toHaveBeenCalledOnce(); const reconciled = await standby.service.reconcileProviderRuntimes(); // Earlier integration cases intentionally leave durable active endpoints // behind in this shared database. A fresh standby must recover all // currently unowned Discord gateways, including this test's endpoint. expect(reconciled).toMatchObject({ ownedElsewhere: 0, failed: 0 }); expect(reconciled.local).toBe(reconciled.eligible); expect(reconciled.local).toBeGreaterThanOrEqual(1); expect( standbyRuntime.configurations.get(endpoint.id)?.enableDiscordGateway, ).toBe(true); await standby.service.shutdown(); await expect( db .select({ token: chatEndpointLeases.token }) .from(chatEndpointLeases) .where(eq(chatEndpointLeases.endpointId, endpoint.id)), ).resolves.toEqual([]); }); it("recovers buffered Discord messages and reactions after an unreclaimed host-pause lease expiry", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredDiscordEndpoint(fixture, { discordGatewayLeaseRenewalIntervalMs: 60_000, discordGatewayLeaseTtlMs: 200, }); const ownedRuntime = runtime.endpoints.get(endpoint.id); if (!ownedRuntime) throw new Error("Expected Discord Gateway owner runtime"); const [leaseBeforePause] = await db .select({ token: chatEndpointLeases.token }) .from(chatEndpointLeases) .where( and( eq(chatEndpointLeases.endpointId, endpoint.id), eq(chatEndpointLeases.leaseKey, "discord_gateway_runtime"), ), ); if (!leaseBeforePause) throw new Error("Expected Discord Gateway lease"); await new Promise((resolve) => setTimeout(resolve, 250)); const messageId = "555555555555555601"; const channel = makeThread({ channelId: "333333333333333333", id: `discord:1457808928258658549:333333333333333333:${messageId}`, name: "discord-host-pause", }); const message = makeMessage({ id: messageId, mentioned: true, text: "@maya retain this buffered Discord turn", userId: "444444444444444444", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "discord", thread: channel.thread, message, trigger: "mention", }); await expect( db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, `${channel.thread.id}:${messageId}`, ), ), ), ).resolves.toHaveLength(1); expect(runtime.endpoints.get(endpoint.id)).toBe(ownedRuntime); expect(ownedRuntime.shutdown).not.toHaveBeenCalled(); await expect( db .select({ token: chatEndpointLeases.token }) .from(chatEndpointLeases) .where( and( eq(chatEndpointLeases.endpointId, endpoint.id), eq(chatEndpointLeases.leaseKey, "discord_gateway_runtime"), ), ), ).resolves.toEqual([{ token: leaseBeforePause.token }]); await qualifySetupRoundTrip(service, endpoint.id, message.author.userId); await service.test(endpoint.id, "owner-user"); if (!callbacks.onReaction) { throw new Error("Expected Discord reaction callback"); } const commentCountBefore = await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.companyId, fixture.companyId)) .then((rows) => rows.length); const wakeupCountBefore = wakeup.mock.calls.length; await new Promise((resolve) => setTimeout(resolve, 250)); const emoji = { name: "thumbsup", toJSON: () => "👍", toString: () => "👍", }; const reaction = { endpointId: endpoint.id, provider: "discord" as const, event: { adapter: {} as never, added: true, emoji, message, messageId, raw: { channel_id: "333333333333333333", emoji: { id: null, name: "👍" }, gateway_dispatch: { eventType: "MESSAGE_REACTION_ADD", sequence: 801, sessionFingerprint: "c".repeat(24), shardId: 0, }, guild_id: "1457808928258658549", message_id: messageId, user_id: message.author.userId, }, rawEmoji: "👍", thread: channel.thread, threadId: channel.thread.id, user: message.author, }, }; await callbacks.onReaction(reaction); await callbacks.onReaction(reaction); await expect( db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "reaction_added"), sql`${chatDeliveries.normalizedEvent}->'message'->>'providerMessageId' = ${messageId}`, ), ), ).resolves.toEqual([{ state: "processed" }]); expect( await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.companyId, fixture.companyId)) .then((rows) => rows.length), ).toBe(commentCountBefore); expect(wakeup).toHaveBeenCalledTimes(wakeupCountBefore); expect(runtime.endpoints.get(endpoint.id)).toBe(ownedRuntime); expect(ownedRuntime.shutdown).not.toHaveBeenCalled(); await service.shutdown(); }); it("stops a Discord Gateway on lease loss and lets a standby take over", async () => { const fixture = await seedCompany(); const applicationId = uniqueDiscordApplicationId(); const guildId = "1457808928258658549"; const providerFetch = fakeDiscordFetch( applicationId, guildId, ) as typeof globalThis.fetch; const ownerRuntime = new FakeChatSdkRuntime(); const owner = createService(ownerRuntime, providerFetch, { discordGatewayLeaseRenewalIntervalMs: 60_000, discordGatewayLeaseTtlMs: 200, }); const endpoint = await owner.service.create( fixture.companyId, { provider: "discord", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await owner.service.configure( endpoint.id, { action: "configure", credentials: { applicationId, botToken: "discord-secret", guildId, }, }, "owner-user", ); const staleRuntime = ownerRuntime.endpoints.get(endpoint.id); const staleAdmission = ownerRuntime.configurations.get(endpoint.id) ?.callbacks.onDiscordRootMentionAdmission; const staleCallbacks = ownerRuntime.configurations.get( endpoint.id, )?.callbacks; if (!staleAdmission || !staleCallbacks) throw new Error("Expected Discord admission callbacks"); const standbyRuntime = new FakeChatSdkRuntime(); const standby = createService(standbyRuntime, providerFetch); await new Promise((resolve) => setTimeout(resolve, 250)); const reconciled = await standby.service.reconcileProviderRuntimes(); expect(reconciled).toMatchObject({ ownedElsewhere: 0, failed: 0 }); expect(reconciled.local).toBe(reconciled.eligible); expect(reconciled.local).toBeGreaterThanOrEqual(1); expect( standbyRuntime.configurations.get(endpoint.id)?.enableDiscordGateway, ).toBe(true); await expect( staleAdmission({ channelId: "333333333333333333", endpointId: endpoint.id, guildId, messageId: "555555555555555555", threadId: `discord:${guildId}:333333333333333333:555555555555555555`, userId: "444444444444444444", }), ).resolves.toBe(false); expect(staleRuntime?.shutdown).toHaveBeenCalledOnce(); const freshCallbacks = standbyRuntime.configurations.get( endpoint.id, )?.callbacks; if (!freshCallbacks) throw new Error("Expected standby Discord admission callbacks"); const messageId = "555555555555555602"; const channel = makeThread({ channelId: "333333333333333333", id: `discord:${guildId}:333333333333333333:${messageId}`, name: "discord-standby-takeover", }); const message = makeMessage({ id: messageId, mentioned: true, text: "@maya accept this once after Gateway takeover", userId: "444444444444444444", }); await deliverMessage({ callbacks: staleCallbacks, endpointId: endpoint.id, provider: "discord", thread: channel.thread, message, trigger: "mention", }); await deliverMessage({ callbacks: freshCallbacks, endpointId: endpoint.id, provider: "discord", thread: channel.thread, message, trigger: "mention", }); await expect( db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, `${channel.thread.id}:${messageId}`, ), ), ), ).resolves.toHaveLength(1); await Promise.all([owner.service.shutdown(), standby.service.shutdown()]); }); it("fences message and reaction callbacks that resume after Discord Gateway takeover", async () => { const fixture = await seedCompany(); const applicationId = uniqueDiscordApplicationId(); const guildId = "1457808928258658549"; const channelId = "333333333333333334"; let holdOldCallbacks = false; let releaseOldCallbacks!: () => void; let messageAdmissionEntered!: () => void; let reactionAdmissionEntered!: () => void; const oldCallbacksReleased = new Promise((resolve) => { releaseOldCallbacks = resolve; }); const messageAdmissionReached = new Promise((resolve) => { messageAdmissionEntered = resolve; }); const reactionAdmissionReached = new Promise((resolve) => { reactionAdmissionEntered = resolve; }); const providerFetch = fakeDiscordFetch( applicationId, guildId, ) as typeof globalThis.fetch; const ownerRuntime = new FakeChatSdkRuntime(); const owner = createService(ownerRuntime, providerFetch, { discordGatewayLeaseRenewalIntervalMs: 60_000, discordGatewayLeaseTtlMs: 200, discordGatewayMessageAdmissionBarrier: async () => { if (!holdOldCallbacks) return; messageAdmissionEntered(); await oldCallbacksReleased; }, reactionLinkPreflightBarrier: async () => { if (!holdOldCallbacks) return; reactionAdmissionEntered(); await oldCallbacksReleased; }, }); const endpoint = await owner.service.create( fixture.companyId, { provider: "discord", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await owner.service.configure( endpoint.id, { action: "configure", credentials: { applicationId, botToken: "discord-secret", guildId, }, }, "owner-user", ); const rootMessageId = "555555555555555603"; const channel = makeThread({ channelId, id: `discord:${guildId}:${channelId}:${rootMessageId}`, name: "discord-callback-takeover", }); const rootMessage = makeMessage({ id: rootMessageId, mentioned: true, text: "@maya establish the takeover fixture", userId: "444444444444444445", }); const setupCallbacks = ownerRuntime.configurations.get( endpoint.id, )?.callbacks; if (!setupCallbacks) throw new Error("Expected owner Discord callbacks"); await deliverMessage({ callbacks: setupCallbacks, endpointId: endpoint.id, provider: "discord", thread: channel.thread, message: rootMessage, trigger: "mention", }); await qualifySetupRoundTrip( owner.service, endpoint.id, rootMessage.author.userId, ); await owner.service.test(endpoint.id, "owner-user"); const staleRuntime = ownerRuntime.endpoints.get(endpoint.id); const staleCallbacks = ownerRuntime.configurations.get( endpoint.id, )?.callbacks; if (!staleRuntime || !staleCallbacks?.onReaction) { throw new Error("Expected active owner Discord callbacks"); } const [leaseBeforeTakeover] = await db .select({ token: chatEndpointLeases.token }) .from(chatEndpointLeases) .where( and( eq(chatEndpointLeases.endpointId, endpoint.id), eq(chatEndpointLeases.leaseKey, "discord_gateway_runtime"), ), ); if (!leaseBeforeTakeover) throw new Error("Expected owner Gateway lease"); const messageId = "555555555555555604"; const bufferedMessage = makeMessage({ id: messageId, text: "continue this once after takeover", userId: rootMessage.author.userId, }); const emoji = { name: "thumbsup", toJSON: () => "👍", toString: () => "👍", }; const reaction = { endpointId: endpoint.id, provider: "discord" as const, event: { adapter: {} as never, added: true, emoji, message: rootMessage, messageId: rootMessageId, raw: { channel_id: channelId, emoji: { id: null, name: "👍" }, gateway_dispatch: { eventType: "MESSAGE_REACTION_ADD", sequence: 802, sessionFingerprint: "d".repeat(24), shardId: 0, }, guild_id: guildId, message_id: rootMessageId, user_id: rootMessage.author.userId, }, rawEmoji: "👍", thread: channel.thread, threadId: channel.thread.id, user: rootMessage.author, }, }; const commentCountBefore = await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.companyId, fixture.companyId)) .then((rows) => rows.length); const wakeupCountBefore = owner.wakeup.mock.calls.length; holdOldCallbacks = true; const staleMessage = deliverMessage({ callbacks: staleCallbacks, endpointId: endpoint.id, provider: "discord", thread: channel.thread, message: bufferedMessage, trigger: "subscribed_message", }); const staleReaction = staleCallbacks.onReaction(reaction); await Promise.all([messageAdmissionReached, reactionAdmissionReached]); const standbyRuntime = new FakeChatSdkRuntime(); const standby = createService(standbyRuntime, providerFetch); try { await new Promise((resolve) => setTimeout(resolve, 250)); const reconciled = await standby.service.reconcileProviderRuntimes(); expect(reconciled).toMatchObject({ ownedElsewhere: 0, failed: 0 }); const freshCallbacks = standbyRuntime.configurations.get( endpoint.id, )?.callbacks; if (!freshCallbacks?.onReaction) { throw new Error("Expected standby Discord callbacks"); } const [leaseAfterTakeover] = await db .select({ token: chatEndpointLeases.token }) .from(chatEndpointLeases) .where( and( eq(chatEndpointLeases.endpointId, endpoint.id), eq(chatEndpointLeases.leaseKey, "discord_gateway_runtime"), ), ); expect(leaseAfterTakeover?.token).toBeTruthy(); expect(leaseAfterTakeover?.token).not.toBe(leaseBeforeTakeover.token); releaseOldCallbacks(); await Promise.all([staleMessage, staleReaction]); await expect( db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), or( eq( chatDeliveries.providerEventId, `${channel.thread.id}:${messageId}`, ), and( eq(chatDeliveries.eventKind, "reaction_added"), sql`${chatDeliveries.normalizedEvent}->'message'->>'providerMessageId' = ${rootMessageId}`, ), ), ), ), ).resolves.toEqual([]); await vi.waitFor(() => { expect(staleRuntime.shutdown).toHaveBeenCalledOnce(); }); await deliverMessage({ callbacks: freshCallbacks, endpointId: endpoint.id, provider: "discord", thread: channel.thread, message: bufferedMessage, trigger: "subscribed_message", }); await freshCallbacks.onReaction(reaction); await deliverMessage({ callbacks: freshCallbacks, endpointId: endpoint.id, provider: "discord", thread: channel.thread, message: bufferedMessage, trigger: "subscribed_message", }); await freshCallbacks.onReaction(reaction); await expect( db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, `${channel.thread.id}:${messageId}`, ), ), ), ).resolves.toEqual([{ state: "processed" }]); await expect( db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "reaction_added"), sql`${chatDeliveries.normalizedEvent}->'message'->>'providerMessageId' = ${rootMessageId}`, ), ), ).resolves.toEqual([{ state: "processed" }]); expect( await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.companyId, fixture.companyId)) .then((rows) => rows.length), ).toBe(commentCountBefore + 1); expect(owner.wakeup).toHaveBeenCalledTimes(wakeupCountBefore); expect(standby.wakeup).toHaveBeenCalledOnce(); } finally { releaseOldCallbacks(); await Promise.allSettled([staleMessage, staleReaction]); await Promise.allSettled([ owner.service.shutdown(), standby.service.shutdown(), ]); } }); it("releases Discord ownership when runtime construction fails", async () => { const fixture = await seedCompany(); const applicationId = uniqueDiscordApplicationId(); const runtime = new FakeChatSdkRuntime(); runtime.initializeHook = async () => { throw new Error("constructor failed"); }; const { service } = createService( runtime, fakeDiscordFetch(applicationId) as typeof globalThis.fetch, ); const endpoint = await service.create( fixture.companyId, { provider: "discord", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await expect( service.configure( endpoint.id, { action: "configure", credentials: { applicationId, botToken: "discord-secret", guildId: "1457808928258658549", }, }, "owner-user", ), ).rejects.toThrow("constructor failed"); await expect( db .select({ token: chatEndpointLeases.token }) .from(chatEndpointLeases) .where(eq(chatEndpointLeases.endpointId, endpoint.id)), ).resolves.toEqual([]); await service.shutdown(); }); it("serializes Discord Gateway health events and recovers transient guild outages", async () => { const fixture = await seedCompany(); const applicationId = uniqueDiscordApplicationId(); let releaseDisconnect!: () => void; let disconnectEntered!: () => void; const entered = new Promise((resolve) => { disconnectEntered = resolve; }); const disconnectBarrier = new Promise((resolve) => { releaseDisconnect = resolve; }); const runtime = new FakeChatSdkRuntime(); const context = createService( runtime, fakeDiscordFetch(applicationId) as typeof globalThis.fetch, { discordGatewayEventBarrier: async (callback) => { if (callback.event.type !== "disconnected") return; disconnectEntered(); await disconnectBarrier; }, }, ); const endpoint = await context.service.create( fixture.companyId, { provider: "discord", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await context.service.configure( endpoint.id, { action: "configure", credentials: { applicationId, botToken: "discord-secret", guildId: "1457808928258658549", }, }, "owner-user", ); const setup = await db .select({ setup: chatEndpoints.setup }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)) .then((rows) => rows[0]!.setup); await db .update(chatEndpoints) .set({ status: "active", setup: { ...setup, step: "complete" } }) .where(eq(chatEndpoints.id, endpoint.id)); const onGatewayEvent = runtime.configurations.get(endpoint.id)?.callbacks .onDiscordGatewayEvent; if (!onGatewayEvent) throw new Error("Expected Discord Gateway callback"); const disconnect = Promise.resolve( onGatewayEvent({ endpointId: endpoint.id, provider: "discord", sequence: 1, event: { type: "disconnected", fatal: false, code: 1001 }, }), ); await entered; let recoveryFinished = false; const recovery = Promise.resolve( onGatewayEvent({ endpointId: endpoint.id, provider: "discord", sequence: 2, event: { type: "ready" }, }), ).then(() => { recoveryFinished = true; }); await Promise.resolve(); expect(recoveryFinished).toBe(false); releaseDisconnect(); await Promise.all([disconnect, recovery]); await expect(context.service.get(endpoint.id)).resolves.toMatchObject({ status: "active", healthMessage: "Connected", }); await onGatewayEvent({ endpointId: endpoint.id, provider: "discord", sequence: 3, event: { type: "guild_unavailable", guildId: "1457808928258658549", }, }); await expect(context.service.get(endpoint.id)).resolves.toMatchObject({ status: "active", healthMessage: expect.stringContaining("temporarily unavailable"), }); await onGatewayEvent({ endpointId: endpoint.id, provider: "discord", sequence: 4, event: { type: "guild_available", guildId: "1457808928258658549", }, }); await expect(context.service.get(endpoint.id)).resolves.toMatchObject({ status: "active", healthMessage: "Connected", }); await context.service.shutdown(); }); it("does not let pause overtake a credential-fenced Discord root thread creation", async () => { const fixture = await seedCompany(); const applicationId = uniqueDiscordApplicationId(); let releaseTransport!: () => void; let transportEntered!: () => void; const entered = new Promise((resolve) => { transportEntered = resolve; }); const transportBarrier = new Promise((resolve) => { releaseTransport = resolve; }); const runtime = new FakeChatSdkRuntime(); const { service } = createService( runtime, fakeDiscordFetch(applicationId) as typeof globalThis.fetch, { discordRootThreadTransportBarrier: async () => { transportEntered(); await transportBarrier; }, scheduleDeferredWork: () => undefined, }, ); const endpoint = await service.create( fixture.companyId, { provider: "discord", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await service.configure( endpoint.id, { action: "configure", credentials: { applicationId, botToken: "discord-secret", guildId: "1457808928258658549", }, }, "owner-user", ); const setup = await db .select({ setup: chatEndpoints.setup }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)) .then((rows) => rows[0]!.setup); await db .update(chatEndpoints) .set({ allowUnlinkedPeople: true, status: "active", setup: { ...setup, step: "complete" }, }) .where(eq(chatEndpoints.id, endpoint.id)); await db .update(chatEndpointResources) .set({ enabled: true }) .where(eq(chatEndpointResources.endpointId, endpoint.id)); const admission = runtime.configurations.get(endpoint.id)?.callbacks .onDiscordRootMentionAdmission; if (!admission) throw new Error("Expected Discord admission callback"); const ownedRuntime = runtime.endpoints.get(endpoint.id); if (!ownedRuntime) throw new Error("Expected Discord Gateway owner runtime"); const messageId = "555555555555555590"; const threadId = `discord:1457808928258658549:333333333333333333:${messageId}`; const message = { ...makeMessage({ id: messageId, mentioned: true, text: "@maya serialize this root", userId: "444444444444444444", }), threadId, } as Message; const admitting = Promise.resolve( admission({ channelId: "333333333333333333", endpointId: endpoint.id, guildId: "1457808928258658549", message, messageId, threadId, userId: "444444444444444444", }), ); await entered; let pauseSettled = false; const pausing = service .configure(endpoint.id, { action: "pause" }, "owner-user") .then((result) => { pauseSettled = true; return result; }); await Promise.resolve(); expect(pauseSettled).toBe(false); releaseTransport(); await expect(admitting).resolves.toBe(false); await expect(pausing).resolves.toMatchObject({ status: "paused" }); expect(ownedRuntime.ensuredDiscordRootThreads).toEqual([ { channelId: "333333333333333333", content: "@maya serialize this root", messageId, }, ]); expect(runtime.configurations.get(endpoint.id)?.enableDiscordGateway).toBe( true, ); expect(runtime.endpoints.has(endpoint.id)).toBe(false); await service.shutdown(); }); it("configures a Discord bot only after identity, intent, server, and channel permissions verify", async () => { const fixture = await seedCompany(); const applicationId = uniqueDiscordApplicationId(); const guildId = "1457808928258658549"; const providerFetch = vi.fn( async (input: string | URL | Request, init?: RequestInit) => { expect(new Headers(init?.headers).get("authorization")).toMatch( /^Bot discord-secret(?:-rotated)?$/, ); const path = new URL(String(input)).pathname; const responses: Record = { "/api/v10/users/@me": { id: applicationId, username: "maya", global_name: "Maya", bot: true, }, "/api/v10/oauth2/applications/@me": { id: applicationId, name: "Maya", flags: 1 << 18, }, [`/api/v10/guilds/${guildId}`]: { id: guildId, name: "Clawd" }, [`/api/v10/guilds/${guildId}/members/${applicationId}`]: { roles: ["222222222222222222"], user: { id: applicationId }, }, [`/api/v10/guilds/${guildId}/roles`]: [ { id: "222222222222222222", permissions: "309237763136", }, ], [`/api/v10/guilds/${guildId}/channels`]: [ { id: "333333333333333333", name: "agent-lab", position: 1, type: 0, }, ], }; return new Response(JSON.stringify(responses[path]), { status: path in responses ? 200 : 404, headers: { "content-type": "application/json" }, }); }, ) as unknown as typeof globalThis.fetch; const { runtime, service } = createService( new FakeChatSdkRuntime(), providerFetch, ); const endpoint = await service.create( fixture.companyId, { provider: "discord", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); const configured = await service.configure( endpoint.id, { action: "configure", credentials: { applicationId, botToken: "discord-secret", guildId, }, }, "owner-user", ); expect(configured).toMatchObject({ provider: "discord", providerAccountId: guildId, providerAccountLabel: "Clawd", botExternalId: applicationId, botUsername: "maya", status: "verifying", setup: { step: "test" }, }); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ providerResourceId: "333333333333333333", parentProviderResourceId: guildId, label: "#agent-lab", enabled: false, }), ]); expect( runtime.configurations.get(endpoint.id)?.providerConfig, ).toMatchObject({ provider: "discord", credentials: { applicationId, botToken: "discord-secret", guildId, }, }); const connection = await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)) .then((rows) => rows[0]!); expect(connection.refs).toHaveLength(3); expect(JSON.stringify(connection)).not.toContain("discord-secret"); const callbacks = runtime.configurations.get(endpoint.id)?.callbacks; const admitRootMention = callbacks?.onDiscordRootMentionAdmission; if (!admitRootMention) throw new Error("Expected Discord root-mention admission callback"); const rootMention = (messageId: string) => { const threadId = `discord:${guildId}:333333333333333333:${messageId}`; const message = { ...makeMessage({ id: messageId, text: "@maya investigate the queue", mentioned: true, userId: "444444444444444444", }), threadId, } as Message; return { message, thread: makeThread({ channelId: "333333333333333333", id: threadId, }).thread, threadId, }; }; const admission = async (messageId: string) => { const root = rootMention(messageId); return await admitRootMention({ endpointId: endpoint.id, guildId, channelId: "333333333333333333", messageId, message: root.message, threadId: root.threadId, userId: "444444444444444444", }); }; await expect(admission("555555555555555551")).resolves.toBe(false); const [admittedRoot] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, `discord:${guildId}:333333333333333333:555555555555555551:555555555555555551`, ), ), ); expect(admittedRoot).toMatchObject({ state: "received", redactedError: null, normalizedEvent: expect.objectContaining({ message: expect.objectContaining({ text: "@maya investigate the queue", }), }), }); expect(admittedRoot?.nextAttemptAt).toBeNull(); expect( runtime.endpoints.get(endpoint.id)?.ensuredDiscordRootThreads, ).toContainEqual({ channelId: "333333333333333333", messageId: "555555555555555551", content: "@maya investigate the queue", }); await expect(service.listConversations(endpoint.id)).resolves.toHaveLength( 0, ); await service.processPendingDeliveries(25, admittedRoot!.id); await vi.waitFor(async () => { await expect( db .select({ normalizedEvent: chatDeliveries.normalizedEvent, state: chatDeliveries.state, }) .from(chatDeliveries) .where(eq(chatDeliveries.id, admittedRoot!.id)), ).resolves.toEqual([ expect.objectContaining({ state: "processed", normalizedEvent: expect.not.objectContaining({ providerThreadPending: true, }), }), ]); }); await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, endpoint.id)); await db .update(chatEndpointResources) .set({ enabled: false }) .where(eq(chatEndpointResources.endpointId, endpoint.id)); await expect(admission("555555555555555552")).resolves.toBe(false); await expect( db .select({ normalizedEvent: chatDeliveries.normalizedEvent, redactedError: chatDeliveries.redactedError, state: chatDeliveries.state, }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, `discord:${guildId}:333333333333333333:555555555555555552:555555555555555552`, ), ), ), ).resolves.toEqual([ { normalizedEvent: expect.objectContaining({ filtering: { contentRetained: false, providerThreadCreated: false, }, }), redactedError: "Destination is not enabled in Paperclip", state: "filtered", }, ]); await db .update(chatEndpointResources) .set({ enabled: true }) .where(eq(chatEndpointResources.endpointId, endpoint.id)); await db .update(chatEndpoints) .set({ allowGroupChats: false }) .where(eq(chatEndpoints.id, endpoint.id)); await expect(admission("555555555555555553")).resolves.toBe(false); await db .update(chatEndpoints) .set({ allowGroupChats: true, allowUnlinkedPeople: false }) .where(eq(chatEndpoints.id, endpoint.id)); await expect(admission("555555555555555554")).resolves.toBe(false); await db .update(chatEndpoints) .set({ allowUnlinkedPeople: true }) .where(eq(chatEndpoints.id, endpoint.id)); const activeRuntime = runtime.endpoints.get(endpoint.id); if (!activeRuntime) throw new Error("Expected active Discord runtime"); const ensureDiscordRootThread = activeRuntime.ensureDiscordRootThread; activeRuntime.ensureDiscordRootThread = vi.fn(async () => { throw new Error("temporary Discord API failure"); }); await expect(admission("555555555555555555")).resolves.toBe(false); const [interruptedRoot] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, `discord:${guildId}:333333333333333333:555555555555555555:555555555555555555`, ), ), ); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, interruptedRoot!.id)); activeRuntime.ensureDiscordRootThread = ensureDiscordRootThread; await service.processPendingDeliveries(25, interruptedRoot!.id); expect( runtime.endpoints.get(endpoint.id)?.ensuredDiscordRootThreads, ).toContainEqual({ channelId: "333333333333333333", messageId: "555555555555555555", content: "@maya investigate the queue", }); await expect( db .select({ normalizedEvent: chatDeliveries.normalizedEvent, state: chatDeliveries.state, }) .from(chatDeliveries) .where(eq(chatDeliveries.id, interruptedRoot!.id)), ).resolves.toEqual([ expect.objectContaining({ state: "processed", normalizedEvent: expect.not.objectContaining({ providerThreadPending: true, }), }), ]); const runtimeBeforeRotation = runtime.endpoints.get(endpoint.id); const firstSecretIds = new Set(connection.refs.map((ref) => ref.secretId)); const conversationsBeforeRotation = await service.listConversations( endpoint.id, ); expect(conversationsBeforeRotation.length).toBeGreaterThan(0); const reconnected = await service.configure( endpoint.id, { action: "reconnect", // Rotation accepts the one changed secret and merges the immutable // Application/Server identity from Paperclip's existing vault refs. credentials: { botToken: "discord-secret-rotated" }, }, "owner-user", ); expect(reconnected).toMatchObject({ id: endpoint.id, providerAccountId: guildId, botExternalId: applicationId, status: "verifying", setup: { step: "test" }, }); expect(runtimeBeforeRotation?.shutdown).toHaveBeenCalledTimes(1); expect( runtime.configurations.get(endpoint.id)?.providerConfig, ).toMatchObject({ provider: "discord", credentials: { applicationId, botToken: "discord-secret-rotated", guildId, }, }); const [rotatedConnection] = await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(rotatedConnection?.refs).toHaveLength(3); expect( rotatedConnection?.refs.every((ref) => !firstSecretIds.has(ref.secretId)), ).toBe(true); await expect(service.listConversations(endpoint.id)).resolves.toMatchObject( conversationsBeforeRotation.map((conversation) => ({ id: conversation.id, issueId: conversation.issueId, })), ); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ providerResourceId: "333333333333333333", enabled: true, }), ]); const reconnectActivity = await db .select({ action: activityLog.action, details: activityLog.details }) .from(activityLog) .where(eq(activityLog.entityId, endpoint.connectionId)) .then((rows) => rows.find((row) => row.action === "chat_endpoint.reconnected"), ); expect(reconnectActivity).toEqual({ action: "chat_endpoint.reconnected", details: { endpointId: endpoint.id, provider: "discord" }, }); expect(JSON.stringify(reconnectActivity)).not.toContain( "discord-secret-rotated", ); const reconnectedCallbacks = runtime.configurations.get( endpoint.id, )?.callbacks; const reconnectedAdmission = reconnectedCallbacks?.onDiscordRootMentionAdmission; const onDiscordGatewayEvent = reconnectedCallbacks?.onDiscordGatewayEvent; if (!reconnectedAdmission || !onDiscordGatewayEvent) { throw new Error("Expected Discord lifecycle callbacks after reconnect"); } const inaccessibleRoot = rootMention("555555555555555556"); const currentRuntime = runtime.endpoints.get(endpoint.id); if (!currentRuntime) throw new Error("Expected active Discord runtime"); currentRuntime.ensureDiscordRootThread = vi.fn(async () => { throw Object.assign(new Error("provider detail must not escape"), { adapter: "discord", code: 50013, status: 403, }); }); await expect( reconnectedAdmission({ endpointId: endpoint.id, guildId, channelId: "333333333333333333", messageId: inaccessibleRoot.message.id, message: inaccessibleRoot.message, threadId: inaccessibleRoot.threadId, userId: "444444444444444444", }), ).resolves.toBe(false); const [inaccessibleDelivery] = await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, `discord:${guildId}:333333333333333333:555555555555555556:555555555555555556`, ), ), ) .returning({ id: chatDeliveries.id }); if (!inaccessibleDelivery) { throw new Error("Expected provisional inaccessible Discord root"); } await service.processPendingDeliveries(25, inaccessibleDelivery.id); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ providerResourceId: "333333333333333333", availability: "unavailable", }), ]); await expect(service.listConversations(endpoint.id)).resolves.toEqual( conversationsBeforeRotation.map((conversation) => expect.objectContaining({ id: conversation.id, state: "unavailable", }), ), ); await db .update(chatEndpointResources) .set({ availability: "available" }) .where(eq(chatEndpointResources.endpointId, endpoint.id)); await db .update(chatConversations) .set({ state: "active" }) .where(eq(chatConversations.endpointId, endpoint.id)); await onDiscordGatewayEvent({ endpointId: endpoint.id, provider: "discord", sequence: 1, event: { type: "disconnected", fatal: false, code: 1001 }, }); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "attention", healthMessage: "Discord Gateway disconnected (code 1001); reconnecting", }); await expect( db .select({ enabled: toolConnections.enabled, healthStatus: toolConnections.healthStatus, status: toolConnections.status, }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)), ).resolves.toEqual([ { enabled: true, healthStatus: "degraded", status: "active" }, ]); await onDiscordGatewayEvent({ endpointId: endpoint.id, provider: "discord", sequence: 2, event: { type: "ready", botUserId: applicationId }, }); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "verifying", healthMessage: "Waiting for a test conversation", }); await onDiscordGatewayEvent({ endpointId: endpoint.id, provider: "discord", sequence: 3, event: { type: "channel_removed", channelId: "333333333333333333", guildId, label: "agent-lab", }, }); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ providerResourceId: "333333333333333333", availability: "removed", }), ]); await expect(service.listConversations(endpoint.id)).resolves.toEqual( conversationsBeforeRotation.map((conversation) => expect.objectContaining({ id: conversation.id, state: "unavailable", }), ), ); await onDiscordGatewayEvent({ endpointId: endpoint.id, provider: "discord", sequence: 4, event: { type: "guild_removed", guildId }, }); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "revoked", healthMessage: "Discord bot was removed from the configured server", }); await expect( db .select({ enabled: toolConnections.enabled, healthStatus: toolConnections.healthStatus, status: toolConnections.status, }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)), ).resolves.toEqual([ { enabled: false, healthStatus: "failed", status: "disabled" }, ]); await service.configure(endpoint.id, { action: "remove" }, "owner-user"); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "archived", }); await expect(service.listConversations(endpoint.id)).resolves.toEqual( conversationsBeforeRotation.map((conversation) => expect.objectContaining({ id: conversation.id, issueId: conversation.issueId, state: "endpoint_removed", }), ), ); await expect( db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)) .then((rows) => rows[0]?.refs), ).resolves.toEqual([]); await expect( db .select({ id: companySecrets.id }) .from(companySecrets) .where(eq(companySecrets.companyId, fixture.companyId)), ).resolves.toHaveLength(0); expect(runtime.endpoints.has(endpoint.id)).toBe(false); await service.shutdown(); }); it("allows only one concurrent live endpoint to claim a Discord bot across guilds", async () => { const fixture = await seedCompany(); const applicationId = "623456789012345678"; const guildIds = ["723456789012345678", "823456789012345678"]; const providerFetch = vi.fn( async (input: string | URL | Request, init?: RequestInit) => { expect(new Headers(init?.headers).get("authorization")).toMatch( /^Bot discord-secret-/, ); const path = new URL(String(input)).pathname; if (path === "/api/v10/users/@me") { return new Response( JSON.stringify({ id: applicationId, username: "shared-maya", global_name: "Shared Maya", bot: true, }), { status: 200, headers: { "content-type": "application/json" } }, ); } if (path === "/api/v10/oauth2/applications/@me") { return new Response( JSON.stringify({ id: applicationId, name: "Shared Maya", flags: 1 << 18, }), { status: 200, headers: { "content-type": "application/json" } }, ); } const guildId = guildIds.find((candidate) => path.includes(candidate)); if (!guildId) { return new Response(JSON.stringify({ message: "not found" }), { status: 404, headers: { "content-type": "application/json" }, }); } let body: unknown; if (path === `/api/v10/guilds/${guildId}`) { body = { id: guildId, name: `Guild ${guildId}` }; } else if (path.endsWith(`/members/${applicationId}`)) { body = { roles: ["923456789012345678"], user: { id: applicationId }, }; } else if (path.endsWith("/roles")) { body = [ { id: "923456789012345678", permissions: "309237763136", }, ]; } else if (path.endsWith("/channels")) { body = [ { id: guildId === guildIds[0] ? "333333333333333334" : "333333333333333335", name: "agent-lab", position: 1, type: 0, }, ]; } return new Response(JSON.stringify(body), { status: 200, headers: { "content-type": "application/json" }, }); }, ) as unknown as typeof globalThis.fetch; const { service } = createService(new FakeChatSdkRuntime(), providerFetch); const endpoints = await Promise.all([ service.create( fixture.companyId, { provider: "discord", assignedAgentId: fixture.assignedAgentId }, "owner-user", ), service.create( fixture.companyId, { provider: "discord", assignedAgentId: fixture.replacementAgentId }, "owner-user", ), ]); const outcomes = await Promise.allSettled( endpoints.map((endpoint, index) => service.configure( endpoint.id, { action: "configure", credentials: { applicationId, botToken: `discord-secret-${index}`, guildId: guildIds[index]!, }, }, "owner-user", ), ), ); expect( outcomes.filter((outcome) => outcome.status === "fulfilled"), ).toHaveLength(1); const rejected = outcomes.find( (outcome): outcome is PromiseRejectedResult => outcome.status === "rejected", ); expect(rejected?.reason).toMatchObject({ status: 409, details: { code: "chat_bot_identity_in_use" }, }); const live = await db .select({ id: chatEndpoints.id }) .from(chatEndpoints) .where( and( eq(chatEndpoints.provider, "discord"), eq(chatEndpoints.botExternalId, applicationId), inArray(chatEndpoints.status, [ "verifying", "active", "paused", "attention", ]), ), ); expect(live).toHaveLength(1); }); it("rotates Microsoft Teams secrets without changing bot identity and preserves the last good credentials on rejection", async () => { const fixture = await seedCompany(); const clientId = "00000000-0000-4000-8000-000000000011"; const tenantId = "00000000-0000-4000-8000-000000000012"; const providerFetch = vi.fn( async (_input: string | URL | Request, init?: RequestInit) => { const body = new URLSearchParams(String(init?.body)); if (body.get("client_secret") === "revoked-secret") { return new Response( JSON.stringify({ error_description: "client secret is invalid or expired", }), { status: 401, headers: { "content-type": "application/json" }, }, ); } return new Response(JSON.stringify({ access_token: "teams-access" }), { status: 200, headers: { "content-type": "application/json" }, }); }, ) as unknown as typeof globalThis.fetch; const { runtime, service } = createService( new FakeChatSdkRuntime(), providerFetch, ); const endpoint = await service.create( fixture.companyId, { provider: "microsoft-teams", assignedAgentId: fixture.assignedAgentId, }, "owner-user", ); await service.configure( endpoint.id, { action: "configure", credentials: { clientId, tenantId, clientSecret: "first-secret" }, }, "owner-user", ); const refs = () => db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)) .then((rows) => rows[0]!.refs); const firstRefs = await refs(); expect(firstRefs).toHaveLength(3); await expect( service.configure( endpoint.id, { action: "reconnect", credentials: { clientId, tenantId, clientSecret: "revoked-secret" }, }, "owner-user", ), ).rejects.toMatchObject({ status: 422 }); expect(await refs()).toEqual(firstRefs); expect(runtime.endpoints.has(endpoint.id)).toBe(true); await expect( service.configure( endpoint.id, { action: "reconnect", credentials: { clientId, tenantId, clientSecret: "rotated-secret" }, }, "owner-user", ), ).resolves.toMatchObject({ providerAccountId: tenantId, botExternalId: clientId, status: "verifying", }); const rotatedRefs = await refs(); expect(rotatedRefs).toHaveLength(3); expect( rotatedRefs.every((ref) => firstRefs.every((firstRef) => firstRef.secretId !== ref.secretId), ), ).toBe(true); await expect( db .select({ id: companySecrets.id }) .from(companySecrets) .where(eq(companySecrets.companyId, fixture.companyId)), ).resolves.toHaveLength(3); await expect( service.configure( endpoint.id, { action: "reconnect", credentials: { clientId, tenantId: "00000000-0000-4000-8000-000000000099", clientSecret: "other-tenant-secret", }, }, "owner-user", ), ).rejects.toMatchObject({ status: 409, details: { code: "chat_bot_identity_changed" }, }); expect(await refs()).toEqual(rotatedRefs); await expect(service.get(endpoint.id)).resolves.toMatchObject({ providerAccountId: tenantId, botExternalId: clientId, }); }); it("coalesces one Microsoft Teams run into one provider reply", async () => { const fixture = await seedCompany(); const clientId = randomUUID(); const tenantId = randomUUID(); const context = createService( new FakeChatSdkRuntime(), (async () => new Response(JSON.stringify({ access_token: "teams-run-access" }), { status: 200, headers: { "content-type": "application/json" }, })) as typeof globalThis.fetch, ); const endpoint = await context.service.create( fixture.companyId, { provider: "microsoft-teams", assignedAgentId: fixture.assignedAgentId, }, "owner-user", ); await context.service.configure( endpoint.id, { action: "configure", credentials: { clientId, tenantId, clientSecret: "teams-run-secret", }, }, "owner-user", ); const callbacks = context.runtime.configurations.get( endpoint.id, )?.callbacks; if (!callbacks) throw new Error("Expected Teams callbacks"); const thread = makeThread({ channelId: "teams-personal-run", id: "teams:personal-run:root-1", isDM: true, name: "Alex External", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: thread.thread, message: makeMessage({ id: "teams-run-root-1", text: "@Maya produce one quiet Teams response", mentioned: true, }), trigger: "mention", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); if (!conversation) throw new Error("Expected Teams conversation"); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "microsoft-teams", providerMessageId: "teams-run-root-1", }), }); for (const progressState of ["queued", "working"] as const) { await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${runId}:${progressState}:${endpoint.id}`, payload: { text: progressState === "queued" ? "Maya is queued." : "Maya is working…", progressState, }, state: "pending", }); await context.service.processPendingPublications(); } await addSelectedChatFinal({ agentId: fixture.assignedAgentId, body: "Final Teams result", companyId: fixture.companyId, issueId: conversation.issueId, runId, }); await context.service.processPendingPublications(); const providerRuntime = context.runtime.endpoints.get(endpoint.id); expect(providerRuntime?.posts).toEqual([ { threadId: thread.thread.id, text: "Maya is queued." }, ]); expect(providerRuntime?.edits).toEqual([ { threadId: thread.thread.id, messageId: "outbound-1", text: "Maya is working…", }, { threadId: thread.thread.id, messageId: "outbound-1", text: "Final Teams result", }, ]); const publications = await db .select() .from(chatPublications) .where(eq(chatPublications.conversationId, conversation.id)); expect(publications).toHaveLength(3); expect( publications.every( (publication) => publication.state === "published" && publication.providerMessageId === "outbound-1", ), ).toBe(true); await context.service.shutdown(); }); it("rejects unsupported Telegram webhook ports before provider access or secret persistence", async () => { const fixture = await seedCompany(); const providerFetch = vi.fn(async () => { throw new Error("Telegram provider access must not start"); }) as unknown as typeof globalThis.fetch; const { service } = createService(new FakeChatSdkRuntime(), providerFetch, { webhookPublicBaseUrl: "https://unsupported-telegram-origin.example:10000", }); const endpoint = await service.create( fixture.companyId, { provider: "telegram", assignedAgentId: fixture.assignedAgentId, }, "owner-user", ); const failure = await service .configure( endpoint.id, { action: "configure", credentials: { botToken: "123456:telegram-port-canary" }, }, "owner-user", ) .then( () => null, (error: unknown) => error, ); expect(failure).toMatchObject({ status: 422, message: "Telegram webhooks require PAPERCLIP_CHAT_WEBHOOK_PUBLIC_URL to use HTTPS on port 443, 80, 88, or 8443", details: { code: "chat_telegram_webhook_url_unsupported", provider: "telegram", supportedPorts: [443, 80, 88, 8443], }, }); expect(String((failure as Error | null)?.message)).not.toContain( "unsupported-telegram-origin", ); expect(String((failure as Error | null)?.message)).not.toContain( "telegram-port-canary", ); expect(providerFetch).not.toHaveBeenCalled(); await expect( db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)), ).resolves.toEqual([{ refs: [] }]); await expect( db .select({ id: companySecretBindings.id }) .from(companySecretBindings) .where( and( eq(companySecretBindings.companyId, fixture.companyId), eq(companySecretBindings.targetType, "tool_connection"), eq(companySecretBindings.targetId, endpoint.connectionId), ), ), ).resolves.toEqual([]); await expect( db .select({ id: companySecrets.id }) .from(companySecrets) .where(eq(companySecrets.companyId, fixture.companyId)), ).resolves.toEqual([]); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "draft", providerAccountId: null, botExternalId: null, }); await service.shutdown(); }); it("preserves stored Telegram credentials and lifecycle when reconnect uses an unsupported webhook port", async () => { const fixture = await seedCompany(); const configured = await configuredTelegramEndpoint(fixture); await configured.service.shutdown(); await db .update(chatEndpoints) .set({ status: "attention", healthMessage: "Telegram webhook registration needs attention", lastError: "Telegram webhook registration failed", updatedAt: new Date(), }) .where(eq(chatEndpoints.id, configured.endpoint.id)); const [endpointBefore] = await db .select() .from(chatEndpoints) .where(eq(chatEndpoints.id, configured.endpoint.id)); if (!endpointBefore) throw new Error("Expected Telegram endpoint row"); const [connectionBefore] = await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, configured.endpoint.connectionId)); const secretIdsBefore = await db .select({ id: companySecrets.id }) .from(companySecrets) .where(eq(companySecrets.companyId, fixture.companyId)); expect(connectionBefore?.refs).toHaveLength(2); const providerFetch = vi.fn(async () => { throw new Error("Telegram provider access must not start"); }) as unknown as typeof globalThis.fetch; const reconnect = createService(new FakeChatSdkRuntime(), providerFetch, { webhookPublicBaseUrl: "https://unsupported-telegram-reconnect.example:10000", }); await expect( reconnect.service.configure( configured.endpoint.id, { action: "reconnect" }, "owner-user", ), ).rejects.toMatchObject({ status: 422, details: { code: "chat_telegram_webhook_url_unsupported" }, }); expect(providerFetch).not.toHaveBeenCalled(); await expect( db .select() .from(chatEndpoints) .where(eq(chatEndpoints.id, configured.endpoint.id)), ).resolves.toEqual([endpointBefore]); await expect( db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, configured.endpoint.connectionId)), ).resolves.toEqual([connectionBefore]); await expect( db .select({ id: companySecrets.id }) .from(companySecrets) .where(eq(companySecrets.companyId, fixture.companyId)), ).resolves.toEqual(secretIdsBefore); await reconnect.service.shutdown(); }); it.each([ undefined, "https://ingress.example", "https://ingress.example:8443", ])( "configures Telegram and preserves queued updates with webhook origin %s", async (webhookPublicBaseUrl) => { const fixture = await seedCompany(); const botToken = "123456:telegram-test-token"; const botId = Number.parseInt( randomUUID().replaceAll("-", "").slice(0, 12), 16, ); const observedUrls: string[] = []; let existingWebhookUrl = ""; let observedWebhook: Record | null = null; let observedCommands: Record | null = null; let observedWebhookDelete: Record | null = null; let observedCommandsDelete: Record | null = null; const providerFetch = vi.fn( async (input: string | URL | Request, init?: RequestInit) => { const url = String(input); observedUrls.push(url); if (url.endsWith("/getMe")) { expect(init?.signal).toBeInstanceOf(AbortSignal); return new Response( JSON.stringify({ ok: true, result: { id: botId, username: "maya_paperclip_bot", first_name: "Maya", }, }), { status: 200, headers: { "content-type": "application/json" } }, ); } if (url.endsWith("/getWebhookInfo")) { expect(init?.signal).toBeInstanceOf(AbortSignal); return new Response( JSON.stringify({ ok: true, result: { url: existingWebhookUrl } }), { status: 200, headers: { "content-type": "application/json" }, }, ); } if (url.endsWith("/setWebhook")) { expect(init?.method).toBe("POST"); observedWebhook = JSON.parse(String(init?.body)) as Record< string, unknown >; existingWebhookUrl = String(observedWebhook.url ?? ""); return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } if (url.endsWith("/setMyCommands")) { expect(init?.method).toBe("POST"); observedCommands = JSON.parse(String(init?.body)) as Record< string, unknown >; return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } if (url.endsWith("/deleteWebhook")) { expect(init?.method).toBe("POST"); observedWebhookDelete = JSON.parse(String(init?.body)) as Record< string, unknown >; return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } if (url.endsWith("/deleteMyCommands")) { expect(init?.method).toBe("POST"); observedCommandsDelete = JSON.parse(String(init?.body)) as Record< string, unknown >; return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } throw new Error(`Unexpected provider request: ${url}`); }, ) as unknown as typeof globalThis.fetch; const { runtime, service } = createService( new FakeChatSdkRuntime(), providerFetch, { webhookPublicBaseUrl }, ); const endpoint = await service.create( fixture.companyId, { provider: "telegram", assignedAgentId: fixture.assignedAgentId, }, "owner-user", ); expect(endpoint.setup).toMatchObject({ step: "provider_setup", providerUrl: "https://t.me/BotFather", }); const configured = await service.configure( endpoint.id, { action: "configure", credentials: { botToken }, }, "owner-user", ); expect(observedUrls).toEqual([ `https://api.telegram.org/bot${encodeURIComponent(botToken)}/getMe`, `https://api.telegram.org/bot${encodeURIComponent(botToken)}/getWebhookInfo`, `https://api.telegram.org/bot${encodeURIComponent(botToken)}/setWebhook`, `https://api.telegram.org/bot${encodeURIComponent(botToken)}/setMyCommands`, ]); expect(configured).toMatchObject({ status: "verifying", providerAccountId: String(botId), botExternalId: String(botId), botUsername: "maya_paperclip_bot", capabilities: { messageEdits: true, messageDeletes: false }, setup: { step: "test", providerUrl: "https://t.me/maya_paperclip_bot", }, }); const providerConfig = runtime.configurations.get( endpoint.id, )?.providerConfig; expect(providerConfig).toMatchObject({ provider: "telegram", credentials: { botToken, secretToken: expect.any(String) }, }); if (providerConfig?.provider !== "telegram") throw new Error("Telegram runtime configuration was not created"); expect(observedWebhook).toEqual({ url: `${webhookPublicBaseUrl ?? "https://paperclip.example"}/api/chat-webhooks/${endpoint.publicId}/telegram`, secret_token: providerConfig.credentials.secretToken, allowed_updates: [ "message", "edited_message", "callback_query", "stopped_message_generation", "message_reaction", "my_chat_member", ], drop_pending_updates: true, }); expect(observedCommands).toEqual({ commands: [ { command: "task", description: "Start or continue a Paperclip task", }, { command: "status", description: "Show the active Paperclip task" }, { command: "new", description: "Start a new task after the current one", }, { command: "close", description: "Close the active chat conversation", }, ], }); const [connection] = await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(connection.refs.map((ref) => ref.configPath).sort()).toEqual([ "credentials.botToken", "credentials.webhookSecret", ]); await db .update(chatEndpoints) .set({ status: "active", setup: { step: "complete" } }) .where(eq(chatEndpoints.id, endpoint.id)); existingWebhookUrl = "https://expired.example/api/chat-webhooks/old-public-id/telegram"; observedUrls.length = 0; observedWebhook = null; observedCommands = null; const reconnected = await service.configure( endpoint.id, { action: "reconnect" }, "owner-user", ); expect(reconnected).toMatchObject({ status: "verifying", setup: { step: "test" }, }); expect(observedUrls).toEqual([ `https://api.telegram.org/bot${encodeURIComponent(botToken)}/getMe`, `https://api.telegram.org/bot${encodeURIComponent(botToken)}/getWebhookInfo`, `https://api.telegram.org/bot${encodeURIComponent(botToken)}/setWebhook`, `https://api.telegram.org/bot${encodeURIComponent(botToken)}/setMyCommands`, ]); expect(observedWebhook).toMatchObject({ url: `${webhookPublicBaseUrl ?? "https://paperclip.example"}/api/chat-webhooks/${endpoint.publicId}/telegram`, // Repointing an existing bot must retain updates Telegram queued while // the old callback URL was unavailable. drop_pending_updates: false, }); await service.configure(endpoint.id, { action: "remove" }, "owner-user"); expect(observedWebhookDelete).toEqual({ drop_pending_updates: false }); expect(observedCommandsDelete).toEqual({}); expect(observedUrls.at(-2)).toBe( `https://api.telegram.org/bot${encodeURIComponent(botToken)}/deleteWebhook`, ); expect(observedUrls.at(-1)).toBe( `https://api.telegram.org/bot${encodeURIComponent(botToken)}/deleteMyCommands`, ); }, ); it("durably recovers rate-limited Telegram menu registration and removal cleanup", async () => { const fixture = await seedCompany(); const botToken = "123456:telegram-maintenance-test"; const botId = Number.parseInt( randomUUID().replaceAll("-", "").slice(0, 12), 16, ); let commandRateLimits = 0; let deleteWebhookRateLimits = 0; let deleteMyCommandsCalls = 0; const providerFetch = vi.fn(async (input: string | URL | Request) => { const url = String(input); if (url.endsWith("/getMe")) { return new Response( JSON.stringify({ ok: true, result: { id: botId, username: "paperclip_maintenance_bot", first_name: "Paperclip Maintenance", }, }), { status: 200, headers: { "content-type": "application/json" } }, ); } if (url.endsWith("/getWebhookInfo")) { return new Response(JSON.stringify({ ok: true, result: { url: "" } }), { status: 200, headers: { "content-type": "application/json" }, }); } if (url.endsWith("/setWebhook")) { return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } if (url.endsWith("/setMyCommands")) { if (commandRateLimits > 0) { commandRateLimits -= 1; return new Response( JSON.stringify({ ok: false, description: "Too Many Requests", parameters: { retry_after: 60 * 60 }, }), { status: 429, headers: { "content-type": "application/json" }, }, ); } return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } if (url.endsWith("/deleteWebhook")) { if (deleteWebhookRateLimits > 0) { deleteWebhookRateLimits -= 1; return new Response( JSON.stringify({ ok: false, description: "Too Many Requests", parameters: { retry_after: 2 * 60 * 60 }, }), { status: 429, headers: { "content-type": "application/json" }, }, ); } return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } if (url.endsWith("/deleteMyCommands")) { deleteMyCommandsCalls += 1; return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } throw new Error(`Unexpected provider request: ${url}`); }) as typeof globalThis.fetch; const { service } = createService(new FakeChatSdkRuntime(), providerFetch); const endpoint = await service.create( fixture.companyId, { provider: "telegram", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); const configured = await service.configure( endpoint.id, { action: "configure", credentials: { botToken } }, "owner-user", ); await db .update(chatEndpoints) .set({ status: "active", setup: { ...configured.setup, step: "complete" }, }) .where(eq(chatEndpoints.id, endpoint.id)); commandRateLimits = 1; const reconnectStartedAt = Date.now(); await expect( service.configure(endpoint.id, { action: "reconnect" }, "owner-user"), ).resolves.toMatchObject({ status: "verifying" }); const reconnectAction = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "telegram_maintenance"), sql`${chatActions.payload}->>'operation' = 'register_commands'`, ), ) .orderBy(desc(chatActions.createdAt)) .then((rows) => rows[0]); expect(reconnectAction).toMatchObject({ status: "failed", result: { retryable: true, attempts: 1 }, }); expect( Date.parse(String(reconnectAction.result?.retryAt)) - reconnectStartedAt, ).toBeGreaterThanOrEqual(60 * 60 * 1000 - 1_000); await db .update(chatActions) .set({ result: { ...reconnectAction.result, retryAt: new Date(0).toISOString(), }, }) .where(eq(chatActions.id, reconnectAction.id)); await service.processPendingDeliveries(); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, reconnectAction.id)), ).resolves.toEqual([{ status: "processed" }]); deleteWebhookRateLimits = 1; const removeStartedAt = Date.now(); await expect( service.configure(endpoint.id, { action: "remove" }, "owner-user"), ).resolves.toMatchObject({ status: "archived" }); const removeAction = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "telegram_maintenance"), sql`${chatActions.payload}->>'operation' = 'remove_endpoint'`, ), ) .then((rows) => rows[0]); expect(removeAction).toMatchObject({ status: "failed", result: { retryable: true, attempts: 1 }, }); expect( Date.parse(String(removeAction.result?.retryAt)) - removeStartedAt, ).toBeGreaterThanOrEqual(2 * 60 * 60 * 1000 - 1_000); await expect( db .select({ action: activityLog.action, details: activityLog.details }) .from(activityLog) .where(eq(activityLog.entityId, endpoint.connectionId)), ).resolves.toEqual( expect.arrayContaining([ expect.objectContaining({ action: "chat_endpoint.telegram_register_commands_deferred", details: expect.objectContaining({ endpointId: endpoint.id, attempt: 1, disposition: "retry", }), }), expect.objectContaining({ action: "chat_endpoint.telegram_remove_endpoint_deferred", details: expect.objectContaining({ endpointId: endpoint.id, attempt: 1, disposition: "retry", }), }), ]), ); expect(deleteMyCommandsCalls).toBe(0); const [pendingCleanupConnection] = await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(pendingCleanupConnection.refs.length).toBeGreaterThan(0); await db .update(chatActions) .set({ result: { ...removeAction.result, retryAt: new Date(0).toISOString() }, }) .where(eq(chatActions.id, removeAction.id)); await service.processPendingDeliveries(); expect(deleteMyCommandsCalls).toBe(1); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, removeAction.id)), ).resolves.toEqual([{ status: "processed" }]); await expect( db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)), ).resolves.toEqual([{ refs: [] }]); }); it("serializes recovered Telegram maintenance behind credential rotation", async () => { const fixture = await seedCompany(); const oldBotToken = "123456:telegram-old-maintenance-token"; const newBotToken = "123456:telegram-new-maintenance-token"; const botId = Number.parseInt( randomUUID().replaceAll("-", "").slice(0, 12), 16, ); let blockNewCredentialCheck = false; let markCredentialCheckReady!: () => void; let releaseCredentialCheck!: () => void; const credentialCheckReady = new Promise((resolve) => { markCredentialCheckReady = resolve; }); const credentialCheckRelease = new Promise((resolve) => { releaseCredentialCheck = resolve; }); const commandTokens: string[] = []; const providerFetch = vi.fn(async (input: string | URL | Request) => { const url = String(input); if (url.endsWith("/getMe")) { if ( blockNewCredentialCheck && url.includes(encodeURIComponent(newBotToken)) ) { markCredentialCheckReady(); await credentialCheckRelease; } return new Response( JSON.stringify({ ok: true, result: { id: botId, username: "paperclip_maintenance_race_bot", first_name: "Paperclip Maintenance Race", }, }), { status: 200, headers: { "content-type": "application/json" } }, ); } if (url.endsWith("/getWebhookInfo")) { return new Response(JSON.stringify({ ok: true, result: { url: "" } }), { status: 200, headers: { "content-type": "application/json" }, }); } if (url.endsWith("/setWebhook")) { return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } if (url.endsWith("/setMyCommands")) { commandTokens.push( url.includes(encodeURIComponent(newBotToken)) ? newBotToken : oldBotToken, ); return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } throw new Error(`Unexpected provider request: ${url}`); }) as typeof globalThis.fetch; const { service } = createService(new FakeChatSdkRuntime(), providerFetch); const endpoint = await service.create( fixture.companyId, { provider: "telegram", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await service.configure( endpoint.id, { action: "configure", credentials: { botToken: oldBotToken } }, "owner-user", ); const staleAction = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "telegram_maintenance"), ), ) .then((rows) => rows[0]!); await db .update(chatActions) .set({ status: "failed", result: { attempts: 1, retryable: true, retryAt: new Date(0).toISOString(), }, }) .where(eq(chatActions.id, staleAction.id)); commandTokens.length = 0; blockNewCredentialCheck = true; const reconnect = service.configure( endpoint.id, { action: "reconnect", credentials: { botToken: newBotToken } }, "owner-user", ); await credentialCheckReady; const recovery = service.processPendingDeliveries(); releaseCredentialCheck(); await reconnect; await recovery; expect(commandTokens).toEqual([newBotToken]); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, staleAction.id)), ).resolves.toEqual([ { status: "cancelled", result: expect.objectContaining({ code: "telegram_maintenance_credentials_superseded", }), }, ]); }); it("reconciles Slack membership during configure, resume, and reconnect", async () => { const fixture = await seedCompany(); let channels = [ { id: "C-ONE", name: "one", is_member: true, is_archived: false }, ]; const providerFetch = vi.fn(async (input: string | URL | Request) => { const url = String(input); if (url === "https://slack.com/api/auth.test") { return new Response( JSON.stringify({ ok: true, team_id: "T-RECONCILE", team: "Reconcile Test", user_id: "U-RECONCILE", user: "maya-reconcile", }), { status: 200, headers: { "content-type": "application/json", "x-oauth-scopes": TEST_SLACK_BOT_SCOPES, }, }, ); } if (url.startsWith("https://slack.com/api/conversations.list")) { return new Response( JSON.stringify({ ok: true, channels, response_metadata: { next_cursor: "" }, }), { status: 200, headers: { "content-type": "application/json" } }, ); } throw new Error(`Unexpected provider request: ${url}`); }) as typeof globalThis.fetch; const { service } = createService(new FakeChatSdkRuntime(), providerFetch); const endpoint = await service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await service.configure( endpoint.id, { action: "configure", credentials: { botToken: "xoxb-reconcile", signingSecret: "reconcile-signing-secret", }, }, "owner-user", ); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ providerResourceId: "C-ONE", availability: "available", enabled: false, }), ]); await db .update(chatEndpoints) .set({ status: "active", setup: { step: "complete" } }) .where(eq(chatEndpoints.id, endpoint.id)); await service.configure(endpoint.id, { action: "pause" }, "owner-user"); channels = [ { id: "C-TWO", name: "two", is_member: true, is_archived: false }, ]; await service.configure(endpoint.id, { action: "resume" }, "owner-user"); expect( (await service.listResources(endpoint.id)).map((resource) => ({ id: resource.providerResourceId, availability: resource.availability, })), ).toEqual([ { id: "C-ONE", availability: "unavailable" }, { id: "C-TWO", availability: "available" }, ]); await db .update(chatEndpoints) .set({ status: "attention" }) .where(eq(chatEndpoints.id, endpoint.id)); channels = [ { id: "C-THREE", name: "three", is_member: true, is_archived: false }, ]; await service.configure( endpoint.id, { action: "reconnect", credentials: { botToken: "xoxb-reconcile", signingSecret: "reconcile-signing-secret", }, }, "owner-user", ); expect( (await service.listResources(endpoint.id)).map((resource) => ({ id: resource.providerResourceId, availability: resource.availability, })), ).toEqual([ { id: "C-ONE", availability: "unavailable" }, { id: "C-THREE", availability: "available" }, { id: "C-TWO", availability: "unavailable" }, ]); }); it("rejects reconnect credentials for a different Slack bot without rotating secrets", async () => { const fixture = await seedCompany(); let botId = "U-ORIGINAL-BOT"; const providerFetch = vi.fn(async (input: string | URL | Request) => { const url = String(input); if (url === "https://slack.com/api/auth.test") { return new Response( JSON.stringify({ ok: true, team_id: "T-IMMUTABLE", team: "Immutable Test", user_id: botId, user: botId === "U-ORIGINAL-BOT" ? "maya-original" : "maya-other", }), { status: 200, headers: { "content-type": "application/json", "x-oauth-scopes": TEST_SLACK_BOT_SCOPES, }, }, ); } if (url.startsWith("https://slack.com/api/conversations.list")) { return new Response( JSON.stringify({ ok: true, channels: [], response_metadata: { next_cursor: "" }, }), { status: 200, headers: { "content-type": "application/json" } }, ); } throw new Error(`Unexpected provider request: ${url}`); }) as typeof globalThis.fetch; const { service } = createService(new FakeChatSdkRuntime(), providerFetch); const endpoint = await service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await service.configure( endpoint.id, { action: "configure", credentials: { botToken: "xoxb-original", signingSecret: "original-secret", }, }, "owner-user", ); const [before] = await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); botId = "U-DIFFERENT-BOT"; await expect( service.configure( endpoint.id, { action: "reconnect", credentials: { botToken: "xoxb-different", signingSecret: "different-secret", }, }, "owner-user", ), ).rejects.toMatchObject({ status: 409, details: { code: "chat_bot_identity_changed" }, }); const [after] = await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(after.refs).toEqual(before.refs); await expect(service.get(endpoint.id)).resolves.toMatchObject({ botExternalId: "U-ORIGINAL-BOT", }); }); it("persists Slack membership, uninstall, and same-bot reinstall lifecycle before acknowledging", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const configuredEndpoint = await service.get(endpoint.id); const send = (payload: unknown) => service.handleWebhook( endpoint.publicId, "slack", new Request("https://paperclip.example/slack", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(payload), }), ); await send({ event_id: "Ev-member-joined", event: { type: "member_joined_channel", event_ts: "100.000000", user: configuredEndpoint.botExternalId, channel: "C-LIFECYCLE", channel_type: "C", }, }); const [resource] = await db .select() .from(chatEndpointResources) .where(eq(chatEndpointResources.endpointId, endpoint.id)); expect(resource).toMatchObject({ providerResourceId: "C-LIFECYCLE", availability: "available", enabled: false, }); await vi.waitFor(async () => { await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ label: "#c-lifecycle" }), ]); }); await service.replaceResources(endpoint.id, [ { id: resource.id, enabled: true }, ]); const thread = makeThread({ channelId: "C-LIFECYCLE", id: "slack:C-LIFECYCLE:123.45", name: "slack:C-LIFECYCLE", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: "123.45", text: "@maya preserve this task", mentioned: true, }), trigger: "mention", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); expect(conversation).toBeDefined(); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ label: "#c-lifecycle" }), ]); const leftEvent = { event_id: "Ev-member-left", event: { type: "channel_left", event_ts: "200.000000", channel: "C-LIFECYCLE", }, }; await send(leftEvent); await send(leftEvent); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ providerResourceId: "C-LIFECYCLE", availability: "unavailable", label: "#c-lifecycle", }), ]); expect( await db .select({ state: chatConversations.state }) .from(chatConversations) .where(eq(chatConversations.id, conversation.id)), ).toEqual([{ state: "unavailable" }]); expect( await db.select().from(issues).where(eq(issues.id, conversation.issueId)), ).toHaveLength(1); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: makeThread({ channelId: "C-LIFECYCLE", id: "slack:C-LIFECYCLE:999.01", name: "lifecycle", }).thread, message: makeMessage({ id: "999.01", text: "@maya this delayed root must stay blocked", mentioned: true, }), trigger: "mention", }); expect( await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toHaveLength(1); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ availability: "unavailable", enabled: true }), ]); await send({ event_id: "Ev-stale-member-joined", event: { type: "member_joined_channel", event_ts: "150.000000", user: configuredEndpoint.botExternalId, channel: "C-LIFECYCLE", }, }); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ availability: "unavailable", enabled: true }), ]); expect( ( await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)) ).filter( (delivery) => delivery.providerEventId === "lifecycle:Ev-member-left", ), ).toHaveLength(1); await send({ event_id: "Ev-uninstalled", event: { type: "app_uninstalled" }, }); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "revoked", healthMessage: "Slack app was uninstalled", }); expect(runtime.endpoints.has(endpoint.id)).toBe(false); const [connection] = await db .select() .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(connection).toMatchObject({ status: "disabled", enabled: false, healthStatus: "failed", }); const reconnecting = await service.configure( endpoint.id, { action: "reconnect", credentials: { botToken: "xoxb-test-token", signingSecret: "test-signing-secret", }, }, "owner-user", ); expect(reconnecting).toMatchObject({ status: "verifying", botExternalId: configuredEndpoint.botExternalId, setup: { step: "provider_setup", webhookVerifiedAt: null }, }); expect(runtime.endpoints.has(endpoint.id)).toBe(true); await recordSlackUrlVerification(service, endpoint.publicId); await expect( service.configure(endpoint.id, { action: "verify" }, "owner-user"), ).resolves.toMatchObject({ status: "verifying", setup: { step: "test", webhookVerifiedAt: expect.any(String) }, }); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ providerResourceId: "C-LIFECYCLE", availability: "unavailable", enabled: true, label: "#c-lifecycle", }), ]); }); it("orders Slack private-channel archive, unarchive, and rename lifecycle", async () => { const fixture = await seedCompany(); const { endpoint, service } = await configuredSlackEndpoint(fixture); const webhookUrl = `https://paperclip.example/api/chat-webhooks/${endpoint.publicId}/slack`; const send = async ( eventId: string, type: string, eventTs: string, channel: string | { id: string; name: string }, ) => { const body = JSON.stringify({ event_id: eventId, event: { type, event_ts: eventTs, channel }, }); await service.handleWebhook( endpoint.publicId, "slack", signedSlackWebhookRequest({ url: webhookUrl, contentType: "application/json", body, }), ); }; const resource = async () => (await service.listResources(endpoint.id)).find( (candidate) => candidate.providerResourceId === "G-PRIVATE-LIFECYCLE", ); await send( "Ev-group-archive", "group_archive", "300.000000", "G-PRIVATE-LIFECYCLE", ); await expect(resource()).resolves.toMatchObject({ availability: "unavailable", }); await send( "Ev-group-unarchive", "group_unarchive", "400.000000", "G-PRIVATE-LIFECYCLE", ); await send( "Ev-stale-group-archive", "group_archive", "350.000000", "G-PRIVATE-LIFECYCLE", ); await expect(resource()).resolves.toMatchObject({ availability: "available", }); await send("Ev-group-rename", "group_rename", "500.000000", { id: "G-PRIVATE-LIFECYCLE", name: "private-renamed", }); await expect(resource()).resolves.toMatchObject({ availability: "available", label: "private-renamed", }); }); it("applies Teams installation and Telegram membership resource lifecycle", async () => { const fixture = await seedCompany(); const teams = createService( new FakeChatSdkRuntime(), (async () => new Response(JSON.stringify({ access_token: "teams-access" }), { status: 200, headers: { "content-type": "application/json" }, })) as typeof globalThis.fetch, ); const teamsEndpoint = await teams.service.create( fixture.companyId, { provider: "microsoft-teams", assignedAgentId: fixture.assignedAgentId, }, "owner-user", ); const teamsClientId = "00000000-0000-4000-8000-000000000111"; await teams.service.configure( teamsEndpoint.id, { action: "configure", credentials: { clientId: teamsClientId, tenantId: "00000000-0000-4000-8000-000000000222", clientSecret: "teams-secret", }, }, "owner-user", ); const teamsTenantId = "00000000-0000-4000-8000-000000000222"; const teamsPayload = ( action: "add" | "remove", id = `teams-${action}`, timestamp = action === "add" ? "2026-09-05T14:00:00.000Z" : "2026-09-05T14:01:00.000Z", ) => ({ id, type: "installationUpdate", action, timestamp, conversation: { id: "19:conversation@thread.tacv2", isGroup: true, tenantId: teamsTenantId, }, channelData: { tenant: { id: teamsTenantId }, team: { id: "team-1", name: "Paperclip" }, channel: { id: "channel-1", name: "Engineering" }, }, }); const deliverTeamsLifecycle = (payload: unknown) => teams.service.handleWebhook( teamsEndpoint.publicId, "microsoft-teams", new Request("https://paperclip.example/teams", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(payload), }), ); const foreignTenantPayload = teamsPayload("add"); foreignTenantPayload.id = "teams-foreign-tenant"; foreignTenantPayload.conversation.tenantId = "foreign-tenant"; foreignTenantPayload.channelData.tenant.id = "foreign-tenant"; await expect( deliverTeamsLifecycle(foreignTenantPayload), ).resolves.toMatchObject({ status: 202 }); const missingTenantPayload = teamsPayload("add") as Omit< ReturnType, "conversation" | "channelData" > & { conversation: Omit< ReturnType["conversation"], "tenantId" >; channelData: Omit< ReturnType["channelData"], "tenant" >; }; delete (missingTenantPayload.conversation as { tenantId?: string }) .tenantId; delete (missingTenantPayload.channelData as { tenant?: { id: string } }) .tenant; missingTenantPayload.id = "teams-missing-tenant"; await expect( deliverTeamsLifecycle(missingTenantPayload), ).resolves.toMatchObject({ status: 202 }); await expect( teams.service.listResources(teamsEndpoint.id), ).resolves.toEqual([]); await expect( db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, teamsEndpoint.id)), ).resolves.toHaveLength(0); await deliverTeamsLifecycle(teamsPayload("add")); const [teamsResource] = await teams.service.listResources(teamsEndpoint.id); expect(teamsResource).toMatchObject({ providerResourceId: "19:conversation@thread.tacv2", availability: "available", enabled: false, }); await teams.service.replaceResources(teamsEndpoint.id, [ { id: teamsResource!.id, enabled: true }, ]); const teamsCallbacks = teams.runtime.configurations.get( teamsEndpoint.id, )?.callbacks; if (!teamsCallbacks) throw new Error("Expected Teams callbacks"); const teamsServiceUrl = "https://smba.trafficmanager.net/amer/"; const teamsConversationId = "19:conversation@thread.tacv2"; const teamsChannelId = `teams:${Buffer.from(teamsConversationId).toString("base64url")}:${Buffer.from(teamsServiceUrl).toString("base64url")}`; const teamsThreadId = `teams:${Buffer.from(`${teamsConversationId};messageid=1729`).toString("base64url")}:${Buffer.from(teamsServiceUrl).toString("base64url")}`; await deliverMessage({ callbacks: teamsCallbacks, endpointId: teamsEndpoint.id, provider: "microsoft-teams", thread: makeThread({ channelId: teamsChannelId, id: teamsThreadId, name: "Engineering", }).thread, message: makeMessage({ id: "teams-root-1729", text: "@Maya investigate the alert", mentioned: true, }), trigger: "mention", }); await expect( db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, teamsEndpoint.id)), ).resolves.toHaveLength(1); await deliverTeamsLifecycle(teamsPayload("remove")); await expect( teams.service.listResources(teamsEndpoint.id), ).resolves.toEqual([ expect.objectContaining({ providerResourceId: "19:conversation@thread.tacv2", label: "Engineering", availability: "removed", }), ]); await expect( db .select({ state: chatConversations.state }) .from(chatConversations) .where(eq(chatConversations.endpointId, teamsEndpoint.id)), ).resolves.toEqual([{ state: "unavailable" }]); await deliverTeamsLifecycle( teamsPayload("add", "teams-stale-add", "2026-09-05T14:00:30.000Z"), ); await expect( teams.service.listResources(teamsEndpoint.id), ).resolves.toEqual([ expect.objectContaining({ providerResourceId: "19:conversation@thread.tacv2", availability: "removed", }), ]); await deliverTeamsLifecycle( teamsPayload("add", "teams-reinstalled", "2026-09-05T14:02:00.000Z"), ); await expect( teams.service.listResources(teamsEndpoint.id), ).resolves.toEqual([ expect.objectContaining({ id: teamsResource!.id, providerResourceId: "19:conversation@thread.tacv2", label: "Engineering", availability: "available", enabled: true, }), ]); await expect( db .select({ id: chatConversations.id, issueId: chatConversations.issueId, state: chatConversations.state, }) .from(chatConversations) .where(eq(chatConversations.endpointId, teamsEndpoint.id)), ).resolves.toEqual([ { id: expect.any(String), issueId: expect.any(String), state: "active", }, ]); await expect( db .select({ id: issues.id }) .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).resolves.toHaveLength(1); const personalPayload = (action: "add" | "remove") => ({ id: `teams-personal-${action}`, type: "installationUpdate", action, conversation: { id: "a:teams-personal-conversation", conversationType: "personal", tenantId: teamsTenantId, }, channelData: { tenant: { id: teamsTenantId } }, }); await deliverTeamsLifecycle(personalPayload("add")); await expect( db .select() .from(chatEndpointResources) .where( and( eq(chatEndpointResources.endpointId, teamsEndpoint.id), eq( chatEndpointResources.providerResourceId, "a:teams-personal-conversation", ), ), ), ).resolves.toEqual([ expect.objectContaining({ providerResourceId: "a:teams-personal-conversation", type: "direct_message", availability: "available", }), ]); await deliverTeamsLifecycle(personalPayload("remove")); await expect( db .select() .from(chatEndpointResources) .where( and( eq(chatEndpointResources.endpointId, teamsEndpoint.id), eq( chatEndpointResources.providerResourceId, "a:teams-personal-conversation", ), ), ), ).resolves.toEqual([ expect.objectContaining({ providerResourceId: "a:teams-personal-conversation", type: "direct_message", availability: "removed", }), ]); await deliverTeamsLifecycle({ id: "teams-group-member-added", type: "conversationUpdate", conversation: { id: "19:teams-group-conversation@unq.gbl.spaces", conversationType: "group", tenantId: teamsTenantId, }, channelData: { tenant: { id: teamsTenantId } }, membersAdded: [{ id: `28:${teamsClientId}` }], }); await expect( teams.service.listResources(teamsEndpoint.id), ).resolves.toEqual( expect.arrayContaining([ expect.objectContaining({ providerResourceId: "19:teams-group-conversation@unq.gbl.spaces", type: "group_chat", availability: "available", }), ]), ); await deliverTeamsLifecycle({ id: "teams-group-member-removed", type: "conversationUpdate", conversation: { id: "19:teams-group-conversation@unq.gbl.spaces", conversationType: "groupChat", tenantId: teamsTenantId, }, channelData: { tenant: { id: teamsTenantId } }, membersRemoved: [{ id: `28:${teamsClientId}` }], }); await expect( teams.service.listResources(teamsEndpoint.id), ).resolves.toEqual( expect.arrayContaining([ expect.objectContaining({ providerResourceId: "19:teams-group-conversation@unq.gbl.spaces", type: "group_chat", availability: "unavailable", }), ]), ); const telegram = createService( new FakeChatSdkRuntime(), fakeTelegramFetch(445566) as typeof globalThis.fetch, ); const telegramEndpoint = await telegram.service.create( fixture.companyId, { provider: "telegram", assignedAgentId: fixture.replacementAgentId }, "owner-user", ); await telegram.service.configure( telegramEndpoint.id, { action: "configure", credentials: { botToken: "445566:telegram-lifecycle" }, }, "owner-user", ); const telegramMembership = (updateId: number, status: string) => telegram.service.handleWebhook( telegramEndpoint.publicId, "telegram", new Request("https://paperclip.example/telegram", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ update_id: updateId, my_chat_member: { chat: { id: -100123, type: "supergroup", title: "Engineering" }, new_chat_member: { status }, }, }), }), ); await telegramMembership(1, "member"); const [telegramResource] = await telegram.service.listResources( telegramEndpoint.id, ); await telegram.service.replaceResources(telegramEndpoint.id, [ { id: telegramResource!.id, enabled: true }, ]); const telegramCallbacks = telegram.runtime.configurations.get( telegramEndpoint.id, )?.callbacks; if (!telegramCallbacks) throw new Error("Expected Telegram callbacks"); await deliverMessage({ callbacks: telegramCallbacks, endpointId: telegramEndpoint.id, provider: "telegram", thread: makeThread({ channelId: "-100123", id: "telegram:-100123:77", // The native adapter may expose this fallback when the message and // my_chat_member callbacks are interleaved. It must not replace the // human title already learned from the membership payload. name: "telegram:-100123", }).thread, message: makeMessage({ id: "telegram-root-77", text: "@paperclip investigate the alert", mentioned: true, }), trigger: "mention", }); await expect( db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, telegramEndpoint.id)), ).resolves.toHaveLength(1); await expect( telegram.service.listResources(telegramEndpoint.id), ).resolves.toEqual([ expect.objectContaining({ providerResourceId: "-100123", label: "Engineering", availability: "available", }), ]); await telegramMembership(2, "left"); await expect( telegram.service.listResources(telegramEndpoint.id), ).resolves.toEqual([ expect.objectContaining({ providerResourceId: "-100123", label: "Engineering", availability: "unavailable", }), ]); await telegramMembership(0, "administrator"); await expect( telegram.service.listResources(telegramEndpoint.id), ).resolves.toEqual([ expect.objectContaining({ providerResourceId: "-100123", availability: "unavailable", }), ]); }); it("does not acknowledge lifecycle callbacks whose durable write fails", async () => { const fixture = await seedCompany(); const { endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const configuredEndpoint = await service.get(endpoint.id); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected provider runtime"); providerRuntime.handleWebhook = vi.fn(async () => { await db.delete(chatEndpoints).where(eq(chatEndpoints.id, endpoint.id)); return new Response("accepted", { status: 202 }); }); await expect( service.handleWebhook( endpoint.publicId, "slack", new Request("https://paperclip.example/slack", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ event_id: "Ev-must-persist", event: { type: "member_joined_channel", user: configuredEndpoint.botExternalId, channel: "C-MUST-PERSIST", }, }), }), ), ).rejects.toBeDefined(); }); it("retries a durable lifecycle row without treating it as a message delivery", async () => { const fixture = await seedCompany(); const { endpoint, service } = await configuredSlackEndpoint(fixture); const configuredEndpoint = await service.get(endpoint.id); const transaction = vi.spyOn(db, "transaction"); transaction.mockRejectedValueOnce( new Error("injected lifecycle persistence failure"), ); await expect( service.handleWebhook( endpoint.publicId, "slack", new Request("https://paperclip.example/slack", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ event_id: "Ev-lifecycle-retry", event: { type: "member_joined_channel", user: configuredEndpoint.botExternalId, channel: "C-RETRY-LIFECYCLE", }, }), }), ), ).rejects.toThrow("injected lifecycle persistence failure"); transaction.mockRestore(); const [retry] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.providerEventId, "lifecycle:Ev-lifecycle-retry"), ), ); expect(retry).toMatchObject({ state: "retry", attempts: 1 }); await service.processPendingDeliveries(25, retry!.id); await expect( db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where(eq(chatDeliveries.id, retry!.id)), ).resolves.toEqual([{ state: "processed" }]); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ providerResourceId: "C-RETRY-LIFECYCLE", availability: "available", }), ]); }); it("activates Slack only after provider verification and a real test message", async () => { const fixture = await seedCompany(); const { runtime, service } = createService(); const endpoint = await service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId, }, "owner-user", ); const configured = await service.configure( endpoint.id, { action: "configure", credentials: { botToken: "xoxb-test-token", signingSecret: "test-signing-secret", }, }, "owner-user", ); expect(configured).toMatchObject({ status: "verifying", setup: { step: "provider_setup" }, }); const [storedConfigured] = await db .select({ setup: chatEndpoints.setup }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)); expect(storedConfigured.setup).toMatchObject({ step: "provider_setup", testStartedAt: null, webhookVerifiedAt: null, runtimeGeneration: expect.any(Number), }); await expect( service.configure(endpoint.id, { action: "verify" }, "owner-user"), ).rejects.toMatchObject({ status: 409, details: { code: "chat_webhook_not_verified" }, }); await recordSlackUrlVerification(service, endpoint.publicId); const providerVerified = await service.configure( endpoint.id, { action: "verify" }, "owner-user", ); expect(providerVerified).toMatchObject({ status: "verifying", setup: { step: "test" }, }); const [storedProviderVerified] = await db .select({ setup: chatEndpoints.setup }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)); expect(storedProviderVerified.setup).toMatchObject({ step: "test", testStartedAt: expect.any(String), }); await expect(service.test(endpoint.id)).rejects.toMatchObject({ status: 409, details: { code: "chat_test_message_missing" }, }); const callbacks = runtime.configurations.get(endpoint.id)?.callbacks; if (!callbacks) throw new Error("Fake runtime did not receive endpoint callbacks"); const testThread = makeThread({ channelId: "C-SETUP", id: "slack:C-SETUP:9000.1", name: "setup", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: testThread.thread, message: makeMessage({ id: "9000.1", text: "@maya verify this connection", mentioned: true, }), trigger: "mention", }); await expect(service.test(endpoint.id)).rejects.toMatchObject({ status: 409, details: { code: "chat_test_follow_up_missing" }, }); await qualifySetupRoundTrip(service, endpoint.id); const activated = await service.test(endpoint.id); expect(activated).toMatchObject({ status: "active", healthMessage: "Connected", activatedAt: expect.any(String), setup: { step: "complete" }, }); const [storedActivated] = await db .select({ setup: chatEndpoints.setup }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)); expect(storedActivated.setup).toMatchObject({ step: "complete", testStartedAt: null, runtimeGeneration: expect.any(Number), }); }); it("serializes setup activation ahead of a concurrent reconnect without overwriting the newer runtime", async () => { const fixture = await seedCompany(); let enterActivation!: () => void; const activationEntered = new Promise((resolve) => { enterActivation = resolve; }); let releaseActivation!: () => void; const activationReleased = new Promise((resolve) => { releaseActivation = resolve; }); const runtime = new FakeChatSdkRuntime(); const { service } = createService( runtime, fakeSlackFetch() as typeof globalThis.fetch, { setupTestActivationBarrier: async () => { enterActivation(); await activationReleased; }, }, ); const endpoint = await service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await service.configure( endpoint.id, { action: "configure", credentials: { botToken: "xoxb-test-token", signingSecret: "test-signing-secret", }, }, "owner-user", ); await recordSlackUrlVerification(service, endpoint.publicId); await service.configure(endpoint.id, { action: "verify" }, "owner-user"); const callbacks = runtime.configurations.get(endpoint.id)?.callbacks; if (!callbacks) throw new Error("Expected Slack runtime callbacks"); const testThread = makeThread({ channelId: "C-SETUP-RECONNECT-RACE", id: "slack:C-SETUP-RECONNECT-RACE:9001.1", name: "setup-reconnect-race", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: testThread.thread, message: makeMessage({ id: "9001.1", text: "@maya verify setup before reconnect", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); const activation = service.test(endpoint.id); await activationEntered; let reconnectSettled = false; const reconnect = service .configure(endpoint.id, { action: "reconnect" }, "owner-user") .finally(() => { reconnectSettled = true; }); await new Promise((resolve) => setTimeout(resolve, 25)); expect(reconnectSettled).toBe(false); releaseActivation(); await expect(activation).resolves.toMatchObject({ status: "active", setup: { step: "complete" }, }); await expect(reconnect).resolves.toMatchObject({ status: "verifying", setup: { step: "provider_setup" }, }); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "verifying", setup: { step: "provider_setup" }, }); await service.shutdown(); }); it("separates verified webhook ingress from board identity links for every webhook provider", async () => { const fixture = await seedCompany(); for (const publicBaseUrl of [ null, "http://127.0.0.1:3103", "https://board.example", ]) { const { service } = createService( new FakeChatSdkRuntime(), fakeSlackFetch(), { publicBaseUrl, webhookPublicBaseUrl: "https://ingress.example", }, ); for (const provider of [ "slack", "github", "microsoft-teams", "telegram", ] as const) { const endpoint = await service.create( fixture.companyId, { provider, assignedAgentId: fixture.assignedAgentId, }, "owner-user", ); const callback = `https://ingress.example/api/chat-webhooks/${endpoint.publicId}/${provider}`; expect(endpoint.setup).toMatchObject( provider === "microsoft-teams" ? { messagingEndpoint: callback } : { webhookUrl: callback }, ); const [principal] = await db .insert(chatExternalPrincipals) .values({ companyId: fixture.companyId, provider, providerAccountId: "", externalId: randomUUID(), kind: "user", isBot: false, }) .returning(); const intent = await service.createLinkIntent( endpoint.id, principal.id, 1800, ); expect(intent.confirmationUrl).toMatch( new RegExp( `^${publicBaseUrl ? publicBaseUrl.replaceAll(".", "\\.") : ""}/chat-identity/confirm\\?token=`, ), ); expect(intent.confirmationUrl).not.toContain("ingress.example"); } await service.shutdown(); } }); it("tracks Slack callback surfaces independently and reports public URL drift", async () => { const fixture = await seedCompany(); const { endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const webhookUrl = `https://paperclip.example/api/chat-webhooks/${endpoint.publicId}/slack`; await expect(service.get(endpoint.id)).resolves.toMatchObject({ setup: { callbacksNeedUpdate: false, callbackSurfaces: { events: { status: "current", observedAt: expect.any(String) }, interactivity: { status: "unverified" }, slashCommands: { status: "unverified" }, }, }, }); const interactiveBody = new URLSearchParams({ payload: JSON.stringify({ type: "block_actions", team: { id: "T-PAPERCLIP" }, }), }).toString(); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack runtime"); providerRuntime.webhookResponse = new Response("rejected", { status: 401 }); await service.handleWebhook( endpoint.publicId, "slack", signedSlackWebhookRequest({ url: webhookUrl, contentType: "application/x-www-form-urlencoded", body: interactiveBody, }), ); await expect(service.get(endpoint.id)).resolves.toMatchObject({ setup: { callbackSurfaces: { interactivity: { status: "unverified" } }, }, }); providerRuntime.webhookResponse = new Response("accepted", { status: 202 }); await service.handleWebhook( endpoint.publicId, "slack", signedSlackWebhookRequest({ url: webhookUrl, contentType: "application/x-www-form-urlencoded", body: interactiveBody, }), ); const slashBody = new URLSearchParams({ command: "/maya-paperclip", team_id: "T-PAPERCLIP", }).toString(); await service.handleWebhook( endpoint.publicId, "slack", signedSlackWebhookRequest({ url: webhookUrl, contentType: "application/x-www-form-urlencoded", body: slashBody, }), ); await expect(service.get(endpoint.id)).resolves.toMatchObject({ setup: { callbacksNeedUpdate: false, callbackSurfaces: { events: { status: "current" }, interactivity: { status: "current", observedAt: expect.any(String) }, slashCommands: { status: "current", observedAt: expect.any(String), }, }, }, }); const boardOnlyMove = createService( new FakeChatSdkRuntime(), fakeSlackFetch(), { publicBaseUrl: "https://board-moved.example", webhookPublicBaseUrl: "https://paperclip.example", }, ); await expect(boardOnlyMove.service.get(endpoint.id)).resolves.toMatchObject( { setup: { webhookUrl, callbacksNeedUpdate: false, callbackSurfaces: { events: { status: "current" }, interactivity: { status: "current" }, slashCommands: { status: "current" }, }, }, }, ); await boardOnlyMove.service.shutdown(); const rotated = createService(new FakeChatSdkRuntime(), fakeSlackFetch(), { publicBaseUrl: "https://rotated.example", }); await expect(rotated.service.get(endpoint.id)).resolves.toMatchObject({ setup: { webhookUrl: `https://rotated.example/api/chat-webhooks/${endpoint.publicId}/slack`, callbacksNeedUpdate: true, callbackSurfaces: { events: { status: "stale" }, interactivity: { status: "stale" }, slashCommands: { status: "stale" }, }, }, }); }); it("does not let an old Slack runtime verify a reconnected configuration", async () => { const fixture = await seedCompany(); const { endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const oldRuntime = runtime.endpoints.get(endpoint.id); if (!oldRuntime) throw new Error("Expected configured Slack runtime"); let releaseWebhook!: () => void; let markWebhookEntered!: () => void; const webhookGate = new Promise((resolve) => { releaseWebhook = resolve; }); const webhookEntered = new Promise((resolve) => { markWebhookEntered = resolve; }); oldRuntime.webhookHook = async () => { markWebhookEntered(); await webhookGate; }; const oldVerification = service.handleWebhook( endpoint.publicId, "slack", new Request("https://paperclip.example/slack", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ type: "url_verification", challenge: "old-runtime-challenge", }), }), ); await webhookEntered; await service.configure(endpoint.id, { action: "reconnect" }, "owner-user"); releaseWebhook(); await expect(oldVerification).resolves.toMatchObject({ status: 202 }); const [stored] = await db .select({ setup: chatEndpoints.setup }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)); expect(stored!.setup).toMatchObject({ step: "provider_setup", webhookVerifiedAt: null, runtimeGeneration: expect.any(Number), }); }); it("requires a successful final response for setup qualification", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service } = await configuredSlackEndpoint(fixture); const testThread = makeThread({ channelId: "C-SETUP-FAILED", id: "slack:C-SETUP-FAILED:9001.1", name: "setup-failed", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: testThread.thread, message: makeMessage({ id: "9001.1", text: "@maya verify a failed setup turn", mentioned: true, }), trigger: "mention", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: testThread.thread, message: makeMessage({ id: "9001.2", text: "Setup follow-up", }), trigger: "subscribed_message", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); if (!conversation) throw new Error("Expected setup conversation"); for (const progressState of ["queued", "working"] as const) { await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `setup-transport:${progressState}:${endpoint.id}`, payload: { text: `Maya is ${progressState}.`, progressState, }, state: "pending", }); await service.processPendingPublications(); await expect(service.test(endpoint.id)).rejects.toMatchObject({ status: 409, details: { code: "chat_test_round_trip_incomplete" }, }); } await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `setup-transport:failed:${endpoint.id}`, payload: { text: "Maya stopped before completing this turn.", progressState: "failed", }, state: "pending", }); await service.processPendingPublications(); await expect(service.test(endpoint.id)).rejects.toMatchObject({ status: 409, details: { code: "chat_test_round_trip_incomplete" }, }); const contextSnapshot = await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "slack", providerMessageId: "9001.2", }); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: endpoint.assignedAgentId, status: "succeeded", contextSnapshot, }); await addSelectedChatFinal({ agentId: endpoint.assignedAgentId, body: "Setup completed successfully.", companyId: fixture.companyId, issueId: conversation.issueId, runId, }); await service.processPendingPublications(); await expect(service.test(endpoint.id)).resolves.toMatchObject({ status: "active", setup: { step: "complete" }, }); }); it("requires the successful setup final to consume the qualifying follow-up", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service } = await configuredSlackEndpoint(fixture); const testThread = makeThread({ channelId: "C-SETUP-RUN-PROVENANCE", id: "slack:C-SETUP-RUN-PROVENANCE:9002.1", name: "setup-run-provenance", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: testThread.thread, message: makeMessage({ id: "9002.1", text: "@maya begin a deliberately slow setup answer", mentioned: true, }), trigger: "mention", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); if (!conversation) throw new Error("Expected setup conversation"); const rootContext = await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "slack", providerMessageId: "9002.1", }); const rootRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: rootRunId, companyId: fixture.companyId, agentId: endpoint.assignedAgentId, status: "succeeded", contextSnapshot: rootContext, }); await addSelectedChatFinal({ agentId: endpoint.assignedAgentId, body: "The earlier root turn finished after the follow-up arrived.", companyId: fixture.companyId, issueId: conversation.issueId, runId: rootRunId, }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: testThread.thread, message: makeMessage({ id: "9002.2", text: "This is the required setup follow-up.", }), trigger: "subscribed_message", }); // Publish the older root run only after the follow-up has been accepted. // Timestamp ordering alone must not make that unrelated final qualify. await service.processPendingPublications(); await expect(service.test(endpoint.id)).rejects.toMatchObject({ status: 409, details: { code: "chat_test_round_trip_incomplete" }, }); const followUpContext = await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "slack", providerMessageId: "9002.2", }); const followUpRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: followUpRunId, companyId: fixture.companyId, agentId: endpoint.assignedAgentId, status: "succeeded", contextSnapshot: { issueId: followUpContext.issueId, source: followUpContext.source, // Deferred wakeups may coalesce several accepted messages and retain // only the durable list of causal comment ids. wakeCommentIds: [followUpContext.wakeCommentId], }, }); await addSelectedChatFinal({ agentId: endpoint.assignedAgentId, body: "The qualifying follow-up was handled successfully.", companyId: fixture.companyId, issueId: conversation.issueId, runId: followUpRunId, }); await service.processPendingPublications(); await expect(service.test(endpoint.id)).resolves.toMatchObject({ status: "active", setup: { step: "complete" }, }); await service.shutdown(); }); it("durably retries a signed Slack session stop and cancels its exact linked run once", async () => { const fixture = await seedCompany(); let cancellationAttempt = 0; const cancelRun = vi.fn(async (runId: string) => { cancellationAttempt += 1; if (cancellationAttempt === 1) { throw new Error("synthetic cancellation transport failure"); } await db .update(heartbeatRuns) .set({ status: "cancelled", errorCode: "slack_session_stopped", finishedAt: new Date(), updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, runId)); return { id: runId, status: "cancelled" }; }); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture, { cancelRun }); const externalUserId = "USTOPPER1"; const channelId = "CSTOPSESSION1"; const threadTs = `${Math.floor(Date.now() / 1_000) - 5}.100000`; const thread = makeThread({ channelId, id: `slack:${channelId}:${threadTs}`, name: "slack-session-stop", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: threadTs, text: "@maya keep this run cancellable", mentioned: true, userId: externalUserId, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id, externalUserId); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const providerAccountId = (await service.get(endpoint.id)) .providerAccountId; const principal = await db .select() .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, fixture.companyId), eq(chatExternalPrincipals.provider, "slack"), eq( chatExternalPrincipals.providerAccountId, providerAccountId ?? "unknown", ), eq(chatExternalPrincipals.externalId, externalUserId), ), ) .then((rows) => rows[0]); if (!conversation || !principal) { throw new Error("Expected Slack stop conversation and principal"); } const intent = await service.createLinkIntent( endpoint.id, principal.id, 1_800, ); const token = new URL(intent.confirmationUrl).searchParams.get("token"); if (!token) throw new Error("Expected Slack identity-link token"); await service.confirmIdentityLink(token, "owner-user"); const eventSecond = Math.floor(Date.now() / 1_000); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", createdAt: new Date((eventSecond - 2) * 1_000), startedAt: new Date((eventSecond - 1) * 1_000), contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "slack", providerMessageId: threadTs, }), }); await db .update(issues) .set({ executionRunId: runId, status: "in_progress", updatedAt: new Date(), }) .where(eq(issues.id, conversation.issueId)); await enqueueChatRunMilestones(db); await service.processPendingPublications(1_000); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `run:${runId}:working:${endpoint.id}`, ), ), ).resolves.toEqual([{ state: "published" }]); const body = JSON.stringify({ type: "event_callback", team_id: (await service.get(endpoint.id)).providerAccountId, event_id: "EvSlackSessionStopRetry1", event: { type: "agent_session_stopped", channel: channelId, thread_ts: threadTs, event_ts: `${eventSecond}.123456`, streaming_message_ts: [`${eventSecond - 1}.500000`], user: externalUserId, }, }); const webhookUrl = `https://paperclip.example/api/chat-webhooks/${endpoint.publicId}/slack`; const first = await service.handleWebhook( endpoint.publicId, "slack", signedSlackWebhookRequest({ body, contentType: "application/json", url: webhookUrl, }), ); expect(first.status).toBe(202); expect(cancelRun).toHaveBeenCalledTimes(1); const stopAction = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slack_session_stop"), ), ) .then((rows) => rows[0]); expect(stopAction).toMatchObject({ conversationId: conversation.id, principalId: principal.id, status: "failed", payload: { issueId: conversation.issueId, sessionGeneration: conversation.sessionGeneration, target: { id: runId, kind: "run" }, threadId: thread.thread.id, userId: externalUserId, }, result: { attempts: 1, code: "slack_session_stop_cancellation_failed", retryable: true, }, }); await expect( service.processPendingSlackSessionStops(25, stopAction!.id), ).resolves.toBe(0); expect(cancelRun).toHaveBeenCalledTimes(1); await db .update(chatActions) .set({ result: { ...(stopAction?.result ?? {}), retryAt: new Date(Date.now() - 1_000).toISOString(), }, updatedAt: new Date(), }) .where(eq(chatActions.id, stopAction!.id)); await expect( service.processPendingSlackSessionStops(25, stopAction!.id), ).resolves.toBe(1); expect(cancelRun).toHaveBeenCalledTimes(2); expect(cancelRun).toHaveBeenNthCalledWith( 2, runId, "Stopped from the bound Slack agent session", expect.objectContaining({ errorCode: "slack_session_stopped", eventPayload: expect.objectContaining({ conversationId: conversation.id, endpointId: endpoint.id, provider: "slack", }), }), ); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, stopAction!.id)), ).resolves.toEqual([ { status: "processed", result: { attempts: 2, code: "slack_session_stop_cancelled", runId, }, }, ]); await expect( db .select({ action: activityLog.action, actorId: activityLog.actorId }) .from(activityLog) .where( and( eq(activityLog.entityType, "chat_action"), eq(activityLog.entityId, stopAction!.id), ), ), ).resolves.toEqual([ { action: "chat.slack_session_stopped", actorId: "owner-user" }, ]); await service.processPendingPublications(1_000); expect( runtime.endpoints .get(endpoint.id) ?.edits.some((edit) => edit.text === "Maya stopped at your request.") ?? false, ).toBe(true); await service.handleWebhook( endpoint.publicId, "slack", signedSlackWebhookRequest({ body, contentType: "application/json", url: webhookUrl, }), ); expect(cancelRun).toHaveBeenCalledTimes(2); await expect( db .select({ id: chatActions.id }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slack_session_stop"), ), ), ).resolves.toHaveLength(1); await service.shutdown(); }); it("denies guest Slack stops, fences delayed events from newer runs, and cancels queued wakes", async () => { const fixture = await seedCompany(); const cancelRun = vi.fn(async () => ({ status: "cancelled" })); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture, { allowUnlinkedPeople: true, cancelRun, }); const externalUserId = "USTOPPER2"; const channelId = "CSTOPSESSION2"; const threadTs = `${Math.floor(Date.now() / 1_000) - 10}.200000`; const thread = makeThread({ channelId, id: `slack:${channelId}:${threadTs}`, name: "slack-session-governance", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: threadTs, text: "@maya create a guest-started task", mentioned: true, userId: externalUserId, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id, externalUserId); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const providerAccountId = (await service.get(endpoint.id)) .providerAccountId; const principal = await db .select() .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, fixture.companyId), eq(chatExternalPrincipals.provider, "slack"), eq( chatExternalPrincipals.providerAccountId, providerAccountId ?? "unknown", ), eq(chatExternalPrincipals.externalId, externalUserId), ), ) .then((rows) => rows[0]); if (!conversation || !principal) { throw new Error("Expected Slack governance conversation and principal"); } const webhookUrl = `https://paperclip.example/api/chat-webhooks/${endpoint.publicId}/slack`; const workspaceId = (await service.get(endpoint.id)).providerAccountId; const sendStop = (input: { eventId: string; eventSecond: number }) => { const body = JSON.stringify({ type: "event_callback", team_id: workspaceId, event_id: input.eventId, event: { type: "agent_session_stopped", channel: channelId, thread_ts: threadTs, event_ts: `${input.eventSecond}.234567`, streaming_message_ts: [], user: externalUserId, }, }); return service.handleWebhook( endpoint.publicId, "slack", signedSlackWebhookRequest({ body, contentType: "application/json", url: webhookUrl, }), ); }; const guestEventSecond = Math.floor(Date.now() / 1_000); const guestRunId = randomUUID(); // The stub scheduler's initial admission is now represented by the // running run seeded below, not an extra queued target for Slack Stop. await db .update(agentWakeupRequests) .set({ status: "claimed", runId: guestRunId }) .where(eq(agentWakeupRequests.companyId, fixture.companyId)); await db.insert(heartbeatRuns).values({ id: guestRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", createdAt: new Date((guestEventSecond - 2) * 1_000), startedAt: new Date((guestEventSecond - 1) * 1_000), contextSnapshot: { issueId: conversation.issueId, source: "chat:slack", }, }); await db .update(issues) .set({ executionRunId: guestRunId, status: "in_progress" }) .where(eq(issues.id, conversation.issueId)); await expect( sendStop({ eventId: "EvSlackGuestStopDenied1", eventSecond: guestEventSecond, }), ).resolves.toMatchObject({ status: 202 }); expect(cancelRun).not.toHaveBeenCalled(); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq( chatActions.providerActionId, "slack_session_stop:EvSlackGuestStopDenied1", ), ), ), ).resolves.toEqual([ { status: "cancelled", result: { attempts: 1, code: "slack_session_stop_no_longer_authorized", retryable: false, }, }, ]); const intent = await service.createLinkIntent( endpoint.id, principal.id, 1_800, ); const token = new URL(intent.confirmationUrl).searchParams.get("token"); if (!token) throw new Error("Expected Slack identity-link token"); await service.confirmIdentityLink(token, "owner-user"); await db .update(heartbeatRuns) .set({ status: "cancelled", finishedAt: new Date() }) .where(eq(heartbeatRuns.id, guestRunId)); const delayedEventSecond = Math.floor(Date.now() / 1_000) - 5; const newerRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: newerRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", // This run begins less than one second after Slack's fractional Stop // timestamp. It is already newer work and must not inherit that Stop. createdAt: new Date(delayedEventSecond * 1_000 + 500), startedAt: new Date(delayedEventSecond * 1_000 + 750), contextSnapshot: { issueId: conversation.issueId, source: "chat:slack", }, }); await db .update(issues) .set({ executionRunId: newerRunId, updatedAt: new Date() }) .where(eq(issues.id, conversation.issueId)); await expect( sendStop({ eventId: "EvSlackDelayedStopBeforeNewRun1", eventSecond: delayedEventSecond, }), ).resolves.toMatchObject({ status: 202 }); expect(cancelRun).not.toHaveBeenCalled(); await expect( db .select({ status: chatActions.status, payload: chatActions.payload }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq( chatActions.providerActionId, "slack_session_stop:EvSlackDelayedStopBeforeNewRun1", ), ), ), ).resolves.toEqual([ expect.objectContaining({ status: "cancelled", payload: expect.objectContaining({ target: null }), }), ]); await db .update(heartbeatRuns) .set({ status: "cancelled", finishedAt: new Date() }) .where(eq(heartbeatRuns.id, newerRunId)); await db .update(issues) .set({ executionRunId: null, updatedAt: new Date() }) .where(eq(issues.id, conversation.issueId)); const queuedAt = new Date(Date.now() - 1_000); const [queuedWake] = await db .insert(agentWakeupRequests) .values({ companyId: fixture.companyId, agentId: fixture.assignedAgentId, source: "assignment", status: "queued", payload: { issueId: conversation.issueId }, requestedAt: queuedAt, }) .returning(); const queuedEventSecond = Math.floor(Date.now() / 1_000); await expect( sendStop({ eventId: "EvSlackQueuedWakeStop1", eventSecond: queuedEventSecond, }), ).resolves.toMatchObject({ status: 202 }); expect(cancelRun).not.toHaveBeenCalled(); await expect( db .select({ status: agentWakeupRequests.status }) .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, queuedWake!.id)), ).resolves.toEqual([{ status: "cancelled" }]); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq( chatActions.providerActionId, "slack_session_stop:EvSlackQueuedWakeStop1", ), ), ), ).resolves.toEqual([ { status: "processed", result: { attempts: 1, code: "slack_session_stop_cancelled", retryable: false, }, }, ]); await service.processPendingPublications(1_000); expect( runtime.endpoints .get(endpoint.id) ?.posts.some((post) => post.text === "Maya stopped at your request.") ?? false, ).toBe(true); const finishedRaceEventSecond = Math.floor(Date.now() / 1_000); const finishedRaceRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: finishedRaceRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", createdAt: new Date((finishedRaceEventSecond - 2) * 1_000), startedAt: new Date((finishedRaceEventSecond - 1) * 1_000), contextSnapshot: { issueId: conversation.issueId, source: "chat:slack", }, }); await db .update(issues) .set({ executionRunId: finishedRaceRunId, updatedAt: new Date() }) .where(eq(issues.id, conversation.issueId)); cancelRun.mockImplementationOnce(async (runId) => { await db .update(heartbeatRuns) .set({ status: "succeeded", finishedAt: new Date(), updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, runId)); return { id: runId, status: "succeeded" }; }); await expect( sendStop({ eventId: "EvSlackStopLosesToFinishedRun1", eventSecond: finishedRaceEventSecond, }), ).resolves.toMatchObject({ status: 202 }); expect(cancelRun).toHaveBeenCalledTimes(1); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq( chatActions.providerActionId, "slack_session_stop:EvSlackStopLosesToFinishedRun1", ), ), ), ).resolves.toEqual([ { status: "cancelled", result: { attempts: 1, code: "slack_session_stop_target_superseded", runId: finishedRaceRunId, }, }, ]); await service.shutdown(); const promotedWakeEventSecond = Math.floor(Date.now() / 1_000); await db .update(issues) .set({ executionRunId: null, updatedAt: new Date() }) .where(eq(issues.id, conversation.issueId)); const [promotedWake] = await db .insert(agentWakeupRequests) .values({ companyId: fixture.companyId, agentId: fixture.assignedAgentId, source: "assignment", status: "queued", payload: { issueId: conversation.issueId }, requestedAt: new Date((promotedWakeEventSecond - 1) * 1_000), }) .returning(); if (!promotedWake) throw new Error("Expected queued wake to promote"); const deferred: Array<() => void> = []; const promotedCancelRun = vi.fn(async (runId: string) => db .update(heartbeatRuns) .set({ status: "cancelled", errorCode: "slack_session_stopped", finishedAt: new Date(), updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, runId)) .returning() .then((rows) => rows[0] ?? null), ); const recovery = createService( new FakeChatSdkRuntime(), fakeSlackFetch() as typeof globalThis.fetch, { cancelRun: promotedCancelRun, deferWebhookProcessing: true, scheduleDeferredWork: (task) => deferred.push(task), }, ); const promotedBody = JSON.stringify({ type: "event_callback", team_id: providerAccountId, event_id: "EvSlackQueuedWakePromotedAfterStop1", event: { type: "agent_session_stopped", channel: channelId, thread_ts: threadTs, event_ts: `${promotedWakeEventSecond}.123456`, streaming_message_ts: [], user: externalUserId, }, }); await expect( recovery.service.handleWebhook( endpoint.publicId, "slack", signedSlackWebhookRequest({ body: promotedBody, contentType: "application/json", url: webhookUrl, }), ), ).resolves.toMatchObject({ status: 202 }); expect(deferred).toHaveLength(1); expect(promotedCancelRun).not.toHaveBeenCalled(); const promotedAction = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq( chatActions.providerActionId, "slack_session_stop:EvSlackQueuedWakePromotedAfterStop1", ), ), ) .then((rows) => rows[0]); expect(promotedAction).toMatchObject({ status: "received", payload: { target: { id: promotedWake.id, kind: "wakeup" } }, }); const promotedRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: promotedRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", wakeupRequestId: promotedWake.id, // Promotion happens after Slack's Stop instant, but this run remains the // exact durable continuation of the wake snapshotted before that instant. createdAt: new Date(promotedWakeEventSecond * 1_000 + 500), startedAt: new Date(promotedWakeEventSecond * 1_000 + 750), contextSnapshot: { issueId: conversation.issueId }, }); await db .update(agentWakeupRequests) .set({ status: "claimed", runId: promotedRunId, claimedAt: new Date(promotedWakeEventSecond * 1_000 + 500), updatedAt: new Date(), }) .where(eq(agentWakeupRequests.id, promotedWake.id)); await db .update(issues) .set({ executionRunId: promotedRunId, updatedAt: new Date() }) .where(eq(issues.id, conversation.issueId)); deferred.shift()?.(); await vi.waitFor(async () => { const settled = await db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, promotedAction!.id)) .then((rows) => rows[0]); expect(settled).toEqual({ status: "processed", result: { attempts: 1, code: "slack_session_stop_cancelled", runId: promotedRunId, }, }); }); expect(promotedCancelRun).toHaveBeenCalledOnce(); expect(promotedCancelRun).toHaveBeenCalledWith( promotedRunId, "Stopped from the bound Slack agent session", expect.objectContaining({ errorCode: "slack_session_stopped" }), ); await recovery.service.shutdown(); }); it("reorders rapid Slack callbacks by provider time before one conversation drain", async () => { const fixture = await seedCompany(); const runtime = new FakeChatSdkRuntime(); const deferred: Array<() => void> = []; const wakeup = vi.fn(async () => ({ accepted: true })); const service = chatChannelService(db, { deferWebhookProcessing: true, fetch: fakeSlackFetch() as typeof globalThis.fetch, heartbeat: { wakeup: receiptBackedWakeup(wakeup) }, publicBaseUrl: "https://paperclip.example", runtime: runtime as unknown as ChatSdkRuntime, scheduleDeferredWork: (task) => deferred.push(task), }); const endpoint = await service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await service.configure( endpoint.id, { action: "configure", credentials: { botToken: "xoxb-async-ingress", signingSecret: "async-ingress-secret", }, }, "owner-user", ); await recordSlackUrlVerification(service, endpoint.publicId); await service.configure(endpoint.id, { action: "verify" }, "owner-user"); const callbacks = runtime.configurations.get(endpoint.id)?.callbacks; if (!callbacks) throw new Error("Expected endpoint callbacks"); const slackTimestamp = (milliseconds: number) => `${Math.floor(Date.parse("2026-09-05T17:50:03.000Z") / 1_000)}.${String(milliseconds * 1_000).padStart(6, "0")}`; const thread = makeThread({ channelId: "C-ASYNC", id: `slack:C-ASYNC:${slackTimestamp(200)}`, name: "async-ingress", }); const laterReply = makeMessage({ id: slackTimestamp(517), raw: { ts: slackTimestamp(517) }, text: "and include the rollback status", }); laterReply.metadata.dateSent = new Date("2026-09-05T17:50:03.517Z"); // Slack Events API callbacks use independent HTTP requests. Reproduce the // live failure by receiving the later provider message first. await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: laterReply, trigger: "subscribed_message", }); const earlierMention = makeMessage({ id: slackTimestamp(200), raw: { ts: slackTimestamp(200) }, text: "@maya acknowledge quickly", mentioned: true, }); earlierMention.metadata.dateSent = new Date("2026-09-05T17:50:03.200Z"); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: earlierMention, trigger: "mention", }); for (let index = 3; index <= 8; index += 1) { const followUp = makeMessage({ id: slackTimestamp(index * 100), raw: { ts: slackTimestamp(index * 100) }, text: `follow-up ${index}`, }); followUp.metadata.dateSent = new Date(`2026-09-05T17:50:03.${index}00Z`); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: followUp, trigger: "subscribed_message", }); } const durable = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(durable).toHaveLength(8); expect(durable.every((delivery) => delivery.nextAttemptAt !== null)).toBe( true, ); // The first callback fixes one bounded batch deadline. Later arrivals do // not slide it forward and therefore cannot starve a busy conversation. expect( new Set( durable.map((delivery) => delivery.nextAttemptAt?.getTime() ?? null), ).size, ).toBe(1); expect(durable).toEqual( expect.arrayContaining([ expect.objectContaining({ providerEventId: `${thread.thread.id}:${earlierMention.id}`, state: "received", attempts: 0, }), expect.objectContaining({ providerEventId: `${thread.thread.id}:${laterReply.id}`, state: "received", attempts: 0, }), ]), ); expect( await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)), ).toHaveLength(0); expect(deferred).toHaveLength(1); const competingService = chatChannelService(db, { fetch: fakeSlackFetch() as typeof globalThis.fetch, heartbeat: { wakeup: receiptBackedWakeup(wakeup) }, publicBaseUrl: "https://paperclip.example", runtime: new FakeChatSdkRuntime() as unknown as ChatSdkRuntime, }); deferred.shift()?.(); // Simulate another server process reconciling the same durable rows at // the same time as the webhook process's deferred drain. await competingService.processPendingDeliveries(); await vi.waitFor(async () => { const rows = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); expect(rows).toHaveLength(1); }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); await vi.waitFor(async () => { const rows = await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)); expect(rows).toHaveLength(8); }); const comments = await db .select({ id: issueComments.id, body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)) .orderBy(asc(issueComments.createdAt), asc(issueComments.id)); expect(comments.map((comment) => comment.body)).toEqual([ "@maya acknowledge quickly", "follow-up 3", "follow-up 4", "follow-up 5", "and include the rollback status", "follow-up 6", "follow-up 7", "follow-up 8", ]); // Comment admission commits before the durable wake. Wait for this // company's last wake too, not merely its already-visible last comment. // The competing sweep may legitimately reconcile another fixture company. await vi.waitFor(() => { const calls = wakeup.mock.calls.filter( (call) => call[0] === fixture.assignedAgentId, ); expect(calls).toHaveLength(8); expect(calls.map((call) => call[1]?.payload?.wakeCommentId)).toEqual( comments.map((comment) => comment.id), ); }); // The last comment and wakeup commit inside the lease. Under full-suite // load the assertions above can observe those effects one microtask before // the deferred owner's `finally` deletes its lease. Require prompt eventual // release; a real leak would remain for the much longer lease TTL. await vi.waitFor(async () => { expect( await db .select() .from(chatEndpointLeases) .where(eq(chatEndpointLeases.endpointId, endpoint.id)), ).toHaveLength(0); }); await competingService.shutdown(); await service.shutdown(); }); it("stops a conversation drain after its lease renewal fails", async () => { const fixture = await seedCompany(); const deferred: Array<() => void> = []; let releaseFirstDelivery!: () => void; let signalFirstDelivery!: () => void; let signalRenewal!: () => void; const firstDeliveryEntered = new Promise((resolve) => { signalFirstDelivery = resolve; }); const firstDeliveryReleased = new Promise((resolve) => { releaseFirstDelivery = resolve; }); const renewalAttempted = new Promise((resolve) => { signalRenewal = resolve; }); let reachChecks = 0; const renewConversationDeliveryLease = vi.fn(async () => { signalRenewal(); throw new Error("simulated lease-renewal database outage"); }); const { callbacks, endpoint, service, wakeup } = await configuredSlackEndpoint(fixture, { conversationLeaseRenewalIntervalMs: 5, deferWebhookProcessing: true, reachAuthorizationBarrier: async () => { reachChecks += 1; if (reachChecks !== 1) return; signalFirstDelivery(); await firstDeliveryReleased; }, renewConversationDeliveryLease, scheduleDeferredWork: (task) => deferred.push(task), }); const thread = makeThread({ channelId: "C-LEASE-FAILURE", id: "slack:C-LEASE-FAILURE:9300.1", name: "lease-failure", }); const first = makeMessage({ id: "9300.1", mentioned: true, text: "@maya process only this turn under the current lease", }); first.metadata.dateSent = new Date("2026-09-06T15:00:00.100Z"); const second = makeMessage({ id: "9300.2", text: "do not overtake a failed lease renewal", }); second.metadata.dateSent = new Date("2026-09-06T15:00:00.200Z"); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: first, trigger: "mention", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: second, trigger: "subscribed_message", }); await db .update(chatDeliveries) .set({ nextAttemptAt: null }) .where(eq(chatDeliveries.endpointId, endpoint.id)); const processing = service.processPendingDeliveries(); await firstDeliveryEntered; await renewalAttempted; releaseFirstDelivery(); await processing; const states = await db .select({ providerEventId: chatDeliveries.providerEventId, state: chatDeliveries.state, }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), inArray(chatDeliveries.eventKind, ["mention", "message"]), ), ) .orderBy(asc(chatDeliveries.receivedAt)); expect(states).toEqual([ { providerEventId: `${thread.thread.id}:9300.1`, state: "processed", }, { providerEventId: `${thread.thread.id}:9300.2`, state: "received", }, ]); expect(renewConversationDeliveryLease).toHaveBeenCalledTimes(1); expect(wakeup).toHaveBeenCalledTimes(1); expect(deferred.length).toBeGreaterThan(0); // The failed renewal deliberately strands the second durable row. Settle // it through a fresh, explicitly targeted drain before this shared-database // test ends; otherwise a later global recovery pass correctly picks it up // with that later service's injected heartbeat client and pollutes its mock // call count. This also proves lease loss stops only the owning drain rather // than making the remaining delivery unrecoverable. const pendingDelivery = await db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, `${thread.thread.id}:${second.id}`, ), eq(chatDeliveries.state, "received"), ), ) .then((rows) => rows[0]); if (!pendingDelivery) throw new Error("Expected stranded second delivery"); renewConversationDeliveryLease.mockResolvedValue(true); await service.processPendingDeliveries(25, pendingDelivery.id); await expect( db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where(eq(chatDeliveries.id, pendingDelivery.id)), ).resolves.toEqual([{ state: "processed" }]); expect(wakeup).toHaveBeenCalledTimes(2); await service.shutdown(); }); describe("first-seen messages behind completed chat controls", () => { async function chronologyFixture( provider: "slack" | "telegram", overrides: Parameters[1] = {}, sameCompany?: Awaited>, ) { const fixture = sameCompany ?? (await seedCompany()); const configured = provider === "slack" ? await configuredSlackEndpoint(fixture, overrides) : await configuredTelegramEndpoint(fixture, overrides); const { callbacks, endpoint, service, wakeup } = configured; const chatId = provider === "slack" ? "D09LATECONTROL" : "77119981"; const userId = provider === "slack" ? "U-LATE-CONTROL" : chatId; const thread = makeThread({ id: provider === "slack" ? `slack:${chatId}:` : `telegram:${chatId}`, channelId: chatId, isDM: true, name: "Late control source", }).thread; const controlSecond = Math.floor(Date.now() / 1_000) - 10; const admit = async ( sequence: number, second: number, text: string, destination = thread, trigger: ChatSdkMessageTrigger = "direct_message", ) => { const sentAt = new Date(second * 1_000); const providerMessageId = provider === "slack" ? `${second}.${String(sequence).padStart(6, "0")}` : `${chatId}:${sequence}`; const message = makeMessage({ id: providerMessageId, text, userId, raw: provider === "slack" ? { ts: providerMessageId, user: userId, text } : { message_id: sequence, date: second, chat: { id: Number(chatId), type: "private" }, from: { id: Number(userId), is_bot: false }, text, }, }); message.metadata.dateSent = provider === "slack" ? new Date(Number(providerMessageId) * 1_000) : sentAt; const currentCallbacks = configured.runtime.configurations.get(endpoint.id)?.callbacks ?? callbacks; await deliverMessage({ callbacks: currentCallbacks, endpointId: endpoint.id, provider, thread: destination, message, trigger, }); const [delivery] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), sql`${chatDeliveries.normalizedEvent}->'message'->>'providerMessageId' = ${providerMessageId}`, sql`${chatDeliveries.normalizedEvent}->'conversation'->>'externalThreadId' = ${destination.id}`, ), ); if (!delivery) throw new Error("Expected exact provider source receipt"); await service.processPendingDeliveries(25, delivery.id); return db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, delivery.id)) .then((rows) => rows[0]!); }; await admit(100, controlSecond - 5, "Initial task before control"); await qualifySetupRoundTrip(service, endpoint.id, userId); await service.test(endpoint.id, "owner-user"); let commandSequence = 200; const control = async (command: "close" | "new", publish = true) => { await callbacks.onSlashCommand!({ endpointId: endpoint.id, provider, event: { command: provider === "slack" ? String(endpoint.setup.command) : `/${command}`, text: provider === "slack" ? command : "", triggerId: randomUUID(), channel: { id: provider === "slack" ? `slack:${chatId}` : thread.id, isDM: true, name: "Late control source", } as never, user: { userId, userName: "late-control-user", fullName: "Late Control User", isBot: false, isMe: false, isSystem: false, }, raw: provider === "slack" ? { command: endpoint.setup.command, text: command, channel_id: chatId, user_id: userId, } : { message_id: commandSequence++, date: controlSecond, chat: { id: Number(chatId), type: "private" }, from: { id: Number(userId), is_bot: false }, text: `/${command}`, entities: [ { offset: 0, length: command.length + 1, type: "bot_command", }, ], }, adapter: {} as never, openModal: async () => undefined, }, }); if (publish) await service.processPendingPublications(); const [publication] = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), like(chatPublications.idempotencyKey, `control:${command}:%`), ), ) .orderBy(desc(chatPublications.createdAt)); expect(publication).toMatchObject({ state: publish ? "published" : "pending", }); if (publish) expect((await service.listConversations(endpoint.id))[0]?.state).toBe( "completed", ); return publication!; }; const snapshot = async () => ({ issues: await db .select({ id: issues.id }) .from(issues) .where(eq(issues.companyId, fixture.companyId)) .orderBy(asc(issues.id)), conversations: await db .select({ id: chatConversations.id, state: chatConversations.state, generation: chatConversations.sessionGeneration, }) .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)) .orderBy(asc(chatConversations.id)), comments: await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.companyId, fixture.companyId)) .orderBy(asc(issueComments.id)), actions: await db .select({ id: chatActions.id }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "inbound_wakeup"), ), ) .orderBy(asc(chatActions.id)), wakeCount: wakeup.mock.calls.length, }); return { ...configured, fixture, thread, userId, chatId, controlSecond, admit, control, snapshot, close: () => retirePublicationFixture(service, endpoint.id), }; } it.each([ ["slack", "close", false], ["slack", "new", false], ["telegram", "close", false], ["telegram", "new", false], ["telegram", "close", true], ["telegram", "new", true], ] as const)( "refuses unseen %s input before %s (same second: %s) without creating work", async (provider, command, sameSecond) => { const f = await chronologyFixture(provider); try { const control = await f.control(command); const before = await f.snapshot(); const delayed = await f.admit( 199, f.controlSecond - (sameSecond ? 0 : 1), "Delayed old input, not a fresh request", ); expect(delayed.receivedAt.getTime()).toBeGreaterThan( control.publishedAt!.getTime(), ); expect(delayed.state).toBe("filtered"); expect(delayed.conversationId).toBeNull(); expect(await f.snapshot()).toEqual(before); const freshSecond = provider === "telegram" && sameSecond ? f.controlSecond : Math.floor(Date.now() / 1_000) + 1; if (freshSecond * 1_000 > Date.now()) await vi.waitFor( () => expect(Date.now()).toBeGreaterThanOrEqual(freshSecond * 1_000), { timeout: 1_500, interval: 10 }, ); const fresh = await f.admit( 201, freshSecond, "A genuinely later explicit request", ); expect(fresh.state).toBe("processed"); const afterFresh = await f.snapshot(); expect(afterFresh.issues).toHaveLength(before.issues.length + 1); expect(afterFresh.wakeCount).toBe(before.wakeCount + 1); expect(afterFresh.conversations).toContainEqual( expect.objectContaining({ state: "active", generation: 2 }), ); const olderIntoNewGeneration = await f.admit( 198, f.controlSecond - 1, "Another unseen old message after the new generation", ); expect(olderIntoNewGeneration.state).toBe("filtered"); expect(await f.snapshot()).toEqual(afterFresh); } finally { await f.close(); } }, ); it.each(["slack", "telegram"] as const)( "accepts delayed %s backlog without an explicit control", async (provider) => { const f = await chronologyFixture(provider); try { const before = await f.snapshot(); const delivery = await f.admit( 99, f.controlSecond - 20, "Legitimate delayed backlog on the same active task", ); expect(delivery.state).toBe("processed"); const after = await f.snapshot(); expect(after.issues).toEqual(before.issues); expect(after.conversations).toEqual(before.conversations); expect(after.comments).toHaveLength(before.comments.length + 1); expect(after.wakeCount).toBe(before.wakeCount + 1); } finally { await f.close(); } }, ); it("rechecks a close committed while old input waits before the endpoint transaction", async () => { let arm = false; let entered!: () => void; let release!: () => void; const waiting = new Promise((resolve) => { entered = resolve; }); const gate = new Promise((resolve) => { release = resolve; }); const f = await chronologyFixture("telegram", { reachAuthorizationBarrier: async () => { if (arm) { entered(); await gate; } }, }); let input: ReturnType | undefined; try { await f.control("close", false); arm = true; input = f.admit( 199, f.controlSecond - 1, "Old input held before current authority transaction", ); await waiting; await f.service.processPendingPublications(); const before = await f.snapshot(); expect(before.conversations[0]?.state).toBe("completed"); release(); expect((await input).state).toBe("filtered"); expect(await f.snapshot()).toEqual(before); } finally { release(); await input; await f.close(); } }); it("honors operator-confirmed close without inventing an outbound message receipt", async () => { const f = await chronologyFixture("telegram"); try { const publication = await f.control("close", false); const providerRuntime = f.runtime.endpoints.get(f.endpoint.id)!; providerRuntime.postError = new Error( "connection closed after provider acceptance became unknown", ); await f.service.processPendingPublications(); const [unknown] = await db .select() .from(chatPublications) .where(eq(chatPublications.id, publication.id)); expect(unknown).toMatchObject({ state: "delivery_unknown", providerMessageId: null, }); providerRuntime.postError = null; await f.service.resolvePublication( f.endpoint.id, publication.id, "mark_delivered", "owner-user", ); const [receipt] = await db .select() .from(chatActions) .where( eq( chatActions.providerActionId, `task-control-authorization:${publication.id}`, ), ); expect(receipt).toMatchObject({ status: "processed", result: { code: "task_control_marked_delivered_by_operator" }, }); expect( await db .select() .from(chatMessageLinks) .where(eq(chatMessageLinks.publicationId, publication.id)), ).toEqual([]); const before = await f.snapshot(); expect( ( await f.admit( 199, f.controlSecond - 1, "Old input after operator confirmation", ) ).state, ).toBe("filtered"); expect(await f.snapshot()).toEqual(before); } finally { await f.close(); } }); it.each(["slack", "telegram"] as const)( "does not promote metadata-only %s timestamps after a completed control", async (provider) => { const f = await chronologyFixture(provider); try { await f.control("close"); const before = await f.snapshot(); const message = makeMessage({ id: provider === "slack" ? `${Math.floor(Date.now() / 1_000)}.000777` : `${f.chatId}:777`, raw: provider === "slack" ? {} : { message_id: 777 }, text: "Metadata now is not provider chronology", userId: f.userId, }); await deliverMessage({ callbacks: f.callbacks, endpointId: f.endpoint.id, provider, thread: f.thread, message, trigger: "direct_message", }); expect(await f.snapshot()).toEqual(before); const [delivery] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, f.endpoint.id), sql`${chatDeliveries.normalizedEvent}->'message'->>'providerMessageId' = ${message.id}`, ), ); expect(delivery).toMatchObject({ state: "filtered", normalizedEvent: expect.objectContaining({ message: expect.objectContaining({ providerSentAt: null }), }), }); expect(delivery?.redactedError).toContain("Send a new request"); } finally { await f.close(); } }, ); it("continues to accept authenticated Slack slash close/new on a genuinely new generation", async () => { const f = await chronologyFixture("slack"); try { await f.control("close"); const freshSecond = Math.floor(Date.now() / 1_000) + 1; await vi.waitFor( () => expect(Date.now()).toBeGreaterThanOrEqual(freshSecond * 1_000), { timeout: 1_500, interval: 10 }, ); expect( ( await f.admit( 901, freshSecond, "A fresh task before native slash new", ) ).state, ).toBe("processed"); await f.control("new"); const controls = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, f.endpoint.id), sql`${chatDeliveries.normalizedEvent}->'admission'->>'origin' = 'slack_slash_control'`, ), ); expect(controls).toHaveLength(2); expect(controls.every((control) => control.state === "processed")).toBe( true, ); expect( (await f.snapshot()).conversations.every( (conversation) => conversation.state === "completed", ), ).toBe(true); } finally { await f.close(); } }); it("keeps controls scoped to their endpoint, immutable thread, and company", async () => { const f = await chronologyFixture("slack"); const others: Awaited>[] = []; try { await f.control("close"); const otherThread = makeThread({ id: `${f.thread.id}111.000001`, channelId: f.chatId, isDM: true, name: "Independent DM thread", }).thread; expect( ( await f.admit( 299, f.controlSecond - 1, "Independent thread backlog", otherThread, ) ).state, ).toBe("processed"); others.push(await chronologyFixture("slack", {}, f.fixture)); others.push(await chronologyFixture("slack")); for (const independent of others) { expect( ( await independent.admit( 299, f.controlSecond - 1, "Independent endpoint or tenant backlog", ) ).state, ).toBe("processed"); } } finally { for (const independent of others) await independent.close(); await f.close(); } }); it("does not recreate or filter a previously committed inbound link when its delivery retries after close", async () => { const f = await chronologyFixture("telegram"); try { const accepted = await f.admit( 101, f.controlSecond - 2, "Already accepted before the close", ); await f.control("close"); const before = await f.snapshot(); const links = await db .select() .from(chatMessageLinks) .where(eq(chatMessageLinks.deliveryId, accepted.id)); await db .update(chatDeliveries) .set({ state: "retry", nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, accepted.id)); await f.service.processPendingDeliveries(25, accepted.id); const [retried] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, accepted.id)); // Existing current-access recovery refuses this now-closed source. The // first-seen filter must not erase or recreate its already saved work. expect(retried).toMatchObject({ state: "failed", conversationId: accepted.conversationId, }); expect(retried?.redactedError).toContain("no longer authorized"); expect( await db .select() .from(chatMessageLinks) .where(eq(chatMessageLinks.deliveryId, accepted.id)), ).toEqual(links); expect(await f.snapshot()).toEqual(before); } finally { await f.close(); } }); it.each(["close", "new"] as const)( "does not let an unseen old Telegram /%s close a newer generation", async (command) => { const f = await chronologyFixture("telegram"); try { await f.control("close"); expect( ( await f.admit( 201, f.controlSecond + 1, "Fresh generation after close", ) ).state, ).toBe("processed"); const before = await f.snapshot(); expect( (await f.admit(198, f.controlSecond - 1, `/${command}`)).state, ).toBe("filtered"); await f.service.processPendingPublications(); expect(await f.snapshot()).toEqual(before); } finally { await f.close(); } }, ); it("retains a committed control boundary through pause, redacted duplicate, and resume", async () => { const f = await chronologyFixture("telegram"); try { const control = await f.control("close"); const [originalAuthorization] = await db .select() .from(chatActions) .where( eq( chatActions.providerActionId, `task-control-authorization:${control.id}`, ), ); await f.service.configure( f.endpoint.id, { action: "pause" }, "owner-user", ); const duplicate = await f.admit( 200, f.controlSecond, "/close", f.thread, "mention", ); expect(duplicate).toMatchObject({ state: "processed", principalId: originalAuthorization!.principalId, normalizedEvent: expect.objectContaining({ filtering: { contentRetained: false }, }), }); expect(duplicate.normalizedEvent.message).not.toHaveProperty("text"); expect(duplicate.eventKind).toBe("direct_message"); expect(duplicate.normalizedEvent.kind).toBe("mention"); await f.service.configure( f.endpoint.id, { action: "resume" }, "owner-user", ); const before = await f.snapshot(); expect( ( await f.admit( 199, f.controlSecond - 1, "Old source after resumed redacted control", ) ).state, ).toBe("filtered"); expect(await f.snapshot()).toEqual(before); const freshSecond = Math.floor(Date.now() / 1_000) + 1; await vi.waitFor( () => expect(Date.now()).toBeGreaterThanOrEqual(freshSecond * 1_000), { timeout: 1_500, interval: 10 }, ); expect( (await f.admit(201, freshSecond, "Fresh request after resumed control")) .state, ).toBe("processed"); const [retained] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, duplicate.id)); expect(retained?.normalizedEvent).toEqual(duplicate.normalizedEvent); expect(control.publishedAt).not.toBeNull(); } finally { await f.close(); } }); it("retains a completed trailing-newline control as a first-seen source boundary", async () => { const f = await chronologyFixture("telegram"); try { const delivery = await f.admit(200, f.controlSecond, "/close\n"); await f.service.processPendingPublications(); const [control] = await db .select() .from(chatPublications) .where( eq(chatPublications.idempotencyKey, `control:close:${delivery.id}`), ); expect(control?.state).toBe("published"); expect((await f.service.listConversations(f.endpoint.id))[0]?.state).toBe( "completed", ); const before = await f.snapshot(); expect( ( await f.admit( 199, f.controlSecond - 1, "Old input after a valid newline-terminated close", ) ).state, ).toBe("filtered"); expect(await f.snapshot()).toEqual(before); } finally { await f.close(); } }); it.each([false, true])( "preserves a Teams close across regional routes (close after route change: %s)", async (closeAfterRouteChange) => { const fixture = await seedCompany(); const f = await configuredTeamsEndpoint(fixture); try { const conversationId = "19:late-control-route@thread.v2"; const base = `teams:${Buffer.from(conversationId).toString("base64url")}`; const oldId = `${base}:${Buffer.from("https://smba.trafficmanager.net/amer/").toString("base64url")}`; const oldThread = makeThread({ id: oldId, channelId: oldId, isDM: true, }).thread; const currentThread = makeThread({ id: base, channelId: base, isDM: true, }).thread; const controlTime = new Date(Date.now() - 10_000); const receive = async ( thread: Thread, id: string, time: Date, text: string, ) => { await deliverMessage({ callbacks: f.callbacks, endpointId: f.endpoint.id, provider: "microsoft-teams", thread, trigger: "direct_message", message: makeMessage({ id, userId: "teams-late-user", text, raw: { timestamp: time.toISOString(), serviceUrl: thread.id === oldId ? "https://smba.trafficmanager.net/amer/" : "https://smba.trafficmanager.net/emea/", from: { aadObjectId: "teams-late-user" }, }, }), }); return db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, f.endpoint.id), sql`${chatDeliveries.normalizedEvent}->'message'->>'providerMessageId' = ${id}`, ), ) .then((rows) => rows[0]!); }; await receive( oldThread, "teams-source-1", new Date(controlTime.getTime() - 5_000), "Initial Teams task", ); await qualifySetupRoundTrip( f.service, f.endpoint.id, "teams-late-user", ); await f.service.test(f.endpoint.id, "owner-user"); await receive( closeAfterRouteChange ? currentThread : oldThread, "teams-close-2", controlTime, "/close", ); await f.service.processPendingPublications(); expect( (await f.service.listConversations(f.endpoint.id))[0]?.state, ).toBe("completed"); const before = f.wakeup.mock.calls.length; expect( ( await receive( currentThread, "teams-delayed-3", new Date(controlTime.getTime() - 1_000), "Old Teams request after close", ) ).state, ).toBe("filtered"); expect(f.wakeup).toHaveBeenCalledTimes(before); expect(await f.service.listConversations(f.endpoint.id)).toHaveLength( 1, ); expect( ( await receive( currentThread, "teams-fresh-4", new Date(controlTime.getTime() + 1_000), "Fresh Teams request", ) ).state, ).toBe("processed"); expect(f.wakeup).toHaveBeenCalledTimes(before + 1); expect(await f.service.listConversations(f.endpoint.id)).toHaveLength( 2, ); } finally { await retirePublicationFixture(f.service, f.endpoint.id); } }, ); it("does not lower a newer provider boundary when an older close is confirmed later", async () => { const f = await chronologyFixture("telegram"); try { const older = await f.control("close", false); const providerRuntime = f.runtime.endpoints.get(f.endpoint.id)!; providerRuntime.postError = new Error( "older control confirmation response lost", ); await f.service.processPendingPublications(); const [unknown] = await db .select() .from(chatPublications) .where(eq(chatPublications.id, older.id)); expect(unknown?.state).toBe("delivery_unknown"); providerRuntime.postError = null; // FIFO correctly blocks later publications in this old conversation. // A normal Board task completion lets fresh DM work start a separate // generation; its control can complete before the old unknown result // receives a delayed operator confirmation. if (!older.issueId) throw new Error("Expected the old control task"); await issueService(db).update(older.issueId, { status: "done", actorUserId: "owner-user", }); const replacement = await f.admit( 275, f.controlSecond + 1, "Start independently after Board task completion", ); expect(replacement.state).toBe("processed"); expect(replacement.conversationId).not.toBe(older.conversationId); await f.admit(300, f.controlSecond + 5, "/close"); await f.service.processPendingPublications(); const controls = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, f.endpoint.id), like(chatPublications.idempotencyKey, "control:close:%"), ), ); expect(controls).toHaveLength(2); expect(controls.find((control) => control.id !== older.id)?.state).toBe( "published", ); await f.service.resolvePublication( f.endpoint.id, older.id, "mark_delivered", "owner-user", ); const [lastConfirmed] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, f.endpoint.id), eq(chatActions.kind, "task_control_authorization"), ), ) .orderBy(desc(chatActions.updatedAt)); expect(lastConfirmed?.payload.publicationId).toBe(older.id); const before = await f.snapshot(); expect( ( await f.admit( 250, f.controlSecond + 2, "Between the older and newer provider controls", ) ).state, ).toBe("filtered"); expect(await f.snapshot()).toEqual(before); expect( (await f.admit(301, f.controlSecond + 6, "After both provider controls")) .state, ).toBe("processed"); } finally { await f.close(); } }); it.each([ "unprocessed_authorization", "unprocessed_source", "unbound_control_key", ] as const)( "does not infer a stop from bare completed state with %s", async (mode) => { const f = await chronologyFixture("telegram"); try { const control = await f.control("close"); if (mode === "unprocessed_authorization") await db .update(chatActions) .set({ status: "issued" }) .where( eq( chatActions.providerActionId, `task-control-authorization:${control.id}`, ), ); else if (mode === "unprocessed_source") await db .update(chatDeliveries) .set({ state: "failed" }) .where( eq(chatDeliveries.id, control.idempotencyKey.split(":")[2]!), ); else await db .update(chatPublications) .set({ idempotencyKey: `control:close:${randomUUID()}` }) .where(eq(chatPublications.id, control.id)); expect( ( await f.admit( 199, f.controlSecond - 1, "A bare completed row is not an explicit control proof", ) ).state, ).toBe("processed"); } finally { await f.close(); } }, ); }); it("reorders reverse-arrival Telegram webhooks by provider sequence before waking the agent", async () => { const fixture = await seedCompany(); const runtime = new FakeChatSdkRuntime(); const deferred: Array<() => void> = []; const wakeup = vi.fn(async () => ({ accepted: true })); const service = chatChannelService(db, { deferWebhookProcessing: true, fetch: fakeTelegramFetch() as typeof globalThis.fetch, heartbeat: { wakeup: receiptBackedWakeup(wakeup) }, publicBaseUrl: "https://paperclip.example", runtime: runtime as unknown as ChatSdkRuntime, scheduleDeferredWork: (task) => deferred.push(task), }); const endpoint = await service.create( fixture.companyId, { provider: "telegram", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await service.configure( endpoint.id, { action: "configure", credentials: { botToken: "123456:telegram-ordering-test" }, }, "owner-user", ); const callbacks = runtime.configurations.get(endpoint.id)?.callbacks; if (!callbacks) throw new Error("Expected Telegram callbacks"); const dm = makeThread({ channelId: "77117711", id: "telegram:77117711", isDM: true, name: "Telegram ordered delivery", }); const providerSecond = new Date("2026-09-05T19:15:20.000Z"); // Only the raw provider date supplies retained Telegram chronology. const later = makeMessage({ id: "telegram:77117711:102", raw: { message_id: 102, date: providerSecond.getTime() / 1_000 }, text: "second Telegram turn", userId: "77117711", }); later.metadata.dateSent = providerSecond; await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", providerUpdateId: 7002, thread: dm.thread, message: later, trigger: "direct_message", }); const earlier = makeMessage({ id: "telegram:77117711:101", raw: { message_id: 101, date: providerSecond.getTime() / 1_000 }, text: "first Telegram turn", userId: "77117711", }); earlier.metadata.dateSent = providerSecond; await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", providerUpdateId: 7001, thread: dm.thread, message: earlier, trigger: "direct_message", }); const durable = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(durable).toHaveLength(2); expect(durable.every((delivery) => delivery.nextAttemptAt !== null)).toBe( true, ); expect( new Set( durable.map((delivery) => delivery.nextAttemptAt?.getTime() ?? null), ).size, ).toBe(1); expect(durable.map((delivery) => delivery.normalizedEvent)).toEqual( expect.arrayContaining([ expect.objectContaining({ message: expect.objectContaining({ providerMessageSequence: 101, providerUpdateId: 7001, providerSentAt: providerSecond.toISOString(), providerSentAtSource: "telegram_message_date", }), }), expect.objectContaining({ message: expect.objectContaining({ providerMessageSequence: 102, providerUpdateId: 7002, providerSentAt: providerSecond.toISOString(), providerSentAtSource: "telegram_message_date", }), }), ]), ); expect( await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)), ).toHaveLength(0); expect(wakeup).not.toHaveBeenCalled(); expect(deferred).toHaveLength(1); deferred.shift()?.(); await vi.waitFor(() => expect(wakeup).toHaveBeenCalledTimes(2), { timeout: 3_000, }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); if (!conversation) throw new Error("Expected Telegram conversation"); const comments = await db .select({ id: issueComments.id, body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)) .orderBy(asc(issueComments.createdAt), asc(issueComments.id)); expect(comments.map((comment) => comment.body)).toEqual([ "first Telegram turn", "second Telegram turn", ]); expect( wakeup.mock.calls.map((call) => call[1]?.payload?.wakeCommentId), ).toEqual(comments.map((comment) => comment.id)); await service.shutdown(); }); it("orders a Telegram new command before the next message from the same provider second", async () => { const fixture = await seedCompany(); const runtime = new FakeChatSdkRuntime(); const deferred: Array<() => void> = []; const wakeup = vi.fn(async () => ({ accepted: true })); const service = chatChannelService(db, { deferWebhookProcessing: true, fetch: fakeTelegramFetch() as typeof globalThis.fetch, heartbeat: { wakeup: receiptBackedWakeup(wakeup) }, publicBaseUrl: "https://paperclip.example", runtime: runtime as unknown as ChatSdkRuntime, scheduleDeferredWork: (task) => deferred.push(task), }); const endpoint = await service.create( fixture.companyId, { provider: "telegram", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await service.configure( endpoint.id, { action: "configure", credentials: { botToken: "123456:telegram-command-order-test" }, }, "owner-user", ); const callbacks = runtime.configurations.get(endpoint.id)?.callbacks; if (!callbacks?.onSlashCommand) throw new Error("Expected Telegram slash-command callbacks"); const providerSecond = new Date(Math.floor(Date.now() / 1_000) * 1_000); const thread = makeThread({ channelId: "D-TELEGRAM-CONTROL-ORDER", id: "telegram:D-TELEGRAM-CONTROL-ORDER", isDM: true, name: "Telegram command ordering", }); await callbacks.onSlashCommand({ endpointId: endpoint.id, provider: "telegram", event: { channel: { id: thread.thread.id, name: thread.thread.name, isDM: true, post: vi.fn(), } as never, command: "/new", text: "", user: { userId: "77117711", userName: "telegram-order-user", fullName: "Telegram Order User", isBot: false, isMe: false, isSystem: false, }, raw: { message_id: 110, date: Math.floor(providerSecond.getTime() / 1_000), chat: { id: 77117711, type: "private" }, from: { id: 77117711, is_bot: false }, text: "/new", entities: [{ offset: 0, length: 4, type: "bot_command" }], }, adapter: {} as never, openModal: async () => undefined, }, }); const request = makeMessage({ id: "telegram:77117711:111", raw: { message_id: 111, date: providerSecond.getTime() / 1_000 }, text: "Start the task after resetting this chat", userId: "77117711", }); request.metadata.dateSent = providerSecond; await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", providerUpdateId: 8011, thread: thread.thread, message: request, trigger: "direct_message", }); const durable = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(durable).toHaveLength(2); expect(durable).toEqual( expect.arrayContaining([ expect.objectContaining({ normalizedEvent: expect.objectContaining({ message: expect.objectContaining({ providerMessageId: "77117711:110", providerMessageSequence: 110, providerSentAt: providerSecond.toISOString(), providerSentAtSource: "telegram_message_date", text: "/new", }), }), }), expect.objectContaining({ normalizedEvent: expect.objectContaining({ message: expect.objectContaining({ providerMessageSequence: 111, providerSentAt: providerSecond.toISOString(), providerSentAtSource: "telegram_message_date", text: "Start the task after resetting this chat", }), }), }), ]), ); expect(deferred).toHaveLength(1); deferred.shift()?.(); await vi.waitFor(() => expect(wakeup).toHaveBeenCalledTimes(1), { timeout: 3_000, }); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ state: "active", sessionGeneration: 1 }), ]); expect(runtime.endpoints.get(endpoint.id)?.posts[0]?.text).toBe( "Send your request to start a new Paperclip task.", ); // Command-only acknowledgements have no run whose final reply can retire // a processing reaction. The subsequent task message has its own receipt. expect(runtime.endpoints.get(endpoint.id)?.reactions).not.toContainEqual({ threadId: "telegram:D-TELEGRAM-CONTROL-ORDER", messageId: "77117711:110", emoji: "eyes", }); await service.shutdown(); }); it("holds a delayed Slack thread reply until its older root mention arrives", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredSlackEndpoint(fixture); await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: "C-DELAYED-ROOT", label: "delayed-root", availability: "available", enabled: true, }); const thread = makeThread({ channelId: "C-DELAYED-ROOT", id: "slack:C-DELAYED-ROOT:9110.1", name: "delayed-root", }); const laterReply = makeMessage({ id: "9110.2", text: "follow-up whose callback arrived first", }); laterReply.metadata.dateSent = new Date("2026-09-05T18:20:02.000Z"); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: laterReply, trigger: "subscribed_message", }); const [deferredReply] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(deferredReply).toMatchObject({ state: "retry", attempts: 1, redactedError: "Waiting briefly for an earlier root mention", }); expect(deferredReply.nextAttemptAt).not.toBeNull(); expect(await service.listConversations(endpoint.id)).toHaveLength(0); const earlierRoot = makeMessage({ id: "9110.1", text: "@maya keep both messages", mentioned: true, }); earlierRoot.metadata.dateSent = new Date("2026-09-05T18:20:01.000Z"); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: earlierRoot, trigger: "mention", }); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, deferredReply.id)); await service.processPendingDeliveries(); const [conversation] = await service.listConversations(endpoint.id); expect(conversation).toMatchObject({ externalThreadId: thread.thread.id }); await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, conversation!.issueId)) .orderBy(asc(issueComments.createdAt), asc(issueComments.id)), ).resolves.toEqual([ { body: "@maya keep both messages" }, { body: "follow-up whose callback arrived first" }, ]); expect(wakeup).toHaveBeenCalledTimes(2); }); it("retains a standalone unaddressed Slack thread reply for the bounded reorder window", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service, wakeup } = await configuredSlackEndpoint(fixture); await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: "C-ORPHAN-ONLY", label: "orphan-only", availability: "available", enabled: true, }); const thread = makeThread({ channelId: "C-ORPHAN-ONLY", id: "slack:C-ORPHAN-ONLY:9120.1", name: "orphan-only", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: "9120.2", text: "not for the bot" }), trigger: "subscribed_message", }); const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(delivery).toMatchObject({ state: "retry", attempts: 1 }); for (let expectedAttempt = 2; expectedAttempt <= 13; expectedAttempt += 1) { await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, delivery.id)); await service.processPendingDeliveries(); await expect( db .select({ state: chatDeliveries.state, attempts: chatDeliveries.attempts, }) .from(chatDeliveries) .where(eq(chatDeliveries.id, delivery.id)), ).resolves.toEqual([ { state: expectedAttempt === 13 ? "filtered" : "retry", attempts: expectedAttempt, }, ]); } expect(await service.listConversations(endpoint.id)).toHaveLength(0); expect(wakeup).not.toHaveBeenCalled(); }); it("keeps Teams group chat closed by default without consuming first-channel setup enablement", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredTeamsEndpoint(fixture); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "verifying", allowGroupChats: false, }); const serviceUrl = "https://smba.trafficmanager.net/amer/"; const groupConversationId = "19:teams-setup-group@unq.gbl.spaces"; const groupThread = makeThread({ channelId: `teams:${Buffer.from(groupConversationId).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, id: `teams:${Buffer.from(groupConversationId).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, name: "Setup group", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: groupThread.thread, message: makeMessage({ id: "teams-setup-group-disabled", text: "@maya this group is still closed", mentioned: true, }), trigger: "mention", }); expect(wakeup).not.toHaveBeenCalled(); const [filteredGroupDelivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(filteredGroupDelivery).toMatchObject({ state: "filtered", attempts: 0, principalId: null, redactedError: "Destination is not enabled in Paperclip", normalizedEvent: { filtering: { contentRetained: false }, message: { providerMessageId: "teams-setup-group-disabled" }, }, }); expect( JSON.stringify(filteredGroupDelivery?.normalizedEvent), ).not.toContain("this group is still closed"); expect( JSON.stringify(filteredGroupDelivery?.normalizedEvent), ).not.toContain("U-EXTERNAL"); await expect( db .select({ id: chatExternalPrincipals.id }) .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, fixture.companyId), eq(chatExternalPrincipals.provider, "microsoft-teams"), ), ), ).resolves.toHaveLength(0); const channelConversationId = "19:teams-first-channel@thread.tacv2"; const rootMessageId = "1740000000091"; const channelThread = makeThread({ channelId: `teams:${Buffer.from(channelConversationId).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, id: `teams:${Buffer.from(`${channelConversationId};messageid=${rootMessageId}`).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, name: "First setup channel", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: channelThread.thread, message: makeMessage({ id: rootMessageId, text: "@maya start the channel setup task", mentioned: true, }), trigger: "mention", }); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ externalThreadId: channelThread.thread.id, isDirectMessage: false, }), ]); await expect(service.listResources(endpoint.id)).resolves.toEqual( expect.arrayContaining([ expect.objectContaining({ providerResourceId: groupConversationId, type: "group_chat", enabled: false, }), expect.objectContaining({ providerResourceId: channelConversationId, type: "channel", enabled: true, }), ]), ); expect(wakeup).toHaveBeenCalledTimes(1); }); it("continues a legacy Teams binding across a regional route change without forking its task", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredTeamsEndpoint(fixture); // The inverted question mark contributes a `/` in standard base64, which // becomes `_` in base64url. Keep this fixture explicit so the canonical // decoder is exercised on an actual URL-safe alphabet difference. const conversationId = "19:stable-route-¿@thread.tacv2"; const rootMessageId = "1740000000199"; const rootedConversationId = `${conversationId};messageid=${rootMessageId}`; expect(Buffer.from(rootedConversationId).toString("base64url")).toContain( "_", ); const amerServiceUrl = "https://smba.trafficmanager.net/amer/"; const emeaServiceUrl = "https://smba.trafficmanager.net/emea/"; const legacyThreadId = `teams:${Buffer.from(rootedConversationId).toString("base64url")}:${Buffer.from(amerServiceUrl).toString("base64url")}`; const canonicalThreadId = `teams:${Buffer.from(rootedConversationId).toString("base64url")}`; const legacyEmeaThreadId = `teams:${Buffer.from(rootedConversationId).toString("base64url")}:${Buffer.from(emeaServiceUrl).toString("base64url")}`; const legacyChannelId = `teams:${Buffer.from(conversationId).toString("base64url")}:${Buffer.from(amerServiceUrl).toString("base64url")}`; const canonicalChannelId = `teams:${Buffer.from(conversationId).toString("base64url")}`; const legacyThread = makeThread({ channelId: legacyChannelId, id: legacyThreadId, name: "Stable Teams channel", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: legacyThread.thread, message: makeMessage({ id: rootMessageId, mentioned: true, raw: { serviceUrl: amerServiceUrl, from: { aadObjectId: "stable-teams-user" }, }, text: "@maya preserve this task across route changes", }), trigger: "mention", }); const canonicalThread = makeThread({ channelId: canonicalChannelId, id: canonicalThreadId, name: "Stable Teams channel", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: canonicalThread.thread, message: makeMessage({ id: "1740000000200", raw: { serviceUrl: emeaServiceUrl, from: { aadObjectId: "stable-teams-user" }, }, text: "continue after Microsoft moved the reply route", }), trigger: "subscribed_message", }); const legacyEmeaThread = makeThread({ channelId: `teams:${Buffer.from(conversationId).toString("base64url")}:${Buffer.from(emeaServiceUrl).toString("base64url")}`, id: legacyEmeaThreadId, name: "Stable Teams channel", }); // A redelivery that straddles an adapter rollout changes only the thread // serialization. Its canonical delivery identity must still deduplicate. await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: legacyEmeaThread.thread, message: makeMessage({ id: "1740000000200", raw: { serviceUrl: emeaServiceUrl, from: { aadObjectId: "stable-teams-user" }, }, text: "continue after Microsoft moved the reply route", }), trigger: "subscribed_message", }); const conversations = await service.listConversations(endpoint.id); expect(conversations).toHaveLength(1); expect(conversations[0]).toMatchObject({ externalThreadId: legacyThreadId, sessionGeneration: 1, }); await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, conversations[0]!.issueId)) .orderBy(asc(issueComments.createdAt), asc(issueComments.id)), ).resolves.toEqual([ { body: "@maya preserve this task across route changes" }, { body: "continue after Microsoft moved the reply route" }, ]); expect(wakeup).toHaveBeenCalledTimes(2); expect( runtime.endpoints.get(endpoint.id)?.recordedMicrosoftTeamsRoutes, ).toEqual([ { threadId: legacyThreadId, serviceUrl: amerServiceUrl }, { threadId: canonicalThreadId, serviceUrl: emeaServiceUrl }, { threadId: legacyEmeaThreadId, serviceUrl: emeaServiceUrl }, ]); // Also cover the reverse rolling-upgrade direction: a canonical row must // accept a legacy callback without creating another task. await db .update(chatConversations) .set({ externalThreadId: canonicalThreadId, updatedAt: new Date() }) .where(eq(chatConversations.id, conversations[0]!.id)); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: legacyEmeaThread.thread, message: makeMessage({ id: "1740000000201", raw: { serviceUrl: emeaServiceUrl, from: { aadObjectId: "stable-teams-user" }, }, text: "continue from a rolling legacy process", }), trigger: "subscribed_message", }); const afterReverseRollout = await service.listConversations(endpoint.id); expect(afterReverseRollout).toHaveLength(1); expect(afterReverseRollout[0]).toMatchObject({ externalThreadId: canonicalThreadId, issueId: conversations[0]!.issueId, }); await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, conversations[0]!.issueId)) .orderBy(asc(issueComments.createdAt), asc(issueComments.id)), ).resolves.toEqual([ { body: "@maya preserve this task across route changes" }, { body: "continue after Microsoft moved the reply route" }, { body: "continue from a rolling legacy process" }, ]); expect(wakeup).toHaveBeenCalledTimes(3); }); it("presents a fresh Teams personal generation after close and a regional route change", async () => { const fixture = await seedCompany(); const f = await configuredTeamsEndpoint(fixture); try { const channel = "a:post-close-regional-personal"; const rootMessageId = "1740000000299"; const rooted = channel; const channelBase = `teams:${Buffer.from(channel).toString("base64url")}`; const threadBase = `teams:${Buffer.from(rooted).toString("base64url")}`; const oldRoute = Buffer.from( "https://smba.trafficmanager.net/amer/", ).toString("base64url"); const oldThread = makeThread({ id: `${threadBase}:${oldRoute}`, channelId: `${channelBase}:${oldRoute}`, isDM: true, name: "Regional post-close proof", }).thread; const currentThread = makeThread({ id: threadBase, channelId: channelBase, isDM: true, name: "Regional post-close proof", }).thread; const controlTime = new Date(Date.now() - 10_000); const receive = async ( thread: Thread, id: string, time: Date, text: string, ) => { await deliverMessage({ callbacks: f.callbacks, endpointId: f.endpoint.id, provider: "microsoft-teams", thread, trigger: "direct_message", message: makeMessage({ id, text, mentioned: true, userId: "teams-regional-author", raw: { timestamp: time.toISOString(), serviceUrl: thread.id === oldThread.id ? "https://smba.trafficmanager.net/amer/" : "https://smba.trafficmanager.net/emea/", from: { aadObjectId: "teams-regional-author" }, }, }), }); return db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, f.endpoint.id), sql`${chatDeliveries.normalizedEvent}->'message'->>'providerMessageId' = ${id}`, ), ) .then((rows) => rows[0]!); }; await receive( oldThread, rootMessageId, new Date(controlTime.getTime() - 1_000), "@maya begin the stable channel task", ); await qualifySetupRoundTrip( f.service, f.endpoint.id, "teams-regional-author", ); await f.service.test(f.endpoint.id, "owner-user"); const [original] = await f.service.listConversations(f.endpoint.id); await receive(oldThread, "teams-regional-close", controlTime, "/close"); await f.service.processPendingPublications(); expect((await f.service.listConversations(f.endpoint.id))[0]?.state).toBe( "completed", ); const delivery = await receive( currentThread, "teams-regional-fresh", new Date(), "@maya answer this fresh request on the current route", ); expect(delivery.state).toBe("processed"); expect(delivery.conversationId).not.toBe(original.id); const current = (await f.service.listConversations(f.endpoint.id)).find( (row) => row.id === delivery.conversationId, )!; expect(current).toMatchObject({ state: "active", externalThreadId: currentThread.id, sessionGeneration: original.sessionGeneration + 1, }); expect(current.issueId).not.toBe(original.issueId); const context = await chatWakeContext({ endpointId: f.endpoint.id, issueId: current.issueId, provider: "microsoft-teams", providerMessageId: "teams-regional-fresh", }); const [action] = await db .select() .from(chatActions) .where( and( eq(chatActions.deliveryId, delivery.id), eq(chatActions.kind, "inbound_wakeup"), ), ); expect(action?.status).toBe("processed"); const runId = randomUUID(); await db .insert(heartbeatRuns) .values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, nativeIssueId: current.issueId, runtimeMode: "native", status: "running", startedAt: new Date(), wakeupRequestId: action!.id, contextSnapshot: { ...context, endpointId: f.endpoint.id, paperclipHarnessCheckedOut: true, }, }); await db .update(agentWakeupRequests) .set({ runId, status: "claimed" }) .where(eq(agentWakeupRequests.id, action!.id)); await expect( resolveChatRunPresentationAuthorizationReason(db, { companyId: fixture.companyId, issueId: current.issueId, runId, }), ).resolves.toBe("allow_chat_run_presentation"); } finally { await retirePublicationFixture(f.service, f.endpoint.id); } }); it("applies the Teams direct-message reach toggle and starts a new generation after task completion", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredTeamsEndpoint(fixture); const directThread = makeThread({ channelId: "teams-personal-reach", id: "teams:personal-reach", isDM: true, name: "Teams personal reach", }); const providerTimestamp = "2026-09-05T18:15:00.000Z"; const send = (id: string, text: string) => deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: directThread.thread, message: makeMessage({ id, raw: { from: { aadObjectId: "6c4dd0ef-f027-4b75-93d9-04d97424220e" }, timestamp: providerTimestamp, }, text, userId: "29:teams-direct-reach", }), trigger: "direct_message", }); await service.update( endpoint.id, { allowDirectMessages: false }, "owner-user", ); await send("teams-direct-disabled", "This must remain outside Paperclip"); expect(await service.listConversations(endpoint.id)).toEqual([]); expect(wakeup).not.toHaveBeenCalled(); await expect( db .select({ state: chatDeliveries.state, redactedError: chatDeliveries.redactedError, normalizedEvent: chatDeliveries.normalizedEvent, principalId: chatDeliveries.principalId, }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)), ).resolves.toEqual([ { state: "filtered", redactedError: "Destination is not enabled in Paperclip", normalizedEvent: expect.objectContaining({ providerEventId: `${directThread.thread.id}:teams-direct-disabled`, kind: "direct_message", trigger: "direct_message", resource: { type: "direct_message", providerResourceId: "teams-personal-reach", }, conversation: { externalThreadId: directThread.thread.id }, message: { providerMessageId: "teams-direct-disabled", providerSentAt: providerTimestamp, }, filtering: { contentRetained: false }, }), principalId: null, }, ]); const disabledDelivery = await db .select({ normalizedEvent: chatDeliveries.normalizedEvent }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)) .then((rows) => rows[0]); expect(JSON.stringify(disabledDelivery)).not.toContain( "This must remain outside Paperclip", ); expect(JSON.stringify(disabledDelivery)).not.toContain( "6c4dd0ef-f027-4b75-93d9-04d97424220e", ); await expect( db .select({ id: chatExternalPrincipals.id }) .from(chatExternalPrincipals) .where(eq(chatExternalPrincipals.companyId, fixture.companyId)), ).resolves.toHaveLength(0); await service.update( endpoint.id, { allowDirectMessages: true }, "owner-user", ); await send("teams-direct-first", "Start the first direct-message task"); const [firstConversation] = await service.listConversations(endpoint.id); expect(firstConversation).toMatchObject({ isDirectMessage: true, sessionGeneration: 1, state: "active", }); await db .update(issues) .set({ status: "done", completedAt: new Date(), updatedAt: new Date() }) .where(eq(issues.id, firstConversation!.issueId)); await send("teams-direct-second", "Start a fresh task after completion"); const conversations = await service.listConversations(endpoint.id); expect(conversations).toHaveLength(2); expect( conversations.find( (conversation) => conversation.id === firstConversation!.id, ), ).toMatchObject({ issueId: firstConversation!.issueId, sessionGeneration: 1, state: "completed", }); const nextConversation = conversations.find( (conversation) => conversation.id !== firstConversation!.id, ); expect(nextConversation).toMatchObject({ sessionGeneration: 2, state: "active", }); expect(nextConversation!.issueId).not.toBe(firstConversation!.issueId); expect( new Set(conversations.map((conversation) => conversation.issueId)).size, ).toBe(2); expect(wakeup).toHaveBeenCalledTimes(2); }); it("uses the Teams group-chat toggle without weakening channel resource gates", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service } = await configuredTeamsEndpoint(fixture); await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, endpoint.id)); await service.update(endpoint.id, { allowGroupChats: false }, "owner-user"); const serviceUrl = "https://smba.trafficmanager.net/amer/"; const groupConversationId = "19:teams-group-reach@unq.gbl.spaces"; await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "group_chat", providerResourceId: groupConversationId, label: "Launch group", availability: "available", enabled: false, }); const groupThread = makeThread({ channelId: `teams:${Buffer.from(groupConversationId).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, id: `teams:${Buffer.from(groupConversationId).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, name: "Launch group", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: groupThread.thread, message: makeMessage({ id: "teams-group-disabled", text: "@maya do not start yet", mentioned: true, }), trigger: "mention", }); await expect(service.listConversations(endpoint.id)).resolves.toHaveLength( 0, ); await service.update(endpoint.id, { allowGroupChats: true }, "owner-user"); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: groupThread.thread, message: makeMessage({ id: "teams-group-enabled", text: "@maya start the group task", mentioned: true, }), trigger: "mention", }); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ externalConversationId: groupThread.thread.channelId, isDirectMessage: false, }), ]); await expect(service.listResources(endpoint.id)).resolves.toEqual( expect.arrayContaining([ expect.objectContaining({ providerResourceId: groupConversationId, type: "group_chat", enabled: false, availability: "available", }), ]), ); const channelConversationId = "19:teams-channel-reach@thread.tacv2"; const channelRootId = "1740000000101"; const [channelResource] = await db .insert(chatEndpointResources) .values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: channelConversationId, label: "Engineering", availability: "available", enabled: false, }) .returning(); const channelThread = makeThread({ channelId: `teams:${Buffer.from(channelConversationId).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, id: `teams:${Buffer.from(`${channelConversationId};messageid=${channelRootId}`).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, name: "Engineering", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: channelThread.thread, message: makeMessage({ id: channelRootId, text: "@maya channel still requires enablement", mentioned: true, userId: "U-TEAMS-DISABLED-CHANNEL", userName: "disabled-channel-user", }), trigger: "mention", }); await expect(service.listConversations(endpoint.id)).resolves.toHaveLength( 1, ); const disabledChannelDelivery = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, `teams:${Buffer.from(`${channelConversationId};messageid=${channelRootId}`).toString("base64url")}:${channelRootId}`, ), ), ) .then((rows) => rows[0] ?? null); expect(disabledChannelDelivery).toMatchObject({ state: "filtered", attempts: 0, principalId: null, normalizedEvent: { filtering: { contentRetained: false }, message: { providerMessageId: channelRootId }, }, }); expect( JSON.stringify(disabledChannelDelivery?.normalizedEvent), ).not.toContain("channel still requires enablement"); await expect( db .select({ id: chatExternalPrincipals.id }) .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, fixture.companyId), eq(chatExternalPrincipals.provider, "microsoft-teams"), eq(chatExternalPrincipals.externalId, "U-TEAMS-DISABLED-CHANNEL"), ), ), ).resolves.toHaveLength(0); await service.replaceResources(endpoint.id, [ { id: channelResource!.id, enabled: true }, ]); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: channelThread.thread, message: makeMessage({ id: `${channelRootId}-enabled`, text: "@maya channel is enabled now", mentioned: true, userId: "U-TEAMS-DISABLED-CHANNEL", userName: "disabled-channel-user", }), trigger: "mention", }); await expect(service.listConversations(endpoint.id)).resolves.toHaveLength( 2, ); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: channelThread.thread, message: makeMessage({ id: `${channelRootId}-reply`, text: "continue the now-authorized Teams task", userId: "U-TEAMS-DISABLED-CHANNEL", userName: "disabled-channel-user", }), trigger: "subscribed_message", }); const channelConversation = ( await service.listConversations(endpoint.id) ).find( (conversation) => conversation.externalThreadId === channelThread.thread.id, ); if (!channelConversation) throw new Error("Expected enabled Teams channel conversation"); await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, channelConversation.issueId)) .orderBy(asc(issueComments.createdAt), asc(issueComments.id)), ).resolves.toEqual([ { body: "@maya channel is enabled now" }, { body: "continue the now-authorized Teams task" }, ]); await expect( db .select({ id: chatExternalPrincipals.id }) .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, fixture.companyId), eq(chatExternalPrincipals.provider, "microsoft-teams"), eq(chatExternalPrincipals.externalId, "U-TEAMS-DISABLED-CHANNEL"), ), ), ).resolves.toHaveLength(1); }); it("atomically rejects Teams messages when DM, group, or channel reach is revoked at the task-mutation boundary", async () => { const serviceUrl = "https://smba.trafficmanager.net/amer/"; const scenarios = [ { kind: "direct_message" as const, label: "direct" }, { kind: "group_chat" as const, label: "group" }, { kind: "channel" as const, label: "channel" }, ]; for (const scenario of scenarios) { const fixture = await seedCompany(); let releaseReachCheck!: () => void; let signalReachCheck!: () => void; const reachCheckEntered = new Promise((resolve) => { signalReachCheck = resolve; }); const reachCheckReleased = new Promise((resolve) => { releaseReachCheck = resolve; }); const { callbacks, endpoint, service, wakeup } = await configuredTeamsEndpoint(fixture, { reachAuthorizationBarrier: async () => { signalReachCheck(); await reachCheckReleased; }, }); const conversationId = scenario.kind === "group_chat" ? `19:teams-revoke-${scenario.label}@unq.gbl.spaces` : scenario.kind === "channel" ? `19:teams-revoke-${scenario.label}@thread.tacv2` : `teams-revoke-${scenario.label}`; const rootMessageId = `teams-revoke-${scenario.label}-message`; const encodedConversationId = Buffer.from(conversationId).toString("base64url"); const encodedServiceUrl = Buffer.from(serviceUrl).toString("base64url"); const thread = makeThread({ channelId: scenario.kind === "direct_message" ? conversationId : `teams:${encodedConversationId}:${encodedServiceUrl}`, id: scenario.kind === "channel" ? `teams:${Buffer.from(`${conversationId};messageid=${rootMessageId}`).toString("base64url")}:${encodedServiceUrl}` : scenario.kind === "group_chat" ? `teams:${encodedConversationId}:${encodedServiceUrl}` : `teams:${conversationId}`, isDM: scenario.kind === "direct_message", name: `Revoked ${scenario.label}`, }); const durableThreadId = scenario.kind === "channel" ? `teams:${Buffer.from(`${conversationId};messageid=${rootMessageId}`).toString("base64url")}` : scenario.kind === "group_chat" ? `teams:${encodedConversationId}` : thread.thread.id; let resourceId: string | null = null; if (scenario.kind !== "direct_message") { const [resource] = await db .insert(chatEndpointResources) .values({ companyId: fixture.companyId, endpointId: endpoint.id, type: scenario.kind, providerResourceId: conversationId, label: `Revoked ${scenario.label}`, availability: "available", enabled: true, }) .returning({ id: chatEndpointResources.id }); resourceId = resource!.id; } if (scenario.kind === "group_chat") { await service.update( endpoint.id, { allowGroupChats: true }, "owner-user", ); } const externalPrincipalId = randomUUID(); const secretText = `must-not-persist-${scenario.label}`; const send = deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: thread.thread, message: makeMessage({ id: rootMessageId, raw: { from: { aadObjectId: externalPrincipalId } }, text: `@maya ${secretText}`, mentioned: scenario.kind !== "direct_message", userId: `29:teams-revoke-${scenario.label}`, }), trigger: scenario.kind === "direct_message" ? "direct_message" : "mention", }); await reachCheckEntered; if (scenario.kind === "direct_message") { await service.update( endpoint.id, { allowDirectMessages: false }, "owner-user", ); } else if (scenario.kind === "group_chat") { await service.update( endpoint.id, { allowGroupChats: false }, "owner-user", ); } else { await service.replaceResources(endpoint.id, [ { id: resourceId!, enabled: false }, ]); } releaseReachCheck(); await send; await expect(service.listConversations(endpoint.id)).resolves.toEqual([]); await expect( db .select({ id: issues.id }) .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).resolves.toEqual([]); const [delivery] = await db .select({ state: chatDeliveries.state, normalizedEvent: chatDeliveries.normalizedEvent, principalId: chatDeliveries.principalId, redactedError: chatDeliveries.redactedError, }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, `${durableThreadId}:${rootMessageId}`, ), ), ); expect(delivery).toMatchObject({ state: "filtered", principalId: null, redactedError: "Destination is not enabled in Paperclip", normalizedEvent: { filtering: { contentRetained: false }, message: { providerMessageId: rootMessageId }, }, }); expect(JSON.stringify(delivery?.normalizedEvent)).not.toContain( secretText, ); expect(JSON.stringify(delivery?.normalizedEvent)).not.toContain( externalPrincipalId, ); await expect( db .select({ id: chatExternalPrincipals.id }) .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, fixture.companyId), eq(chatExternalPrincipals.provider, "microsoft-teams"), eq(chatExternalPrincipals.externalId, externalPrincipalId), ), ), ).resolves.toEqual([]); expect(wakeup).not.toHaveBeenCalled(); } }); it("atomically rejects a Slack DM when reach is revoked after initial admission", async () => { const fixture = await seedCompany(); let releaseReachCheck!: () => void; let signalReachCheck!: () => void; const reachCheckEntered = new Promise((resolve) => { signalReachCheck = resolve; }); const reachCheckReleased = new Promise((resolve) => { releaseReachCheck = resolve; }); const { callbacks, endpoint, service, wakeup } = await configuredSlackEndpoint(fixture, { allowUnlinkedPeople: true, reachAuthorizationBarrier: async () => { signalReachCheck(); await reachCheckReleased; }, }); const thread = makeThread({ channelId: "slack:D-REVOKED-REACH", id: "slack:D-REVOKED-REACH", isDM: true, name: "Revoked Slack DM", }); const messageId = "slack-revoked-reach-message"; const externalPrincipalId = "U-SLACK-REVOKED-REACH"; const secretText = "must-not-persist-slack-race"; const send = deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: messageId, text: secretText, userId: externalPrincipalId, }), trigger: "direct_message", }); await reachCheckEntered; await service.update( endpoint.id, { allowDirectMessages: false }, "owner-user", ); releaseReachCheck(); await send; await expect(service.listConversations(endpoint.id)).resolves.toEqual([]); await expect( db .select({ id: issues.id }) .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).resolves.toEqual([]); const [delivery] = await db .select({ state: chatDeliveries.state, normalizedEvent: chatDeliveries.normalizedEvent, principalId: chatDeliveries.principalId, redactedError: chatDeliveries.redactedError, }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, `${thread.thread.id}:${messageId}`, ), ), ); expect(delivery).toMatchObject({ state: "filtered", principalId: null, redactedError: "Destination is not enabled in Paperclip", normalizedEvent: { filtering: { contentRetained: false }, message: { providerMessageId: messageId }, }, }); expect(JSON.stringify(delivery?.normalizedEvent)).not.toContain(secretText); expect(JSON.stringify(delivery?.normalizedEvent)).not.toContain( externalPrincipalId, ); await expect( db .select({ id: chatExternalPrincipals.id }) .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, fixture.companyId), eq(chatExternalPrincipals.provider, "slack"), eq(chatExternalPrincipals.externalId, externalPrincipalId), ), ), ).resolves.toEqual([]); expect(wakeup).not.toHaveBeenCalled(); }); it("atomically rejects Slack task mutation after a linked identity is revoked or loses write membership", async () => { for (const authorizationChange of ["link_revoked", "viewer"] as const) { const fixture = await seedCompany(); let releaseAuthorizationCheck!: () => void; let signalAuthorizationCheck!: () => void; const authorizationCheckEntered = new Promise((resolve) => { signalAuthorizationCheck = resolve; }); const authorizationCheckReleased = new Promise((resolve) => { releaseAuthorizationCheck = resolve; }); const { callbacks, endpoint, service, wakeup } = await configuredSlackEndpoint(fixture, { allowUnlinkedPeople: false, reachAuthorizationBarrier: async () => { signalAuthorizationCheck(); await authorizationCheckReleased; }, }); const currentEndpoint = await service.get(endpoint.id); if (!currentEndpoint.providerAccountId) { throw new Error("Expected configured Slack account identity"); } const externalPrincipalId = `U-SLACK-AUTH-${authorizationChange}`; const [principal] = await db .insert(chatExternalPrincipals) .values({ companyId: fixture.companyId, provider: "slack", providerAccountId: currentEndpoint.providerAccountId, externalId: externalPrincipalId, kind: "user", displayName: "Linked Slack User", handle: "linked-slack-user", isBot: false, }) .returning(); await db.insert(chatIdentityLinks).values({ companyId: fixture.companyId, endpointId: endpoint.id, principalId: principal.id, paperclipUserId: "owner-user", status: "linked", confirmedAt: new Date(), }); const thread = makeThread({ channelId: `D-SLACK-AUTH-${authorizationChange}`, id: `slack:D-SLACK-AUTH-${authorizationChange}`, isDM: true, name: "Slack authorization race", }); const messageId = `slack-auth-${authorizationChange}`; const secretText = `must-not-persist-${authorizationChange}`; const send = deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: messageId, text: secretText, userId: externalPrincipalId, }), trigger: "direct_message", }); await authorizationCheckEntered; if (authorizationChange === "link_revoked") { await db .update(chatIdentityLinks) .set({ paperclipUserId: null, status: "revoked", revokedAt: new Date(), updatedAt: new Date(), }) .where(eq(chatIdentityLinks.principalId, principal.id)); } else { await db .update(companyMemberships) .set({ membershipRole: "viewer", updatedAt: new Date() }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalType, "user"), eq(companyMemberships.principalId, "owner-user"), ), ); } releaseAuthorizationCheck(); await send; await expect(service.listConversations(endpoint.id)).resolves.toEqual([]); await expect( db .select({ id: issues.id }) .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).resolves.toEqual([]); const [delivery] = await db .select({ normalizedEvent: chatDeliveries.normalizedEvent, principalId: chatDeliveries.principalId, redactedError: chatDeliveries.redactedError, state: chatDeliveries.state, }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, `${thread.thread.id}:${messageId}`, ), ), ); expect(delivery).toMatchObject({ principalId: null, state: "filtered", redactedError: authorizationChange === "viewer" ? "Linked Paperclip account is not currently permitted" : "External identity must be linked to a Paperclip account", normalizedEvent: { filtering: { contentRetained: false }, message: { providerMessageId: messageId }, }, }); expect(JSON.stringify(delivery?.normalizedEvent)).not.toContain( secretText, ); expect(wakeup).not.toHaveBeenCalled(); } }); it("atomically rejects a sponsored Telegram guest after the sponsor is suspended", async () => { const fixture = await seedCompany(); let releaseAuthorizationCheck!: () => void; let signalAuthorizationCheck!: () => void; const authorizationCheckEntered = new Promise((resolve) => { signalAuthorizationCheck = resolve; }); const authorizationCheckReleased = new Promise((resolve) => { releaseAuthorizationCheck = resolve; }); const { callbacks, endpoint, service, wakeup } = await configuredTelegramEndpoint(fixture, { allowUnlinkedPeople: true, reachAuthorizationBarrier: async () => { signalAuthorizationCheck(); await authorizationCheckReleased; }, }); const chatId = "77118898"; const thread = makeThread({ channelId: `telegram:${chatId}`, id: `telegram:${chatId}`, isDM: true, name: "Suspended sponsor race", }); const messageId = `${chatId}:93`; const secretText = "must-not-persist-suspended-sponsor"; const send = deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: thread.thread, message: makeMessage({ id: messageId, text: secretText, userId: chatId, }), trigger: "direct_message", }); await authorizationCheckEntered; await db .update(companyMemberships) .set({ status: "suspended", updatedAt: new Date() }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalType, "user"), eq(companyMemberships.principalId, "owner-user"), ), ); releaseAuthorizationCheck(); await send; await expect(service.listConversations(endpoint.id)).resolves.toEqual([]); await expect( db .select({ id: issues.id }) .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).resolves.toEqual([]); const [delivery] = await db .select({ normalizedEvent: chatDeliveries.normalizedEvent, principalId: chatDeliveries.principalId, redactedError: chatDeliveries.redactedError, state: chatDeliveries.state, }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, `${thread.thread.id}:${messageId}`, ), ), ); expect(delivery).toMatchObject({ principalId: null, state: "filtered", redactedError: "Endpoint sponsor can no longer authorize external guests", normalizedEvent: { filtering: { contentRetained: false }, message: { providerMessageId: messageId }, }, }); expect(JSON.stringify(delivery?.normalizedEvent)).not.toContain(secretText); expect(wakeup).not.toHaveBeenCalled(); }); it("atomically rejects a Telegram DM when reach is revoked at the final task-mutation boundary", async () => { const fixture = await seedCompany(); let releaseReachCheck!: () => void; let signalReachCheck!: () => void; const reachCheckEntered = new Promise((resolve) => { signalReachCheck = resolve; }); const reachCheckReleased = new Promise((resolve) => { releaseReachCheck = resolve; }); const { callbacks, endpoint, service, wakeup } = await configuredTelegramEndpoint(fixture, { allowUnlinkedPeople: true, reachAuthorizationBarrier: async () => { signalReachCheck(); await reachCheckReleased; }, }); const chatId = "77118899"; const thread = makeThread({ channelId: `telegram:${chatId}`, id: `telegram:${chatId}`, isDM: true, name: "Revoked Telegram DM", }); const messageId = `${chatId}:92`; const secretText = "must-not-persist-telegram-race"; const send = deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: thread.thread, message: makeMessage({ id: messageId, text: secretText, userId: chatId, }), trigger: "direct_message", }); await reachCheckEntered; await service.update( endpoint.id, { allowDirectMessages: false }, "owner-user", ); releaseReachCheck(); await send; await expect(service.listConversations(endpoint.id)).resolves.toEqual([]); await expect( db .select({ id: issues.id }) .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).resolves.toEqual([]); const [delivery] = await db .select({ state: chatDeliveries.state, normalizedEvent: chatDeliveries.normalizedEvent, principalId: chatDeliveries.principalId, redactedError: chatDeliveries.redactedError, }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, `${thread.thread.id}:${messageId}`, ), ), ); expect(delivery).toMatchObject({ state: "filtered", principalId: null, redactedError: "Destination is not enabled in Paperclip", normalizedEvent: { filtering: { contentRetained: false }, message: { providerMessageId: messageId }, }, }); expect(JSON.stringify(delivery?.normalizedEvent)).not.toContain(secretText); await expect( db .select({ id: chatExternalPrincipals.id }) .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, fixture.companyId), eq(chatExternalPrincipals.provider, "telegram"), eq(chatExternalPrincipals.externalId, chatId), ), ), ).resolves.toEqual([]); expect(wakeup).not.toHaveBeenCalled(); }); it("audits Microsoft Teams reactions idempotently without treating them as task instructions", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredTeamsEndpoint(fixture); const aadObjectId = "860def28-0dab-44ae-b8cf-30e168181a15"; const serviceUrl = "https://smba.trafficmanager.net/amer/"; const thread = makeThread({ channelId: "teams-personal-reactions", id: "teams:personal-reactions:root-1", isDM: true, name: "Teams personal reactions", }); const original = makeMessage({ id: "teams-reaction-root", raw: { from: { aadObjectId } }, text: "Observe Teams reactions", userId: "29:teams-reaction-session", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: thread.thread, message: original, trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, original.author.userId); await service.test(endpoint.id, "owner-user"); if (!callbacks.onReaction) throw new Error("Teams reaction callback was not registered"); const endpointRuntime = runtime.endpoints.get(endpoint.id); if (!endpointRuntime) throw new Error("Expected Teams endpoint runtime"); const initialRouteCount = endpointRuntime.recordedMicrosoftTeamsRoutes.length; const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const commentCount = await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation!.issueId)) .then((rows) => rows.length); const wakeupCount = wakeup.mock.calls.length; const emoji = { name: "thumbs_up", toJSON: () => "like", toString: () => "like", }; const reaction = ( added: boolean, activityId: string, reactionThread = thread.thread, ) => ({ endpointId: endpoint.id, provider: "microsoft-teams" as const, event: { adapter: {} as never, added, emoji, message: original, messageId: original.id, raw: { id: activityId, serviceUrl, from: { aadObjectId } }, rawEmoji: "like", thread: reactionThread, threadId: reactionThread.id, user: original.author, }, }); const unknownThread = makeThread({ channelId: "teams-personal-reactions-unknown", id: "teams:personal-reactions:unknown-root", isDM: true, name: "Unknown Teams personal reactions", }); await callbacks.onReaction( reaction(true, "teams-reaction-denied", unknownThread.thread), ); expect(endpointRuntime.recordedMicrosoftTeamsRoutes).toHaveLength( initialRouteCount, ); await callbacks.onReaction(reaction(true, "teams-reaction-1")); await callbacks.onReaction(reaction(true, "teams-reaction-1")); await callbacks.onReaction(reaction(false, "teams-reaction-2")); const reactions = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.conversationId, conversation!.id)) .then((rows) => rows.filter((row) => row.eventKind.startsWith("reaction_")), ); expect(reactions).toHaveLength(2); expect(reactions.map((row) => row.eventKind).sort()).toEqual([ "reaction_added", "reaction_removed", ]); expect(reactions.every((row) => row.principalId !== null)).toBe(true); expect( await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation!.issueId)) .then((rows) => rows.length), ).toBe(commentCount); expect(wakeup).toHaveBeenCalledTimes(wakeupCount); expect(endpointRuntime.recordedMicrosoftTeamsRoutes.length).toBeGreaterThan( initialRouteCount, ); expect( (await service.listActivity(endpoint.id)).filter((item) => item.summary.startsWith("reaction "), ), ).toHaveLength(2); }); it("keeps case-variant Teams Entra object ids on one external principal", async () => { const fixture = await seedCompany(); const { callbacks, endpoint } = await configuredTeamsEndpoint(fixture); const aadObjectId = "76d0cb17-5ec4-4b3d-983b-da8a01dc02c4"; const thread = makeThread({ channelId: "teams-personal-identity", id: "teams:personal-identity:root-1", isDM: true, name: "Alex External", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: thread.thread, message: makeMessage({ id: "teams-identity-root", raw: { from: { aadObjectId: aadObjectId.toUpperCase() } }, text: "Start a Teams identity task", userId: "29:adapter-session-one", }), trigger: "direct_message", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: thread.thread, message: makeMessage({ id: "teams-identity-follow-up", raw: { from: { aadObjectId } }, text: "Continue from another adapter session", userId: "29:adapter-session-two", }), trigger: "direct_message", }); const principals = await db .select({ externalId: chatExternalPrincipals.externalId }) .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, fixture.companyId), eq(chatExternalPrincipals.provider, "microsoft-teams"), ), ); expect(principals).toEqual([{ externalId: aadObjectId }]); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, conversation!.issueId)), ).resolves.toEqual( expect.arrayContaining([ { body: "Start a Teams identity task" }, { body: "Continue from another adapter session" }, ]), ); }); describe("Teams inline picture publication", () => { it.each([ { surface: "channel", outcome: "native" }, { surface: "group", outcome: "native" }, { surface: "channel", outcome: "native_http" }, { surface: "group", outcome: "native_http" }, { surface: "channel", outcome: "long_native_http" }, { surface: "channel", outcome: "malformed" }, { surface: "channel", outcome: "too_large" }, { surface: "channel", outcome: "dimensions" }, { surface: "channel", outcome: "source_deleted" }, { surface: "group", outcome: "reach_revoked" }, { surface: "channel", outcome: "lost_receipt" }, { surface: "channel", outcome: "empty_receipt" }, ] as const)( "keeps $surface picture delivery truthful for $outcome", async ({ surface, outcome }) => { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, runtime, service } = await configuredTeamsEndpoint(fixture, { storage: storage.storage }); let pinned: ReturnType | undefined; try { if (surface === "group") { await service.update( endpoint.id, { allowGroupChats: true }, "owner-user", ); } const serviceUrl = "https://smba.trafficmanager.net/amer/"; const conversationId = `19:inline-picture-${surface}-${randomUUID()}@thread.${surface === "channel" ? "tacv2" : "v2"}`; const rootId = "1740000000491"; const providerThread = makeThread({ channelId: `teams:${Buffer.from(conversationId).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, id: `teams:${Buffer.from(surface === "channel" ? `${conversationId};messageid=${rootId}` : conversationId).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, name: "Teams picture qualification", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: providerThread.thread, message: makeMessage({ id: rootId, text: "@Maya inspect pictures here", mentioned: true, }), trigger: "mention", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); expect(conversation).toBeDefined(); const body = outcome === "malformed" ? Buffer.from("not a PNG") : outcome === "too_large" ? Buffer.alloc(1_000_001) : await sharp({ create: { width: outcome === "dimensions" ? 1025 : 2, height: 2, channels: 4, background: { r: 50, g: 100, b: 150, alpha: 1 }, }, }) .png() .toBuffer(); const stored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${conversation!.issueId}`, originalFilename: "teams-picture.png", contentType: "image/png", body, }); const attachment = await issueService(db).createAttachment({ issueId: conversation!.issueId, ...stored, createdByUserId: "owner-user", }); await service.processPendingPublications(); const providerRuntime = runtime.endpoints.get(endpoint.id)!; const httpBodies: Array<{ url: string; body: unknown }> = []; const boardText = outcome === "long_native_http" ? "x".repeat(99_996) + "TAIL" : "Share this picture."; if (outcome === "native_http" || outcome === "long_native_http") { pinned = createChatSdkEndpointRuntime({ ...runtime.configurations.get(endpoint.id)!, logger: "silent", callbacks: { onMessage() {} }, }); await pinned.initialize(); const app = ( pinned.getProviderAdapter() as unknown as { app: { activitySender: { client: { post( url: string, body: unknown, ): Promise<{ data: unknown }>; }; }; }; } ).app; vi.spyOn(app.activitySender.client, "post").mockImplementation( async (url, body) => { httpBodies.push({ url, body }); return { data: { id: `teams-http-receipt-${httpBodies.length}` }, }; }, ); } const originalThread = providerRuntime.thread.bind(providerRuntime); let pictureAttempts = 0; let textSent = false; vi.spyOn(providerRuntime, "thread").mockImplementation((id) => { const target = originalThread(id); return { ...target, post: async (message: unknown) => { const sent = await target.post(message); const hasFiles = Boolean( message && typeof message === "object" && "files" in message, ); if (hasFiles) { pictureAttempts++; if (outcome === "lost_receipt") throw new Error( "Simulated picture response lost after acceptance", ); if (outcome === "empty_receipt") return { ...sent, id: "" }; } else if (!textSent) { textSent = true; // Mutate after the Board text's provider effect, before the // separately ordered picture row claims its own authority. if (outcome === "source_deleted") { await db .delete(issueAttachments) .where(eq(issueAttachments.id, attachment.id)); } if (outcome === "reach_revoked") { await db .update(chatEndpointResources) .set({ availability: "unavailable" }) .where( eq(chatEndpointResources.id, conversation!.resourceId!), ); } } return pinned ? await pinned .thread(id) .post( message as Parameters< ReturnType["post"] >[0], ) : sent; }, }; }); const result = await service.publishBoardMessage( endpoint.id, conversation!.id, boardText, `teams-inline-picture-${surface}`, "owner-user", [attachment.id], ); const denied = ["source_deleted", "reach_revoked"].includes(outcome); const ambiguous = ["lost_receipt", "empty_receipt"].includes(outcome); const fallback = ["malformed", "too_large", "dimensions"].includes( outcome, ); if (denied) expect(["failed", "cancelled"]).toContain(result.state); else expect(result.state).toBe( ambiguous ? "delivery_unknown" : "published", ); const imagePosts = ( runtime.endpoints.get(endpoint.id)?.posts ?? [] ).filter((post) => post.files?.length); expect(imagePosts).toHaveLength(denied || fallback ? 0 : 1); if (!denied && !fallback) { expect(imagePosts[0]!.threadId).toBe( conversation!.externalThreadId, ); expect(imagePosts[0]!.files).toEqual([ { data: body, filename: "teams-picture.png", mimeType: "image/png", }, ]); expect(imagePosts[0]!.text).not.toContain("isn't available"); } if (fallback) { expect( providerRuntime.posts.some((post) => post.text.includes("Direct file delivery isn't available"), ), ).toBe(true); if (outcome === "too_large") expect(storage.storage.getObject).not.toHaveBeenCalled(); } const [publication] = await db .select() .from(chatPublications) .where(eq(chatPublications.id, result.id)); if (!denied) expect(publication).toMatchObject({ state: ambiguous ? "delivery_unknown" : "published", attempts: 1, }); if (!denied && !ambiguous) expect(publication!.providerMessageId).toBeTruthy(); if (outcome === "native_http" || outcome === "long_native_http") { if (outcome === "native_http") expect(httpBodies).toHaveLength(2); else { expect(httpBodies.length).toBeGreaterThan(2); expect( providerRuntime.posts .slice(0, -1) .map((post) => post.text) .join(""), ).toBe(boardText); expect(providerRuntime.posts.at(-1)!.files).toHaveLength(1); } const pictureRequest = httpBodies.at(-1)!; expect(pictureRequest.url).toBe( `https://smba.trafficmanager.net/amer/v3/conversations/${surface === "channel" ? `${conversationId};messageid=${rootId}` : conversationId}/activities`, ); expect(pictureRequest.body).toMatchObject({ type: "message", attachments: [ { name: "teams-picture.png", contentType: "image/png", contentUrl: `data:image/png;base64,${body.toString("base64")}`, }, ], }); expect(JSON.stringify(pictureRequest.body)).not.toMatch( /file\.consent|teams-test-secret/, ); expect(publication!.providerMessageId).toBe( `teams-http-receipt-${httpBodies.length}`, ); } if (ambiguous) expect(publication!.providerMessageId).toBeNull(); await expect( db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.endpointId, endpoint.id)), ).resolves.toEqual([]); const sentCount = runtime.endpoints.get(endpoint.id)!.posts.length; await service.publishBoardMessage( endpoint.id, conversation!.id, boardText, `teams-inline-picture-${surface}`, "owner-user", [attachment.id], ); expect(runtime.endpoints.get(endpoint.id)!.posts).toHaveLength( sentCount, ); await service.processPendingPublications(); expect(providerRuntime.posts).toHaveLength(sentCount); expect(pictureAttempts).toBe(denied || fallback ? 0 : 1); } finally { await service.configure( endpoint.id, { action: "remove" }, "owner-user", ); await service.shutdown(); await pinned?.shutdown(); } }, ); }); it("links outbound Teams files instead of attempting an unsupported native upload", async () => { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, runtime, service } = await configuredTeamsEndpoint(fixture, { storage: storage.storage, }); const serviceUrl = "https://smba.trafficmanager.net/amer/"; const channelConversationId = "19:teams-outbound-files@thread.tacv2"; const channelMessageId = "1740000000291"; const channelThread = makeThread({ channelId: `teams:${Buffer.from(channelConversationId).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, id: `teams:${Buffer.from(`${channelConversationId};messageid=${channelMessageId}`).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, name: "Outbound files channel", }); const personalMessageId = "teams-outbound-personal-root"; const personalThread = makeThread({ channelId: "teams-personal-outbound-files", id: "teams:personal-outbound-files:root-1", isDM: true, name: "Alex External", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: channelThread.thread, message: makeMessage({ id: channelMessageId, text: "@Maya create a channel report", mentioned: true, }), trigger: "mention", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: personalThread.thread, message: makeMessage({ id: personalMessageId, text: "Create a personal report", }), trigger: "direct_message", }); const conversations = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const channelConversation = conversations.find( (conversation) => !conversation.isDirectMessage, ); const personalConversation = conversations.find( (conversation) => conversation.isDirectMessage, ); if (!channelConversation || !personalConversation) throw new Error("Expected Teams channel and personal tasks"); for (const item of [ { conversation: channelConversation, filename: "channel-report.txt", providerMessageId: channelMessageId, text: "Channel report ready.", }, { conversation: personalConversation, filename: "personal-report.txt", providerMessageId: personalMessageId, text: "Personal report ready.", }, ]) { const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: item.conversation.issueId, provider: "microsoft-teams", providerMessageId: item.providerMessageId, }), }); const comment = await addSelectedChatFinal({ agentId: fixture.assignedAgentId, body: item.text, companyId: fixture.companyId, issueId: item.conversation.issueId, runId, }); const stored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${item.conversation.issueId}`, originalFilename: item.filename, contentType: "text/plain", body: Buffer.from(`${item.filename} contents`, "utf8"), }); await issueService(db).createAttachment({ issueId: item.conversation.issueId, issueCommentId: comment.id, provider: stored.provider, objectKey: stored.objectKey, contentType: stored.contentType, byteSize: stored.byteSize, sha256: stored.sha256, originalFilename: stored.originalFilename, createdByAgentId: fixture.assignedAgentId, createdByRunId: runId, }); } await service.processPendingPublications(); const posts = runtime.endpoints.get(endpoint.id)?.posts ?? []; const channelAttachmentPost = posts.find((post) => post.text.includes( "File saved on the Paperclip task: channel-report.txt.", ), ); expect(channelAttachmentPost?.text).toContain( "Direct file delivery isn't available for this Teams conversation.", ); expect(channelAttachmentPost?.text).toContain( `/issues/${channelConversation.issueId}`, ); expect(channelAttachmentPost?.files).toBeUndefined(); const personalAttachmentPost = posts.find((post) => post.text.includes( "File saved on the Paperclip task: personal-report.txt.", ), ); expect(personalAttachmentPost?.text).toContain( "Direct file delivery isn't available for this Teams conversation.", ); expect(personalAttachmentPost?.text).toContain( `/issues/${personalConversation.issueId}`, ); expect(personalAttachmentPost?.files).toBeUndefined(); expect(storage.storage.getObject).not.toHaveBeenCalled(); }); describe("Teams authenticated inline-picture intake", () => { it.each([ ["channel", "current"], ["groupChat", "current"], ["channel", "restart"], ["groupChat", "restart"], ["channel", "revoked_after_receipt"], ["groupChat", "revoked_after_receipt"], ["groupChat", "revoked_during_download"], ["groupChat", "source_updated_during_download"], ["groupChat", "source_deleted_during_download"], ["groupChat", "batch_deadline"], ] as const)( "ingests an actual pinned-parser %s picture into its exact admitted turn (%s)", async (conversationType, mode) => { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredTeamsEndpoint(fixture, { storage: storage.storage, deferWebhookProcessing: mode === "restart" || mode === "revoked_after_receipt", scheduleDeferredWork: () => undefined, }); const configuration = runtime.configurations.get(endpoint.id)!; if (configuration.providerConfig.provider !== "microsoft-teams") throw new Error("Expected Teams configuration"); const pinned = createChatSdkEndpointRuntime({ ...configuration, logger: "silent", }); const image = await sharp({ create: { width: 2, height: 2, channels: 3, background: "#224466" }, }) .png() .toBuffer(); const url = "https://smba.trafficmanager.net/amer/v3/attachments/inline-fixture/views/original"; const http = ( pinned.getProviderAdapter() as unknown as { app: { api: { http: { get: (...args: unknown[]) => Promise } }; }; } ).app.api.http; const get = vi.spyOn(http, "get").mockResolvedValue({ data: image }); let restarted: ReturnType | undefined; let recoveredParser: ReturnType | undefined; let recoveredGet: ReturnType | undefined; let lifecycleCompletion: Promise | undefined; let timeoutSpy: ReturnType | undefined; try { if (conversationType === "groupChat") await service.update( endpoint.id, { allowGroupChats: true }, "owner-user", ); Object.assign(runtime.endpoints.get(endpoint.id)!, { attachmentRecoveryDescriptor: pinned.attachmentRecoveryDescriptor.bind(pinned), rehydrateAttachment: pinned.rehydrateAttachment.bind(pinned), fetchTeamsInlineImage: pinned.fetchTeamsInlineImage.bind(pinned), }); const message = pinned.parseMicrosoftTeamsMessage({ type: "message", channelId: "msteams", id: "1740000000771", timestamp: new Date().toISOString(), text: "Inspect this exact picture", serviceUrl: "https://smba.trafficmanager.net/amer/", from: { id: "29:inline-picture-user", aadObjectId: randomUUID(), name: "Picture User", }, recipient: { id: `28:${configuration.providerConfig.credentials.appId}`, }, conversation: { id: `19:inline-picture-${conversationType}@thread.tacv2${conversationType === "channel" ? ";messageid=1740000000771" : ""}`, conversationType, tenantId: configuration.providerConfig.credentials.appTenantId, }, attachments: [ { contentType: "image/png", contentUrl: url, name: "inline.png" }, ], })!; expect(message.attachments[0]).toMatchObject({ type: "image", mimeType: "image/png", fetchMetadata: { auth: "bot" }, }); const thread = makeThread({ id: message.threadId, channelId: message.threadId, isDM: false, }); if (mode === "batch_deadline") { const raw = message.raw as { attachments: Array>; }; raw.attachments.push({ contentType: "image/png", contentUrl: url.replace("inline-fixture", "inline-second"), name: "second.png", }); const second = pinned.parseMicrosoftTeamsMessage(raw)!.attachments[1]!; message.attachments.push(second); const budget = new AbortController(); const originalTimeout = AbortSignal.timeout.bind(AbortSignal); timeoutSpy = vi .spyOn(AbortSignal, "timeout") .mockImplementation((ms) => ms === 10_000 ? budget.signal : originalTimeout(ms), ); get.mockImplementation(async () => { budget.abort(); throw new Error("Synthetic exhausted image batch budget"); }); } if (mode === "revoked_during_download") { get.mockImplementation(async () => { await db .update(chatEndpoints) .set({ allowGroupChats: false }) .where(eq(chatEndpoints.id, endpoint.id)); return { data: image }; }); } if (mode.startsWith("source_")) { get.mockImplementation(async () => { const updated = pinned.parseMicrosoftTeamsMessage({ ...(message.raw as object), text: "Corrected source without that image", attachments: [], })!; lifecycleCompletion = Promise.resolve( mode === "source_updated_during_download" ? callbacks.onMessageUpdated!({ endpointId: endpoint.id, provider: "microsoft-teams", thread: thread.thread, message: updated, }) : callbacks.onMessageDeleted!({ endpointId: endpoint.id, provider: "microsoft-teams", event: { messageId: message.id, threadId: thread.thread.id, raw: message.raw, deletedAt: new Date(), }, }), ).catch((error: unknown) => error); // The authenticated service callback records before waiting for // this turn's drain. Observe that durable pending source event. await vi.waitFor(async () => { const rows = await db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.eventKind, mode === "source_updated_during_download" ? "message_updated" : "message_deleted", ), ), ); expect(rows).toEqual([{ state: "received" }]); }); return { data: image }; }); } // Actual parser + durable service. Provider HTTP, outer authenticated // callback delivery, and native scheduler are explicitly simulated. const deliver = () => deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: thread.thread, message, trigger: "mention", }); if (mode === "revoked_during_download") await expect(deliver()).rejects.toThrow( "no longer authorized to start work", ); else if (mode.startsWith("source_")) await expect(deliver()).rejects.toThrow("admitted source changed"); else await deliver(); await lifecycleCompletion; if (mode === "restart" || mode === "revoked_after_receipt") { const [received] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(received).toMatchObject({ state: "received", attempts: 0 }); expect(get).not.toHaveBeenCalled(); expect(storage.putFile).not.toHaveBeenCalled(); expect( JSON.stringify(received.normalizedEvent.message), ).not.toContain("https://"); expect(received.normalizedEvent).toMatchObject({ message: { attachments: [ { recovery: { locator: { kind: "teams_inline_image", messageId: message.id, }, }, }, ], }, }); await service.shutdown(); recoveredParser = createChatSdkEndpointRuntime({ ...configuration, logger: "silent", }); const nextHttp = ( recoveredParser.getProviderAdapter() as unknown as { app: { api: { http: { get(...args: unknown[]): Promise } }; }; } ).app.api.http; recoveredGet = vi .spyOn(nextHttp, "get") .mockResolvedValue({ data: image }); const nextRuntime = new FakeChatSdkRuntime(); const replace = nextRuntime.replaceEndpoint.bind(nextRuntime); vi.spyOn(nextRuntime, "replaceEndpoint").mockImplementation( async (options) => { const next = await replace(options); Object.assign(next, { attachmentRecoveryDescriptor: recoveredParser!.attachmentRecoveryDescriptor.bind( recoveredParser, ), rehydrateAttachment: recoveredParser!.rehydrateAttachment.bind(recoveredParser), fetchTeamsInlineImage: recoveredParser!.fetchTeamsInlineImage.bind( recoveredParser, ), }); const originalThread = next.thread.bind(next); vi.spyOn(next, "thread").mockImplementation((id) => ({ ...originalThread(id), isDM: false, })); return next; }, ); if (mode === "revoked_after_receipt") { if (conversationType === "groupChat") await db .update(chatEndpoints) .set({ allowGroupChats: false }) .where(eq(chatEndpoints.id, endpoint.id)); else await db .update(chatEndpointResources) .set({ enabled: false }) .where(eq(chatEndpointResources.endpointId, endpoint.id)); // Leave verifying mode so restart cannot legitimately activate a // first setup channel after this explicit operator revocation. await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, endpoint.id)); } await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, received.id)); restarted = createService(nextRuntime, undefined, { storage: storage.storage, scheduleDeferredWork: () => undefined, }); await restarted.service.processPendingDeliveries(25, received.id); expect(get).not.toHaveBeenCalled(); } const [delivery] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "mention"), ), ); if (mode === "batch_deadline") { expect(delivery.state).toBe("processed"); expect(delivery.redactedError).toContain( "2 external attachments were omitted", ); expect(storage.putFile).not.toHaveBeenCalled(); expect(get).toHaveBeenCalledOnce(); expect(wakeup).toHaveBeenCalledOnce(); await expect( db .select({ id: issueAttachments.id }) .from(issueAttachments) .where(eq(issueAttachments.companyId, fixture.companyId)), ).resolves.toHaveLength(0); return; } if (mode.startsWith("revoked_") || mode.startsWith("source_")) { expect(storage.putFile).not.toHaveBeenCalled(); expect(restarted?.wakeup ?? wakeup).not.toHaveBeenCalled(); expect(delivery.state).toBe( mode === "revoked_after_receipt" ? "filtered" : "failed", ); if (mode === "revoked_after_receipt") expect(recoveredGet).not.toHaveBeenCalled(); else expect(get).toHaveBeenCalledOnce(); await expect( db .select({ id: issueAttachments.id }) .from(issueAttachments) .where(eq(issueAttachments.companyId, fixture.companyId)), ).resolves.toHaveLength(0); return; } expect({ state: delivery.state, error: delivery.redactedError, }).toEqual({ state: "processed", error: null }); expect(storage.putFile).toHaveBeenCalledOnce(); expect(storage.putFile.mock.calls[0]![0]).toMatchObject({ body: image, contentType: "image/png", }); const download = recoveredGet ?? get; expect(download).toHaveBeenCalledOnce(); expect(download.mock.calls[0]![0]).toBe(url); expect(restarted?.wakeup ?? wakeup).toHaveBeenCalledOnce(); expect(delivery).toMatchObject({ state: "processed", redactedError: null, }); expect( JSON.stringify(delivery.normalizedEvent.message), ).not.toContain("https://"); await expect( db .select({ id: issueAttachments.id }) .from(issueAttachments) .where(eq(issueAttachments.companyId, fixture.companyId)), ).resolves.toHaveLength(1); await (restarted?.service ?? service).processPendingDeliveries( 25, delivery.id, ); expect(download).toHaveBeenCalledOnce(); expect(restarted?.wakeup ?? wakeup).toHaveBeenCalledOnce(); } finally { await lifecycleCompletion; timeoutSpy?.mockRestore(); get.mockRestore(); recoveredGet?.mockRestore(); try { await pinned.shutdown(); } finally { try { await recoveredParser?.shutdown(); } finally { try { await restarted?.service.shutdown(); } finally { await retirePublicationFixture(service, endpoint.id); } } } } }, ); }); it("ingests Teams files only from personal chats and keeps non-DM references link-only", async () => { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, service } = await configuredTeamsEndpoint( fixture, { storage: storage.storage }, ); const serviceUrl = "https://smba.trafficmanager.net/amer/"; const fileBody = Buffer.from("teams personal file", "utf8"); const channelFetch = vi.fn(async () => fileBody); const personalFetch = vi.fn(async () => fileBody); const channelConversationId = "19:teams-file-channel@thread.tacv2"; const channelRootId = "1740000000191"; const channelThread = makeThread({ channelId: `teams:${Buffer.from(channelConversationId).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, id: `teams:${Buffer.from(`${channelConversationId};messageid=${channelRootId}`).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, name: "Files channel", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: channelThread.thread, message: makeMessage({ attachments: [ { type: "file", name: "channel-plan.txt", mimeType: "text/plain", size: fileBody.length, fetchData: channelFetch, fetchMetadata: { testRecoveryKey: "teams-channel-file" }, } as Attachment, ], id: channelRootId, text: "", mentioned: true, }), trigger: "mention", }); expect(channelFetch).not.toHaveBeenCalled(); expect(storage.putFile).not.toHaveBeenCalled(); const personalConversationId = "a:teams-file-personal"; const personalThread = makeThread({ channelId: `teams:${Buffer.from(personalConversationId).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}:personal`, id: `teams:${Buffer.from(personalConversationId).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}:personal`, isDM: true, name: "Personal files", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: personalThread.thread, message: makeMessage({ attachments: [ { type: "file", name: "personal-plan.txt", mimeType: "text/plain", size: fileBody.length, fetchData: personalFetch, fetchMetadata: { testRecoveryKey: "teams-personal-file" }, } as Attachment, ], id: "teams-personal-file-message", text: "", }), trigger: "direct_message", }); expect(personalFetch).toHaveBeenCalledTimes(1); expect(storage.putFile).toHaveBeenCalledTimes(1); const deliveries = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); const channelDelivery = deliveries.find( (delivery) => delivery.providerEventId === `teams:${Buffer.from(`${channelConversationId};messageid=${channelRootId}`).toString("base64url")}:${channelRootId}`, ); const personalDelivery = deliveries.find((delivery) => delivery.providerEventId.endsWith(":teams-personal-file-message"), ); expect(channelDelivery?.normalizedEvent).toMatchObject({ message: { attachments: [ expect.objectContaining({ name: "channel-plan.txt", recovery: null, }), ], }, }); expect(personalDelivery?.normalizedEvent).toMatchObject({ message: { attachments: [ expect.objectContaining({ name: "personal-plan.txt", recovery: expect.objectContaining({ provider: "microsoft-teams", }), }), ], }, }); const comments = await db .select({ body: issueComments.body }) .from(issueComments) .innerJoin( chatConversations, eq(chatConversations.issueId, issueComments.issueId), ) .where(eq(chatConversations.endpointId, endpoint.id)); expect(comments).toEqual( expect.arrayContaining([ { body: "Shared 1 Microsoft Teams file reference.", }, { body: "Shared 1 file." }, ]), ); await expect( db .select({ id: issueAttachments.id }) .from(issueAttachments) .where(eq(issueAttachments.companyId, fixture.companyId)), ).resolves.toHaveLength(1); }); it.each(["captioned", "file_only"] as const)( "preserves a Teams personal %s attachment omission after losing its live download closure", async (mode) => { const fixture = await seedCompany(); const storage = createStorageService(); const first = await configuredTeamsEndpoint(fixture, { storage: storage.storage, deferWebhookProcessing: true, scheduleDeferredWork: () => undefined, }); const { endpoint } = first; const configuration = first.runtime.configurations.get(endpoint.id)!; const pinned = createChatSdkEndpointRuntime({ ...configuration, callbacks: { onMessage() {} }, logger: "silent", }); const sourceUrl = "https://contoso.sharepoint.com/download?signature=never-persist-this"; const caption = mode === "captioned" ? "Inspect only this exact new file" : ""; const parsed = pinned.parseMicrosoftTeamsMessage({ id: `teams-personal-unavailable-${mode}`, type: "message", text: caption, timestamp: new Date().toISOString(), serviceUrl: "https://smba.trafficmanager.net/amer/", from: { id: "29:personal-file-user", name: "Personal File User" }, conversation: { id: `a:personal-unavailable-${mode}`, conversationType: "personal", tenantId: configuration.providerConfig.provider === "microsoft-teams" ? configuration.providerConfig.credentials.appTenantId : undefined, }, attachments: [ { contentType: "application/vnd.microsoft.teams.file.download.info", contentUrl: "https://contoso.sharepoint.com/Documents/current-plan.txt", name: "current-plan.txt", content: { downloadUrl: sourceUrl, fileType: "txt", uniqueId: "current-file", }, }, ], }); expect(parsed?.attachments).toHaveLength(1); expect(parsed!.attachments[0]!.mimeType).toBe("text/plain"); expect( pinned.attachmentRecoveryDescriptor(parsed!.attachments[0]!), ).toBeNull(); const fetchData = vi.fn(async () => { throw new Error("Live closure must not survive restart"); }); parsed!.attachments[0]!.fetchData = fetchData; const descriptor = vi .spyOn( first.runtime.endpoints.get(endpoint.id)!, "attachmentRecoveryDescriptor", ) .mockImplementation((attachment) => { expect(pinned.attachmentRecoveryDescriptor(attachment)).toBeNull(); return null; }); const sourceThread = makeThread({ id: parsed!.threadId, channelId: parsed!.threadId, isDM: true, name: "Personal attachment recovery", }); let restarted: ReturnType | undefined; try { await deliverMessage({ callbacks: first.callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: sourceThread.thread, message: parsed!, trigger: "direct_message", }); const [received] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(received).toMatchObject({ state: "received", attempts: 0 }); expect(received!.normalizedEvent).toMatchObject({ message: { attachments: [ { name: "current-plan.txt", mimeType: "text/plain", recovery: null, }, ], }, }); expect(JSON.stringify(received!.normalizedEvent)).not.toMatch( /signature|never-persist-this|downloadUrl|sharepoint/, ); expect(first.wakeup).not.toHaveBeenCalled(); await first.service.shutdown(); // The fixture restarts after the provider reorder window, without // changing the receipt, source metadata, or recovery descriptor. await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, received!.id)); const nextRuntime = new FakeChatSdkRuntime(); const replace = nextRuntime.replaceEndpoint.bind(nextRuntime); vi.spyOn(nextRuntime, "replaceEndpoint").mockImplementation( async (options) => { const runtime = await replace(options); const thread = runtime.thread.bind(runtime); vi.spyOn(runtime, "thread").mockImplementation((threadId) => ({ ...thread(threadId), // Use the real adapter's personal/channel classification after restart. isDM: pinned.getProviderAdapter().isDM!(threadId), })); return runtime; }, ); restarted = createService(nextRuntime, undefined, { storage: storage.storage, scheduleDeferredWork: () => undefined, }); await restarted.service.processPendingDeliveries(); const [processed] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, received!.id)); expect(processed).toMatchObject({ state: "processed", redactedError: "1 external attachment was omitted (download unavailable: 1)", }); const [link] = await db .select() .from(chatMessageLinks) .where( and( eq(chatMessageLinks.deliveryId, received!.id), eq(chatMessageLinks.direction, "inbound"), ), ); const [comment] = await db .select() .from(issueComments) .where(eq(issueComments.id, link!.commentId!)); expect(comment!.body).toBe(caption || "Shared 1 file."); expect(restarted.wakeup).toHaveBeenCalledTimes(1); expect( restarted.wakeup.mock.calls[0]![1].contextSnapshot, ).toMatchObject({ wakeCommentId: comment!.id, externalAttachmentOmissions: [ { commentId: comment!.id, reasons: { download_unavailable: 1 } }, ], }); const [intent] = await db .select() .from(chatActions) .where( and( eq(chatActions.deliveryId, received!.id), eq(chatActions.kind, "inbound_wakeup"), ), ); expect(intent!.payload).toMatchObject({ attachmentOmissionReasons: { download_unavailable: 1 }, }); await restarted.service.processPendingDeliveries(); expect(restarted.wakeup).toHaveBeenCalledTimes(1); expect(fetchData).not.toHaveBeenCalled(); expect(storage.putFile).not.toHaveBeenCalled(); expect( await db .select({ id: issueAttachments.id }) .from(issueAttachments) .where(eq(issueAttachments.companyId, fixture.companyId)), ).toEqual([]); } finally { descriptor.mockRestore(); await first.service.shutdown(); await pinned.shutdown(); await retirePublicationFixture( restarted?.service ?? first.service, endpoint.id, ); } }, ); it.each([ { surface: "channel", mode: "immediate" }, { surface: "channel", mode: "deferred" }, { surface: "channel", mode: "restart" }, { surface: "channel", mode: "retry_restart" }, { surface: "group", mode: "immediate" }, { surface: "group", mode: "deferred" }, { surface: "group", mode: "restart" }, { surface: "group", mode: "retry_restart" }, ] as const)( "preserves unavailable Teams $surface file references through $mode admission", async ({ surface, mode }) => { const fixture = await seedCompany(); const storage = createStorageService(); const context = await configuredTeamsEndpoint(fixture, { storage: storage.storage, deferWebhookProcessing: mode !== "immediate" && mode !== "retry_restart", scheduleDeferredWork: () => undefined, }); const { callbacks, endpoint, service } = context; const serviceUrl = "https://smba.trafficmanager.net/amer/"; const conversationId = `19:teams-unavailable-${surface}-${mode}@thread.${surface === "channel" ? "tacv2" : "v2"}`; const rootId = "1740000000391"; const thread = makeThread({ channelId: `teams:${Buffer.from(conversationId).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, id: `teams:${Buffer.from(`${conversationId}${surface === "channel" ? `;messageid=${rootId}` : ""}`).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, name: "Unavailable current files", }); const fetchData = vi.fn(async () => Buffer.from("must not download")); const providerUrl = "https://private.example/file?signature=do-not-persist"; const attachments = [ { type: "image", name: "../current-photo.png", mimeType: "image/png", size: 128, url: providerUrl, fetchData, fetchMetadata: { authorization: "do-not-persist" }, }, { type: "file", name: "current-plan.txt", mimeType: "text/plain; charset=utf-8", size: 32, url: providerUrl, fetchData, }, ] as Attachment[]; let active = context; let restarted: ReturnType | undefined; try { if (surface === "group") { await service.update( endpoint.id, { allowGroupChats: true }, "owner-user", ); } await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: surface === "channel" ? "channel" : "group_chat", providerResourceId: conversationId, label: "Unavailable current files", availability: "available", enabled: true, }); for (const [index, text] of [ "Inspect this current image and file", "", ].entries()) { if (mode === "retry_restart" && index === 0) { thread.subscribe.mockRejectedValueOnce( new Error("subscription unavailable"), ); } const deliveryAttempt = deliverMessage({ callbacks: index === 0 ? callbacks : active.runtime.configurations.get(endpoint.id)!.callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: thread.thread, message: makeMessage({ id: String(Number(rootId) + index), text, mentioned: true, attachments, }), trigger: "mention", }); if (mode === "retry_restart" && index === 0) { await expect(deliveryAttempt).rejects.toThrow( "subscription unavailable", ); } else { await deliveryAttempt; } const [delivery] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), like( chatDeliveries.providerEventId, `%:${Number(rootId) + index}`, ), ), ); expect(delivery).toBeDefined(); expect(delivery!.normalizedEvent).toMatchObject({ message: { attachments: [ { name: "current-photo.png", mimeType: "image/png", size: 128, recovery: null, }, { name: "current-plan.txt", mimeType: "text/plain", size: 32, recovery: null, }, ], }, }); expect(JSON.stringify(delivery!.normalizedEvent)).not.toContain( "do-not-persist", ); if (mode !== "immediate") { expect(delivery!.state).toBe( mode === "retry_restart" && index === 0 ? "retry" : "received", ); if ( (mode === "restart" || mode === "retry_restart") && index === 0 ) { await service.shutdown(); restarted = createService(new FakeChatSdkRuntime(), undefined, { storage: storage.storage, deferWebhookProcessing: true, scheduleDeferredWork: () => undefined, }); active = { ...context, ...restarted }; } await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, delivery!.id)); await active.service.processPendingDeliveries(); } const [processed] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, delivery!.id)); expect(processed).toMatchObject({ state: "processed", redactedError: "2 external attachments were omitted (download unavailable: 2)", }); const [link] = await db .select() .from(chatMessageLinks) .where( and( eq(chatMessageLinks.deliveryId, delivery!.id), eq(chatMessageLinks.direction, "inbound"), ), ); const [comment] = await db .select() .from(issueComments) .where(eq(issueComments.id, link!.commentId!)); expect(comment!.body).toBe( text || "Shared 2 Microsoft Teams file references.", ); const requests = [ ...context.wakeup.mock.calls, ...(restarted?.wakeup.mock.calls ?? []), ] .map(([, request]) => request) .filter( (request) => request.contextSnapshot?.wakeCommentId === comment!.id, ); expect(requests).toHaveLength(1); expect(requests[0]!.contextSnapshot).toMatchObject({ externalAttachmentOmissions: [ { commentId: comment!.id, reasons: { download_unavailable: 2 }, }, ], }); const [intent] = await db .select() .from(chatActions) .where( and( eq(chatActions.deliveryId, delivery!.id), eq(chatActions.kind, "inbound_wakeup"), ), ); expect(intent!.payload).toMatchObject({ commentId: comment!.id, attachmentOmissionReasons: { download_unavailable: 2 }, }); } await active.service.processPendingDeliveries(); expect( context.wakeup.mock.calls.length + (restarted?.wakeup.mock.calls.length ?? 0), ).toBe(2); expect(fetchData).not.toHaveBeenCalled(); expect(storage.putFile).not.toHaveBeenCalled(); expect( active.runtime.endpoints.get(endpoint.id)! .rehydratedAttachmentDescriptors, ).toEqual([]); await expect( db .select({ id: issueAttachments.id }) .from(issueAttachments) .where(eq(issueAttachments.companyId, fixture.companyId)), ).resolves.toEqual([]); await expect( db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.companyId, fixture.companyId)), ).resolves.toHaveLength(2); } finally { await service.shutdown(); await restarted?.service.shutdown(); } }, ); it("holds a delayed first Teams setup reply until its older root mention enables the channel", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service, wakeup } = await configuredTeamsEndpoint(fixture, { deferWebhookProcessing: true, scheduleDeferredWork: () => undefined, }); const conversationId = "19:teams-delayed-root@thread.tacv2"; const serviceUrl = "https://smba.trafficmanager.net/amer/"; const rootMessageId = "1740000000001"; const thread = makeThread({ channelId: `teams:${Buffer.from(conversationId).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, id: `teams:${Buffer.from(`${conversationId};messageid=${rootMessageId}`).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, name: "delayed-root", }); const laterReply = makeMessage({ id: "1740000000002", raw: { timestamp: "2026-09-05T18:20:02.000Z" }, text: "Teams follow-up whose callback arrived first", userId: "U-TEAMS-PROVISIONAL-PRIVATE", userName: "provisional-private-user", }); laterReply.metadata.dateSent = new Date("2026-09-05T18:20:02.000Z"); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: thread.thread, message: laterReply, trigger: "subscribed_message", }); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.endpointId, endpoint.id)); await service.processPendingDeliveries(); const [deferredReply] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(deferredReply).toMatchObject({ state: "retry", attempts: 1, principalId: null, redactedError: "Waiting briefly for an earlier root mention", normalizedEvent: { providerEventId: `teams:${Buffer.from(`${conversationId};messageid=${rootMessageId}`).toString("base64url")}:${laterReply.id}`, kind: "message", trigger: "subscribed_message", resource: { type: "channel", providerResourceId: conversationId, }, conversation: { externalThreadId: thread.thread.id }, message: { providerMessageId: laterReply.id, providerSentAt: "2026-09-05T18:20:02.000Z", }, filtering: { contentRetained: false }, }, }); const serializedDeferredReply = JSON.stringify(deferredReply); expect(serializedDeferredReply).not.toContain(laterReply.text); expect(serializedDeferredReply).not.toContain("Alex External"); expect(serializedDeferredReply).not.toContain("provisional-private-user"); expect(serializedDeferredReply).not.toContain( "U-TEAMS-PROVISIONAL-PRIVATE", ); await expect( db .select({ id: chatExternalPrincipals.id }) .from(chatExternalPrincipals) .where(eq(chatExternalPrincipals.companyId, fixture.companyId)), ).resolves.toHaveLength(0); expect(deferredReply.nextAttemptAt).not.toBeNull(); expect(await service.listConversations(endpoint.id)).toHaveLength(0); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ providerResourceId: conversationId, type: "channel", enabled: false, }), ]); const earlierRoot = makeMessage({ id: rootMessageId, raw: { timestamp: "2026-09-05T18:20:01.000Z" }, text: "@maya keep both Teams messages", mentioned: true, }); earlierRoot.metadata.dateSent = new Date("2026-09-05T18:20:01.000Z"); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: thread.thread, message: earlierRoot, trigger: "mention", }); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.state, "received"), ), ); await service.processPendingDeliveries(); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, deferredReply.id)); await service.processPendingDeliveries(); const [conversation] = await service.listConversations(endpoint.id); expect(conversation).toMatchObject({ externalThreadId: thread.thread.id }); await expect(service.listResources(endpoint.id)).resolves.toEqual([ expect.objectContaining({ providerResourceId: conversationId, type: "channel", enabled: true, }), ]); await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, conversation!.issueId)) .orderBy(asc(issueComments.createdAt), asc(issueComments.id)), ).resolves.toEqual([ { body: "@maya keep both Teams messages" }, { body: "Teams follow-up whose callback arrived first" }, ]); const [hydratedReply] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, deferredReply.id)); expect(hydratedReply).toMatchObject({ state: "processed", principalId: expect.any(String), normalizedEvent: { principal: { externalId: "U-TEAMS-PROVISIONAL-PRIVATE", displayName: "Alex External", handle: "provisional-private-user", }, message: { providerMessageId: laterReply.id, providerSentAt: "2026-09-05T18:20:02.000Z", providerSentAtSource: "teams_activity_timestamp", text: laterReply.text, }, }, }); expect( (hydratedReply.normalizedEvent as { filtering?: unknown }).filtering, ).toBeUndefined(); expect(wakeup).toHaveBeenCalledTimes(2); }); it("orders rapid Teams follow-ups by provider time when callbacks arrive reversed", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service, wakeup } = await configuredTeamsEndpoint(fixture, { deferWebhookProcessing: true, scheduleDeferredWork: () => undefined, }); const conversationId = "19:teams-follow-up-order@thread.tacv2"; const serviceUrl = "https://smba.trafficmanager.net/amer/"; const rootMessageId = "1740000000020"; await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: conversationId, label: "follow-up-order", availability: "available", enabled: true, }); const thread = makeThread({ channelId: `teams:${Buffer.from(conversationId).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, id: `teams:${Buffer.from(`${conversationId};messageid=${rootMessageId}`).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, name: "follow-up-order", }); const root = makeMessage({ id: rootMessageId, raw: { timestamp: "2026-09-05T18:30:00.000Z" }, text: "@maya start an ordered Teams task", mentioned: true, }); root.metadata.dateSent = new Date("2026-09-05T18:30:00.000Z"); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: thread.thread, message: root, trigger: "mention", }); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.endpointId, endpoint.id)); await service.processPendingDeliveries(); wakeup.mockClear(); const later = makeMessage({ id: "1740000000022", raw: { timestamp: "2026-09-05T18:30:02.000Z" }, text: "Teams follow-up two", }); later.metadata.dateSent = new Date("2026-09-05T18:30:02.000Z"); const earlier = makeMessage({ id: "1740000000021", raw: { timestamp: "2026-09-05T18:30:01.000Z" }, text: "Teams follow-up one", }); earlier.metadata.dateSent = new Date("2026-09-05T18:30:01.000Z"); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: thread.thread, message: later, trigger: "subscribed_message", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: thread.thread, message: earlier, trigger: "subscribed_message", }); await service.processPendingDeliveries(); expect(wakeup).not.toHaveBeenCalled(); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where( and( eq(chatDeliveries.endpointId, endpoint.id), inArray(chatDeliveries.state, ["received", "retry"]), ), ); await service.processPendingDeliveries(); const [conversation] = await service.listConversations(endpoint.id); await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, conversation!.issueId)) .orderBy(asc(issueComments.createdAt), asc(issueComments.id)), ).resolves.toEqual([ { body: "@maya start an ordered Teams task" }, { body: "Teams follow-up one" }, { body: "Teams follow-up two" }, ]); expect(wakeup).toHaveBeenCalledTimes(2); }); it("keeps a first-channel Teams orphan payload-free during grace and filters it without a live retry", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service, wakeup } = await configuredTeamsEndpoint(fixture, { deferWebhookProcessing: true, scheduleDeferredWork: () => undefined, }); const conversationId = "19:teams-orphan-only@thread.tacv2"; const serviceUrl = "https://smba.trafficmanager.net/amer/"; const thread = makeThread({ channelId: `teams:${Buffer.from(conversationId).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, id: `teams:${Buffer.from(`${conversationId};messageid=1740000000011`).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, name: "orphan-only", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: thread.thread, message: makeMessage({ id: "1740000000012", text: "private orphan content not for the Teams bot", userId: "U-TEAMS-ORPHAN-PRIVATE", userName: "orphan-private-user", }), trigger: "unaddressed_message", }); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.endpointId, endpoint.id)); await service.processPendingDeliveries(); const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(delivery).toMatchObject({ state: "retry", attempts: 1, principalId: null, normalizedEvent: { filtering: { contentRetained: false }, resource: { providerResourceId: conversationId, type: "channel" }, conversation: { externalThreadId: thread.thread.id }, message: { providerMessageId: "1740000000012" }, }, }); const serializedDelivery = JSON.stringify(delivery); expect(serializedDelivery).not.toContain("private orphan content"); expect(serializedDelivery).not.toContain("Alex External"); expect(serializedDelivery).not.toContain("orphan-private-user"); expect(serializedDelivery).not.toContain("U-TEAMS-ORPHAN-PRIVATE"); await expect( db .select({ id: chatExternalPrincipals.id }) .from(chatExternalPrincipals) .where(eq(chatExternalPrincipals.companyId, fixture.companyId)), ).resolves.toHaveLength(0); await service.shutdown(); const restarted = createService(); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, delivery.id)); await restarted.service.processPendingDeliveries(); await expect( db .select({ state: chatDeliveries.state, attempts: chatDeliveries.attempts, }) .from(chatDeliveries) .where(eq(chatDeliveries.id, delivery.id)), ).resolves.toEqual([{ state: "filtered", attempts: 1 }]); expect(await restarted.service.listConversations(endpoint.id)).toHaveLength( 0, ); await expect( db .select({ id: chatExternalPrincipals.id }) .from(chatExternalPrincipals) .where(eq(chatExternalPrincipals.companyId, fixture.companyId)), ).resolves.toHaveLength(0); expect(wakeup).not.toHaveBeenCalled(); await restarted.service.shutdown(); }); it.each([ { status: 200, count: 1, expireBatch: false }, { status: 404, count: 1, expireBatch: false }, { status: 200, count: 21, expireBatch: false }, { status: 200, count: 3, expireBatch: true }, ])( "ingests only admitted public GitHub attachments after restart (HTTP $status, $count files, expired batch $expireBatch)", async ({ status, count, expireBatch }) => { const fixture = await seedCompany(); const storage = createStorageService(); const deferred: Array<() => void> = []; const context = await configuredGitHubEndpoint(fixture, { storage: storage.storage, deferWebhookProcessing: true, scheduleDeferredWork: (work) => deferred.push(work), }); const { service, endpoint, callbacks } = context; const sourceThreadId = "github:paperclipai/paperclip:issue:93"; const sourceUrl = "https://github.com/user-attachments/files/31917991/public-proof.txt"; const thread = makeThread({ id: sourceThreadId, channelId: "github:paperclipai/paperclip", name: "paperclipai/paperclip", }); const publicBody = Buffer.from("exact current GitHub public file"); const batch = new AbortController(); const egress = vi .spyOn(attachmentEgress, "guardedRemoteHttpFetch") .mockImplementation(async () => { if (expireBatch) batch.abort(); return new Response( status === 200 ? publicBody : "private response never retained", { status, headers: { "content-type": "text/plain" } }, ); }); let restarted: ReturnType | undefined; let timeoutSpy: { mockRestore(): void } | undefined; try { await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, endpoint.id)); const [resource] = await service.listResources(endpoint.id); await service.replaceResources(endpoint.id, [ { id: resource!.id, enabled: false }, ]); const send = async (id: string) => { const urls = Array.from({ length: count }, (_, index) => index === 0 ? sourceUrl : `https://github.com/user-attachments/files/${31917991 + index}/proof-${index}.txt`, ); const message = makeMessage({ id, text: "@maya inspect the exact current files", mentioned: true, userId: "42", raw: { type: "issue_comment", threadType: "issue", prNumber: 93, repository: { full_name: "paperclipai/paperclip" }, comment: { id: Number(id), body: urls.map((url) => `[file](${url})`).join("\n"), user: { id: 42 }, }, }, }); message.threadId = sourceThreadId; message.formatted = { type: "root", children: urls.map((url) => ({ type: "link", url, children: [] })), }; message.attachments.push( ...githubPublicAttachmentsFromMessage(message), ); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, trigger: "mention", message, }); }; await send("93001"); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.endpointId, endpoint.id)); await service.processPendingDeliveries(); expect(egress).not.toHaveBeenCalled(); expect(storage.putFile).not.toHaveBeenCalled(); await service.replaceResources(endpoint.id, [ { id: resource!.id, enabled: true }, ]); await send("93002"); expect(egress).not.toHaveBeenCalled(); const [delivery] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), like(chatDeliveries.providerEventId, "%:93002"), ), ); expect({ state: delivery!.state, error: delivery!.redactedError, }).toEqual({ state: "received", error: null }); expect( (delivery!.normalizedEvent as { message: { attachments: unknown[] } }) .message.attachments[0], ).toMatchObject({ recovery: { provider: "github", locator: { url: sourceUrl, sourceMessageId: "93002", sourceThreadId, }, }, }); expect( ( delivery!.normalizedEvent as { message: { attachments: unknown[]; attachmentLimitOmissions?: number; }; } ).message, ).toMatchObject({ attachments: expect.any(Array), ...(count > 20 ? { attachmentLimitOmissions: 1 } : {}), }); expect( (delivery!.normalizedEvent as { message: { attachments: unknown[] } }) .message.attachments, ).toHaveLength(Math.min(20, count)); await service.shutdown(); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, delivery!.id)); restarted = createService( new FakeChatSdkRuntime(), context.providerFetch as typeof globalThis.fetch, { storage: storage.storage }, ); if (expireBatch) { const timeout = AbortSignal.timeout.bind(AbortSignal); timeoutSpy = vi .spyOn(AbortSignal, "timeout") .mockImplementation((ms) => ms === GITHUB_ATTACHMENT_BATCH_TIMEOUT_MS ? batch.signal : timeout(ms), ); } await restarted.service.processPendingDeliveries(); const [storedDelivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, delivery!.id)); expect({ state: storedDelivery!.state, error: storedDelivery!.redactedError, recovered: restarted.runtime.endpoints.get(endpoint.id) ?.rehydratedAttachmentDescriptors.length, }).toEqual({ state: "processed", error: status === 200 && count <= 20 && !expireBatch ? null : expect.any(String), recovered: Math.min(20, count), }); expect(egress).toHaveBeenCalledTimes( expireBatch ? 1 : Math.min(20, count), ); expect(restarted.wakeup).toHaveBeenCalledTimes(1); expect(storedDelivery!.state).toBe("processed"); expect(JSON.stringify(storedDelivery)).not.toContain( "private response", ); if (status === 200 && !expireBatch) { expect(storage.putFile).toHaveBeenCalledTimes(Math.min(20, count)); expect(storage.putFile).toHaveBeenCalledWith( expect.objectContaining({ body: publicBody, originalFilename: "public-proof.txt", contentType: "text/plain", }), ); const storedAttachments = await db .select({ sha256: assets.sha256 }) .from(issueAttachments) .innerJoin(assets, eq(assets.id, issueAttachments.assetId)) .where(eq(issueAttachments.companyId, fixture.companyId)); expect(storedAttachments).toHaveLength(Math.min(20, count)); expect( storedAttachments.every( (row) => row.sha256 === createHash("sha256").update(publicBody).digest("hex"), ), ).toBe(true); if (count > 20) { const wakeJson = JSON.stringify(restarted.wakeup.mock.calls); expect(wakeJson).toContain('"attachment_limit":1'); expect(wakeJson).toContain("externalAttachmentOmissions"); } } else { expect(storage.putFile).not.toHaveBeenCalled(); const wakeJson = JSON.stringify(restarted.wakeup.mock.calls); expect(wakeJson).toContain(`"download_unavailable":${count}`); expect(wakeJson).toContain("externalAttachmentOmissions"); expect( await db .select() .from(issueAttachments) .where(eq(issueAttachments.companyId, fixture.companyId)), ).toHaveLength(0); } await restarted.service.processPendingDeliveries(); expect(egress).toHaveBeenCalledTimes( expireBatch ? 1 : Math.min(20, count), ); expect(restarted.wakeup).toHaveBeenCalledTimes(1); } finally { await service.shutdown(); await restarted?.service.shutdown(); egress.mockRestore(); timeoutSpy?.mockRestore(); } }, ); it.each([ "none", "download", "storage", "cancel_race", "delivery_race", "signed_anchor", "ambiguous_anchor", ] as const)( "resolves an admitted GitHub private image after restart with current reach at %s", async (revokeAt) => { const fixture = await seedCompany(); const storage = createStorageService(); const context = await configuredGitHubEndpoint(fixture, { storage: storage.storage, deferWebhookProcessing: true, scheduleDeferredWork: () => {}, }); const { service, endpoint, callbacks } = context; const sourceThreadId = "github:paperclipai/paperclip:issue:93"; const sourceUrl = "https://github.com/user-attachments/assets/11111111-2222-3333-4444-555555555555"; const signedUrl = "https://private-user-images.githubusercontent.com/123/456-11111111-2222-3333-4444-555555555555.png?jwt=header.privatepayload.signature"; const body = revokeAt === "signed_anchor" ? `exact current image` : `![exact current image](${sourceUrl})`; const bytes = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]); const [resource] = await service.listResources(endpoint.id); await service.replaceResources(endpoint.id, [ { id: resource!.id, enabled: true }, ]); const revoke = () => db .update(chatEndpointResources) .set({ enabled: false }) .where(eq(chatEndpointResources.id, resource!.id)); const resolve = vi .spyOn(FakeEndpointRuntime.prototype, "resolveGitHubAttachmentComment") .mockResolvedValue({ id: 93002, url: "https://api.github.com/repos/paperclipai/paperclip/issues/comments/93002", issue_url: "https://api.github.com/repos/paperclipai/paperclip/issues/93", body, body_html: ``.repeat( revokeAt === "ambiguous_anchor" ? 2 : 1, ), }); const warning = vi.spyOn(chatAttachmentLogger, "warn"); let receiptMutation: Promise | undefined; const egress = vi .spyOn(attachmentEgress, "guardedRemoteHttpFetch") .mockImplementation(async (url) => { if (String(url) === sourceUrl) return new Response("private", { status: 404 }); if (String(url) !== signedUrl) throw new Error("unexpected test egress"); if (revokeAt === "download") await revoke(); if (revokeAt === "cancel_race" || revokeAt === "delivery_race") { let entered!: () => void; const ready = new Promise((done) => { entered = done; }); receiptMutation = db .transaction(async (tx) => { const [conversation] = await tx .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); if (revokeAt === "cancel_race") { await tx .select() .from(issues) .where(eq(issues.id, conversation!.issueId)) .for("update"); } else { const [action] = await tx .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "inbound_wakeup"), ), ); await tx.execute( sql`select pg_advisory_xact_lock(hashtextextended(${`chat-identity:${fixture.companyId}:${action!.principalId}`}, 0))`, ); } const [backend] = (await tx.execute( sql`select pg_backend_pid() as pid`, )) as unknown as Array<{ pid: number }>; entered(); // The issue barrier is after the initial action read; the // identity barrier is after the initial delivery read. Observe // the real blocked query before mutating its stale snapshot. await vi.waitFor(async () => { const [state] = (await db.execute(sql`select exists ( select 1 from pg_stat_activity where ${backend!.pid} = any(pg_blocking_pids(pid)) ) as waiting`)) as unknown as Array<{ waiting: boolean }>; expect(state!.waiting).toBe(true); }); if (revokeAt === "cancel_race") { await tx .update(chatActions) .set({ status: "cancelled" }) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "inbound_wakeup"), ), ); } else { await tx .update(chatDeliveries) .set({ state: "failed" }) .where(eq(chatDeliveries.endpointId, endpoint.id)); } }) .then( () => null, (error: unknown) => error, ) .finally(entered); await ready; } return new Response(bytes, { headers: { "content-type": "image/png" }, }); }); if (revokeAt === "storage") { const put = storage.putFile.getMockImplementation()!; storage.putFile.mockImplementation(async (input) => { const result = await put(input); await revoke(); return result; }); } let restarted: ReturnType | undefined; try { const thread = makeThread({ id: sourceThreadId, channelId: "github:paperclipai/paperclip", name: "paperclipai/paperclip", }); const message = makeMessage({ id: "93002", text: "@maya inspect this exact image", mentioned: true, userId: "42", raw: { type: "issue_comment", threadType: "issue", prNumber: 93, repository: { full_name: "paperclipai/paperclip" }, comment: { id: 93002, body, user: { id: 42 } }, }, }); message.threadId = sourceThreadId; message.formatted = { type: "root", children: [{ type: "image", url: sourceUrl }], }; message.attachments.push( ...githubPublicAttachmentsFromMessage(message), ); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, trigger: "mention", message, }); expect(egress).not.toHaveBeenCalled(); expect(resolve).not.toHaveBeenCalled(); await service.shutdown(); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.endpointId, endpoint.id)); restarted = createService( new FakeChatSdkRuntime(), context.providerFetch, { storage: storage.storage }, ); await restarted.service.processPendingDeliveries(); if (receiptMutation) expect(await receiptMutation).toBeNull(); expect(resolve).toHaveBeenCalledExactlyOnceWith( { url: "https://api.github.com/repos/paperclipai/paperclip/issues/comments/93002", accept: "application/vnd.github.full+json", }, expect.any(AbortSignal), ); expect(egress).toHaveBeenCalledTimes( revokeAt === "ambiguous_anchor" ? 1 : 2, ); const stored = await db .select() .from(issueAttachments) .where(eq(issueAttachments.companyId, fixture.companyId)); const imported = revokeAt === "none" || revokeAt === "signed_anchor"; expect(stored).toHaveLength(imported ? 1 : 0); expect(restarted.wakeup).toHaveBeenCalledTimes( imported || revokeAt === "ambiguous_anchor" ? 1 : 0, ); expect(storage.objects.size).toBe(imported ? 1 : 0); if (["download", "cancel_race", "delivery_race"].includes(revokeAt)) expect(storage.putFile).not.toHaveBeenCalled(); if (revokeAt === "storage") expect(storage.storage.deleteObject).toHaveBeenCalledTimes(1); const deliveries = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(JSON.stringify(deliveries)).not.toContain("privatepayload"); expect(JSON.stringify(deliveries)).not.toContain("body_html"); if (revokeAt === "ambiguous_anchor") { const code = "github_attachment_canonical_mapping_ambiguous"; const diagnostic = warning.mock.calls.find( ([fields]) => typeof fields === "object" && fields !== null && "attachmentDiagnosticCode" in fields && fields.attachmentDiagnosticCode === code, ); expect(diagnostic?.[0]).toMatchObject({ endpointId: endpoint.id, deliveryId: deliveries[0]!.id, error: code, attachmentDiagnosticCode: code, }); expect(JSON.stringify(diagnostic)).not.toMatch( /privatepayload|jwt|body_html|https:/, ); const wakeJson = JSON.stringify(restarted.wakeup.mock.calls); expect(wakeJson).toContain('"download_unavailable":1'); expect(wakeJson).not.toContain(code); expect(storage.putFile).not.toHaveBeenCalled(); expect(deliveries[0]!.state).toBe("processed"); } if (imported) { expect([...storage.objects.values()][0]).toEqual(bytes); expect(JSON.stringify(restarted.wakeup.mock.calls)).not.toMatch( /privatepayload|jwt|body_html/, ); await restarted.service.processPendingDeliveries(); expect(egress).toHaveBeenCalledTimes(2); expect(restarted.wakeup).toHaveBeenCalledTimes(1); } } finally { await receiptMutation; await service.shutdown(); await restarted?.service.shutdown(); resolve.mockRestore(); egress.mockRestore(); warning.mockRestore(); } }, ); it("rehydrates a durable attachment descriptor after restart and stores the file on the issue", async () => { const fixture = await seedCompany(); const recoveryKey = `restart-attachment-${randomUUID()}`; const attachmentBody = Buffer.from("restart-safe attachment body", "utf8"); const attachmentBodies = new Map([[recoveryKey, attachmentBody]]); const firstRuntime = new FakeChatSdkRuntime(attachmentBodies); const deferred: Array<() => void> = []; const storage = createStorageService(); const first = createService( firstRuntime, fakeSlackFetch() as typeof globalThis.fetch, { deferWebhookProcessing: true, scheduleDeferredWork: (task) => deferred.push(task), storage: storage.storage, }, ); const endpoint = await first.service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await first.service.configure( endpoint.id, { action: "configure", credentials: { botToken: "xoxb-restart-attachment", signingSecret: "restart-attachment-signing-secret", }, }, "owner-user", ); await recordSlackUrlVerification(first.service, endpoint.publicId); await first.service.configure( endpoint.id, { action: "verify" }, "owner-user", ); const callbacks = firstRuntime.configurations.get(endpoint.id)?.callbacks; if (!callbacks) throw new Error("Expected first-process callbacks"); const thread = makeThread({ channelId: "C-RESTART-FILE", id: "slack:C-RESTART-FILE:9150.1", name: "restart-files", }); const liveFetch = vi.fn(async () => { throw new Error("the original attachment closure must not survive"); }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ attachments: [ { type: "file", name: "restart-note.txt", mimeType: "text/plain", size: attachmentBody.length, fetchData: liveFetch, fetchMetadata: { testRecoveryKey: recoveryKey }, } as Attachment, ], id: "9150.1", text: "@maya preserve this attachment", mentioned: true, }), trigger: "mention", }); const [durableDelivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(durableDelivery).toMatchObject({ state: "received", attempts: 0 }); expect(JSON.stringify(durableDelivery.normalizedEvent)).not.toContain( "xoxb-restart-attachment", ); expect(deferred).toHaveLength(1); expect(liveFetch).not.toHaveBeenCalled(); await first.service.shutdown(); // Restart after the bounded Slack reorder window has elapsed. await db .update(chatDeliveries) .set({ nextAttemptAt: new Date() }) .where(eq(chatDeliveries.id, durableDelivery.id)); const restartedRuntime = new FakeChatSdkRuntime(attachmentBodies); const restarted = createService( restartedRuntime, fakeSlackFetch() as typeof globalThis.fetch, { storage: storage.storage, }, ); await restarted.service.processPendingDeliveries(); const restartedEndpointRuntime = restartedRuntime.endpoints.get( endpoint.id, ); expect( restartedEndpointRuntime?.rehydratedAttachmentDescriptors, ).toHaveLength(1); expect(storage.putFile).toHaveBeenCalledTimes(1); expect(storage.putFile).toHaveBeenCalledWith( expect.objectContaining({ companyId: fixture.companyId, originalFilename: "restart-note.txt", contentType: "text/plain", body: attachmentBody, }), ); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const storedAttachments = await db .select({ attachmentId: issueAttachments.id, issueId: issueAttachments.issueId, issueCommentId: issueAttachments.issueCommentId, contentType: assets.contentType, byteSize: assets.byteSize, originalFilename: assets.originalFilename, }) .from(issueAttachments) .innerJoin(assets, eq(assets.id, issueAttachments.assetId)) .where(eq(issueAttachments.issueId, conversation.issueId)); expect(storedAttachments).toEqual([ expect.objectContaining({ issueId: conversation.issueId, issueCommentId: expect.any(String), contentType: "text/plain", byteSize: attachmentBody.length, originalFilename: "restart-note.txt", }), ]); await restarted.service.shutdown(); }); it("does not duplicate an inbound file when delivery recovery resumes after the task mutation", async () => { const fixture = await seedCompany(); const recoveryKey = `retry-attachment-${randomUUID()}`; const attachmentBody = Buffer.from("retry-safe attachment body", "utf8"); const runtime = new FakeChatSdkRuntime( new Map([[recoveryKey, attachmentBody]]), ); const storage = createStorageService(); const { service, wakeup } = createService( runtime, fakeSlackFetch() as typeof globalThis.fetch, { storage: storage.storage, }, ); const endpoint = await service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await service.configure( endpoint.id, { action: "configure", credentials: { botToken: "xoxb-retry-attachment", signingSecret: "retry-attachment-signing-secret", }, }, "owner-user", ); await recordSlackUrlVerification(service, endpoint.publicId); await service.configure(endpoint.id, { action: "verify" }, "owner-user"); const callbacks = runtime.configurations.get(endpoint.id)?.callbacks; if (!callbacks) throw new Error("Expected Slack callbacks"); const channel = makeThread({ channelId: "C-RETRY-FILE", id: "slack:C-RETRY-FILE:9160.1", name: "retry-files", }); channel.subscribe.mockRejectedValueOnce( new Error("injected provider subscription failure"), ); await expect( deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ attachments: [ { type: "file", name: "retry-note.txt", mimeType: "text/plain", size: attachmentBody.length, fetchData: async () => attachmentBody, fetchMetadata: { testRecoveryKey: recoveryKey }, } as Attachment, ], id: "9160.1", text: "@maya preserve this retrying attachment", mentioned: true, }), trigger: "mention", }), ).rejects.toThrow("injected provider subscription failure"); const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(delivery).toMatchObject({ state: "retry", attempts: 1 }); expect(wakeup).not.toHaveBeenCalled(); const [pendingIntent] = await db .select() .from(chatActions) .where( and( eq(chatActions.deliveryId, delivery.id), eq(chatActions.kind, "inbound_wakeup"), ), ); expect(pendingIntent.status).toBe("preparing"); // Generic stranded-task recovery cannot bypass the not-yet-accepted input. await expect( heartbeatService(db).wakeup(fixture.assignedAgentId, { source: "assignment", triggerDetail: "system", reason: "issue_assigned", requestedByActorType: "system", contextSnapshot: { issueId: pendingIntent.payload.issueId, source: "issue.assignment", }, payload: { issueId: pendingIntent.payload.issueId }, }), ).rejects.toMatchObject({ status: 409, details: { code: "chat_inbound_wakeup_unadmitted" }, }); expect( await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.companyId, fixture.companyId)), ).toEqual([]); expect(storage.putFile).toHaveBeenCalledTimes(1); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, delivery.id)); await service.processPendingDeliveries(25, delivery.id); expect(storage.putFile).toHaveBeenCalledTimes(1); await expect( db .select({ id: issueAttachments.id }) .from(issueAttachments) .where(eq(issueAttachments.companyId, fixture.companyId)), ).resolves.toHaveLength(1); await expect( db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where(eq(chatDeliveries.id, delivery.id)), ).resolves.toEqual([{ state: "processed" }]); expect(wakeup).toHaveBeenCalledTimes(1); const [action] = await db .select() .from(chatActions) .where( and( eq(chatActions.deliveryId, delivery.id), eq(chatActions.kind, "inbound_wakeup"), ), ); expect(action).toMatchObject({ status: "processed", result: { wakeupRequestId: action.id }, }); // Crash after the scheduler committed its receipt but before the action // acknowledgement: a new worker settles it without invoking wakeup again. await db .update(chatActions) .set({ status: "processing", result: null, updatedAt: new Date(0) }) .where(eq(chatActions.id, action.id)); await service.processPendingDeliveries(25, delivery.id); expect(wakeup).toHaveBeenCalledTimes(1); expect( await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, action.id)), ).toHaveLength(1); expect( await db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, action.id)), ).toEqual([ expect.objectContaining({ status: "processed", result: expect.objectContaining({ code: "inbound_wakeup_already_durable", }), }), ]); await service.shutdown(); }); it("retries an accepted inbound wake outbox in FIFO order without blocking other conversations", async () => { const fixture = await seedCompany(); let failAdmission = true; const admittedComments: string[] = []; const { service, callbacks, endpoint, wakeup } = await configuredSlackEndpoint(fixture, { deferWebhookProcessing: true, scheduleDeferredWork: () => {}, wakeup: async (_agentId, opts) => { const request = opts.durableChatRequest!; const [delivery] = await db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .innerJoin( chatActions, eq(chatActions.deliveryId, chatDeliveries.id), ) .where(eq(chatActions.id, request.id)); expect(delivery?.state).toBe("processed"); if (failAdmission) { failAdmission = false; throw new Error("injected scheduler unavailable"); } admittedComments.push(request.commentId); return { accepted: true }; }, }); const firstThread = makeThread({ channelId: "C-OUTBOX", id: "slack:C-OUTBOX:9200.1", }); const send = async (thread: Thread, id: string, mentioned = true) => { await deliverMessage({ callbacks, endpointId: endpoint.id, thread, message: makeMessage({ id, text: "@maya durable wake test", mentioned, }), trigger: mentioned ? "mention" : "subscribed_message", }); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.state, "received"), ), ); await service.processPendingDeliveries(25); }; await send(firstThread.thread, "9200.1"); const [firstAction] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "inbound_wakeup"), ), ); expect(firstAction).toMatchObject({ status: "issued", result: { code: "inbound_wakeup_retry" }, }); expect( await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, firstAction.id)), ).toEqual([]); await send(firstThread.thread, "9200.2", false); expect(wakeup).toHaveBeenCalledTimes(1); await send( makeThread({ channelId: "C-OUTBOX", id: "slack:C-OUTBOX:9300.1" }).thread, "9300.1", ); expect(wakeup).toHaveBeenCalledTimes(2); await db .update(chatActions) .set({ result: { ...firstAction.result, retryAt: new Date(0).toISOString() }, }) .where(eq(chatActions.id, firstAction.id)); await service.processPendingDeliveries(25, firstAction.deliveryId!); expect(wakeup).toHaveBeenCalledTimes(4); const actions = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "inbound_wakeup"), ), ); expect(actions).toHaveLength(3); expect(actions.every((action) => action.status === "processed")).toBe(true); const firstConversation = actions.filter( (action) => action.conversationId === firstAction.conversationId, ); expect(admittedComments.slice(1)).toEqual([ String(firstAction.payload.commentId), ...firstConversation .filter((action) => action.id !== firstAction.id) .map((action) => String(action.payload.commentId)), ]); expect( await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.companyId, fixture.companyId)), ).toHaveLength(3); await service.shutdown(); }); it("does not dispatch an accepted inbound wake after destination access is revoked", async () => { const fixture = await seedCompany(); const { service, callbacks, endpoint, wakeup } = await configuredSlackEndpoint(fixture, { wakeup: async () => { throw new Error("injected scheduler unavailable"); }, }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: makeThread({ channelId: "C-OUTBOX-REVOKE", id: "slack:C-OUTBOX-REVOKE:9400.1", }).thread, message: makeMessage({ id: "9400.1", text: "@maya work must retain current reach", mentioned: true, }), trigger: "mention", }); const [action] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "inbound_wakeup"), ), ); await db .update(chatEndpointResources) .set({ enabled: false }) .where(eq(chatEndpointResources.endpointId, endpoint.id)); await db .update(chatActions) .set({ result: { ...action.result, retryAt: new Date(0).toISOString() } }) .where(eq(chatActions.id, action.id)); await service.processPendingDeliveries(25, action.deliveryId!); expect(wakeup).toHaveBeenCalledTimes(1); expect( await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.companyId, fixture.companyId)), ).toEqual([]); expect( await db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, action.id)), ).toEqual([ expect.objectContaining({ status: "failed", result: expect.objectContaining({ code: "inbound_wakeup_authorization_changed", }), }), ]); await service.shutdown(); }); it("settles rejected pre-acceptance intent and permits a later authorized external message", async () => { const fixture = await seedCompany(); const { service, callbacks, endpoint, wakeup } = await configuredSlackEndpoint(fixture); const thread = makeThread({ channelId: "C-ACCEPT-REVOKE", id: "slack:C-ACCEPT-REVOKE:9500.1", }); thread.subscribe.mockImplementationOnce(async () => { await db .update(chatEndpointResources) .set({ enabled: false }) .where(eq(chatEndpointResources.endpointId, endpoint.id)); }); await expect( deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: "9500.1", text: "@maya first message", mentioned: true, }), trigger: "mention", }), ).rejects.toThrow("no longer authorized"); const [first] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "inbound_wakeup"), ), ); expect(first).toMatchObject({ status: "failed", result: { code: "inbound_wakeup_delivery_rejected" }, }); expect(wakeup).not.toHaveBeenCalled(); const heldIssues = () => db .select({ id: issues.id }) .from(issues) .where( and( eq(issues.companyId, fixture.companyId), unadmittedChatWakeupCondition(issues.id, issues.companyId), ), ); expect(await heldIssues()).toHaveLength(1); await db .update(chatEndpointResources) .set({ enabled: true }) .where(eq(chatEndpointResources.endpointId, endpoint.id)); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: "9500.2", text: "@maya new authorized request", mentioned: true, }), trigger: "mention", }); expect(wakeup).toHaveBeenCalledTimes(1); expect(await heldIssues()).toEqual([]); // A later execution failure does not erase historical authorization. await db .update(agentWakeupRequests) .set({ status: "failed" }) .where(eq(agentWakeupRequests.companyId, fixture.companyId)); expect(await heldIssues()).toEqual([]); await service.shutdown(); }); it("ignores signed Slack callbacks from a different workspace before SDK dispatch", async () => { const fixture = await seedCompany(); const runtime = new FakeChatSdkRuntime(); const { service } = createService( runtime, fakeSlackFetch("U-BOT-WORKSPACE-SCOPE") as typeof globalThis.fetch, ); const endpoint = await service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); const signingSecret = "workspace-scope-signing-secret"; await service.configure( endpoint.id, { action: "configure", credentials: { botToken: "xoxb-workspace-scope", signingSecret, }, }, "owner-user", ); const endpointRuntime = runtime.endpoints.get(endpoint.id)!; expect(endpointRuntime).toBeDefined(); const signedRequest = (body: string, contentType: string) => { const timestamp = String(Math.floor(Date.now() / 1000)); const signature = `v0=${createHmac("sha256", signingSecret).update(`v0:${timestamp}:${body}`).digest("hex")}`; return new Request( `https://paperclip.example/api/chat-webhooks/${endpoint.publicId}/slack`, { method: "POST", headers: { "content-type": contentType, "x-slack-request-timestamp": timestamp, "x-slack-signature": signature, }, body, }, ); }; const formBody = (values: Record) => new URLSearchParams(values).toString(); const cases = [ { name: "Events API JSON", contentType: "application/json", foreignBody: JSON.stringify({ type: "event_callback", team_id: "T-FOREIGN", event: { type: "app_mention" }, }), localBody: JSON.stringify({ type: "event_callback", team_id: "T-PAPERCLIP", enterprise_id: "E-PAPERCLIP", event: { type: "app_mention" }, }), }, { name: "interactive form payload", contentType: "application/x-www-form-urlencoded", foreignBody: formBody({ payload: JSON.stringify({ type: "block_actions", team: { id: "T-FOREIGN" }, }), }), localBody: formBody({ payload: JSON.stringify({ type: "block_actions", team: { id: "T-PAPERCLIP" }, }), }), }, { name: "slash command", contentType: "application/x-www-form-urlencoded", foreignBody: formBody({ team_id: "T-FOREIGN", command: "/maya", text: "status", }), localBody: formBody({ team_id: "T-PAPERCLIP", command: "/maya", text: "status", }), }, { name: "enterprise-scoped callback", contentType: "application/json", foreignBody: JSON.stringify({ type: "event_callback", enterprise_id: "E-FOREIGN", event: { type: "app_mention" }, }), localBody: JSON.stringify({ type: "event_callback", enterprise_id: "T-PAPERCLIP", event: { type: "app_mention" }, }), }, ]; for (const testCase of cases) { endpointRuntime.webhookRequest = null; const ignored = await service.handleWebhook( endpoint.publicId, "slack", signedRequest(testCase.foreignBody, testCase.contentType), ); expect(ignored.status, testCase.name).toBe(200); await expect(ignored.text()).resolves.toBe("ignored"); expect(endpointRuntime.webhookRequest, testCase.name).toBeNull(); const accepted = await service.handleWebhook( endpoint.publicId, "slack", signedRequest(testCase.localBody, testCase.contentType), ); expect(accepted.status, testCase.name).toBe(202); expect(endpointRuntime.webhookRequest, testCase.name).not.toBeNull(); } // Scope inspection is not a substitute for the adapter's signature gate. // An invalid request must continue to the SDK so it receives the normal // authentication failure instead of Paperclip acknowledging it as foreign. endpointRuntime.webhookRequest = null; const forgedForeign = signedRequest( cases[0]!.foreignBody, cases[0]!.contentType, ); forgedForeign.headers.set("x-slack-signature", "v0=forged"); const forgedResponse = await service.handleWebhook( endpoint.publicId, "slack", forgedForeign, ); expect(forgedResponse.status).toBe(202); expect(endpointRuntime.webhookRequest).not.toBeNull(); await service.shutdown(); }); it("returns a retryable webhook failure when the delivery insert fails before durable receipt", async () => { const fixture = await seedCompany(); const service = chatChannelService(db, { deferWebhookProcessing: true, fetch: fakeSlackFetch("U-BOT-DURABILITY") as typeof globalThis.fetch, heartbeat: { wakeup: receiptBackedWakeup(vi.fn(async () => ({ accepted: true }))), }, publicBaseUrl: "https://paperclip.example", }); const endpoint = await service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); const signingSecret = "durable-ingress-signing-secret"; await service.configure( endpoint.id, { action: "configure", credentials: { botToken: "xoxb-durable-ingress", signingSecret, }, }, "owner-user", ); const body = JSON.stringify({ type: "event_callback", event_id: `Ev-${randomUUID()}`, event_time: Math.floor(Date.now() / 1000), team_id: "T-PAPERCLIP", event: { type: "app_mention", user: "U-EXTERNAL", username: "alex", text: "@maya prove durable receipt", ts: "9200.1", channel: "C-DURABILITY", channel_type: "channel", team: "T-PAPERCLIP", }, }); const timestamp = String(Math.floor(Date.now() / 1000)); const signature = `v0=${createHmac("sha256", signingSecret).update(`v0:${timestamp}:${body}`).digest("hex")}`; const providerRequest = (retry = false) => new Request( `https://paperclip.example/api/chat-webhooks/${endpoint.publicId}/slack`, { method: "POST", headers: { "content-type": "application/json", "x-slack-request-timestamp": timestamp, "x-slack-signature": signature, ...(retry ? { "x-slack-retry-num": "1" } : {}), }, body, }, ); const timingEvents: ChatWebhookDiagnosticEvent[] = []; const observedApp = webhookApp(service, (event) => timingEvents.push(event), ); const observedRequest = (retry = false, signatureOverride?: string) => request(observedApp) .post(`/api/chat-webhooks/${endpoint.publicId}/slack`) .set(Object.fromEntries(providerRequest(retry).headers)) .set("x-slack-signature", signatureOverride ?? signature) .send(body); const forged = await observedRequest(false, "v0=forged"); expect(forged.status).toBe(401); expect( timingEvents.some((event) => event.stage === "durable_receipt"), ).toBe(false); expect(timingEvents.at(-1)?.statusCode).toBe(401); expect( await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)), ).toHaveLength(0); const originalTransaction = db.transaction.bind(db); let injectedAdmissionFailure = false; const transactionSpy = vi .spyOn(db, "transaction") .mockImplementation((async ( ...args: Parameters ) => { const [callback, config] = args; return originalTransaction(async (tx) => { const result = await callback(tx); if (!injectedAdmissionFailure) { const [uncommittedReceipt] = await tx .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.companyId, fixture.companyId), eq(chatDeliveries.endpointId, endpoint.id), sql`${chatDeliveries.normalizedEvent}->'message'->>'providerMessageId' = '9200.1'`, ), ); if (uncommittedReceipt) { // Only the receipt transaction can see its uncommitted insert. // Fail before commit/ack, leaving unrelated transactions intact. injectedAdmissionFailure = true; throw new Error("injected durable admission failure"); } } return result; }, config); }) as typeof db.transaction); try { // A Gateway ownership renewal or another endpoint's work may transact // before this receipt. It must not consume this endpoint's insert fault. await expect( db.transaction(async (tx) => tx .select({ id: chatEndpoints.id }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)), ), ).resolves.toEqual([{ id: endpoint.id }]); expect(injectedAdmissionFailure).toBe(false); timingEvents.length = 0; const rejected = await observedRequest(); expect(injectedAdmissionFailure).toBe(true); expect(rejected.status).toBe(503); expect(rejected.headers["retry-after"]).toBe("1"); expect(timingEvents.map((event) => event.stage)).toEqual([ "http_received", "handler_started", "endpoint_resolved", "runtime_requested", "runtime_ready", "response_ready", "response_finished", ]); expect(timingEvents.at(-1)?.statusCode).toBe(503); expect( await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)), ).toHaveLength(0); transactionSpy.mockRestore(); await service.runtime.removeEndpoint(endpoint.id); timingEvents.length = 0; const acceptedRetry = await observedRequest(true); expect(acceptedRetry.status).toBe(200); // This checks eventual processing after durable acknowledgement, not a // one-second worker SLA. Keep the condition bounded under suite load. await vi.waitFor( async () => { const deliveries = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(deliveries).toHaveLength(1); expect(deliveries[0].state).toBe("processed"); }, { timeout: 5_000 }, ); const [initialDelivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); const initialDuplicateCount = Number( initialDelivery.normalizedEvent.deduplication?.duplicateCount ?? 0, ); expect(initialDuplicateCount).toBe(0); const receiptEvent = timingEvents.find( (event) => event.stage === "durable_receipt", ); expect(receiptEvent).toMatchObject({ endpointId: endpoint.id, receiptId: initialDelivery.id, receiptKind: "message_delivery", slackRetryNumHint: 1, }); expect(timingEvents.map((event) => event.stage)).toEqual([ "http_received", "handler_started", "endpoint_resolved", "runtime_requested", "runtime_initializing", "runtime_ready", "durable_receipt", "response_ready", "response_finished", ]); expect(timingEvents.indexOf(receiptEvent!)).toBeLessThan( timingEvents.findIndex((event) => event.stage === "response_ready"), ); expect(JSON.stringify(timingEvents)).not.toContain(signingSecret); expect(JSON.stringify(timingEvents)).not.toContain( "@maya prove durable receipt", ); expect(JSON.stringify(timingEvents)).not.toContain(signature); // A duplicate redelivery can momentarily contend with the first // delivery's settlement and draw the retryable 503 — that is the // webhook contract (Slack re-sends, the dedup path keeps it // idempotent), not a defect. Retry the way the provider would // instead of asserting an accidental no-contention property; this // exact assertion drew a 503 under CI shard load on 2026-09-10. let acceptedRedelivery = await observedRequest(true); for ( let attempt = 0; acceptedRedelivery.status === 503 && attempt < 20; attempt += 1 ) { await new Promise((resolve) => setTimeout(resolve, 250)); acceptedRedelivery = await observedRequest(true); } expect(acceptedRedelivery.status).toBe(200); await vi.waitFor(async () => { const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect( Number(delivery.normalizedEvent.deduplication?.duplicateCount ?? 0), ).toBeGreaterThan(initialDuplicateCount); expect( await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toHaveLength(1); }); } finally { transactionSpy.mockRestore(); await service.shutdown(); } }); it.each(["success", "endpoint_attention", "resource_unavailable"] as const)( "serializes duplicate Slack admission with an owned provider-reply settlement: %s", async (disposition) => { const fixture = await seedCompany(); const { callbacks, endpoint, service, wakeup } = await configuredSlackEndpoint(fixture); const channel = makeThread({ channelId: "C-DUPLICATE-SETTLEMENT", id: "slack:C-DUPLICATE-SETTLEMENT:9201.1", name: "duplicate-settlement", }); const message = makeMessage({ id: "9201.1", text: "", mentioned: true, }); const providerError = disposition === "success" ? null : Object.assign(new Error(`Synthetic provider reply ${disposition}`), { data: { error: disposition === "endpoint_attention" ? "invalid_auth" : "channel_not_found", }, }); if (providerError) channel.post.mockRejectedValue(providerError); const terminalState = providerError ? "failed" : "processed"; const deliver = () => deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message, trigger: "mention", }); let releaseSettlement!: () => void; const settlementGate = new Promise((resolve) => { releaseSettlement = resolve; }); let settlementPid: number | null = null; let heldDeliveryId: string | null = null; let heldActionId: string | null = null; const originalTransaction = db.transaction.bind(db); type ObservedSession = { prepareQuery(...args: unknown[]): { execute(...args: unknown[]): Promise; }; }; const transactionSpy = vi .spyOn(db, "transaction") .mockImplementation((async ( ...args: Parameters ) => { const [callback, config] = args; return originalTransaction(async (tx) => { const session = (tx as unknown as { session: ObservedSession }) .session; const prepareQuery = session.prepareQuery.bind(session); session.prepareQuery = (...queryArgs) => { const query = queryArgs[0] as { sql: string; params: unknown[] }; const prepared = prepareQuery(...queryArgs); const execute = prepared.execute.bind(prepared); prepared.execute = async (...executeArgs) => { const result = await execute(...executeArgs); if ( heldDeliveryId === null && query.sql.startsWith('update "chat_deliveries" set "state"') && query.params[0] === terminalState ) { const [effect] = await tx .select({ id: chatActions.id, deliveryId: chatActions.deliveryId, }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "provider_effect"), eq(chatActions.status, terminalState), ), ); if ( effect?.deliveryId && query.params.includes(effect.deliveryId) ) { if (providerError) { expect((result as { count?: number }).count).toBe(1); } else { expect(result).toEqual([{ id: effect.deliveryId }]); } heldDeliveryId = effect.deliveryId; heldActionId = effect.id; const [backend] = (await tx.execute( sql`select pg_backend_pid() as pid`, )) as unknown as Array<{ pid: number }>; settlementPid = backend!.pid; // The real provider outcome and its real delivery UPDATE have // completed. Hold only the statement return, not fabricated // ledger state, while a duplicate enters normal admission. await settlementGate; } } return result; }; return prepared; }; return callback(tx); }, config); }) as typeof db.transaction); const outcome = (operation: Promise) => operation.then( () => ({ ok: true as const }), (error: unknown) => ({ ok: false as const, error }), ); const first = outcome(deliver()); let duplicate: ReturnType | undefined; try { await vi.waitFor(() => expect(settlementPid).not.toBeNull()); expect(channel.post).toHaveBeenCalledTimes(1); const [uncommitted] = await db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where(eq(chatDeliveries.id, heldDeliveryId!)); expect(uncommitted?.state).toBe("processing"); duplicate = outcome(deliver()); let blockedQuery: string | null = null; await vi.waitFor(async () => { const rows = (await db.execute(sql` select query from pg_stat_activity where datname = current_database() and ${settlementPid} = any(pg_blocking_pids(pid)) and (query like 'insert into "chat_deliveries"%' or (query like 'select %' and query like '%"chat_endpoints"%for %update%')) `)) as unknown as Array<{ query: string }>; expect(rows).toHaveLength(1); blockedQuery = rows[0]!.query; }); // The old order waits at the delivery unique index while owning the // endpoint; endpoint-first settlement instead makes admission wait at // its endpoint lock. Both are real, observed PostgreSQL dependencies. console.info("duplicate settlement blocked statement", blockedQuery); releaseSettlement(); const outcomes = await Promise.all([first, duplicate]); expect(outcomes).toEqual([ providerError ? { ok: false, error: providerError } : { ok: true }, { ok: true }, ]); const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(delivery).toMatchObject({ id: heldDeliveryId, state: terminalState, normalizedEvent: { deduplication: { duplicateCount: 1 } }, }); expect( await db .select({ id: chatActions.id, status: chatActions.status }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "provider_effect"), ), ), ).toEqual([{ id: heldActionId, status: terminalState }]); if (providerError) { const [effect] = await db .select({ result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, heldActionId!)); expect(effect?.result).toMatchObject({ code: `provider_effect_${disposition}`, attempts: 1, retryable: false, }); if (disposition === "endpoint_attention") { const [currentEndpoint] = await db .select({ status: chatEndpoints.status }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)); expect(currentEndpoint?.status).toBe("attention"); } else { const [resource] = await db .select({ availability: chatEndpointResources.availability }) .from(chatEndpointResources) .where( and( eq(chatEndpointResources.endpointId, endpoint.id), eq( chatEndpointResources.providerResourceId, "C-DUPLICATE-SETTLEMENT", ), ), ); expect(resource?.availability).toBe("unavailable"); } } expect(channel.post).toHaveBeenCalledTimes(1); expect(channel.post).toHaveBeenCalledWith( "Please include a request after mentioning me.", ); expect( await db .select({ id: issues.id }) .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toEqual([]); expect( await db .select({ id: heartbeatRuns.id }) .from(heartbeatRuns) .where(eq(heartbeatRuns.companyId, fixture.companyId)), ).toEqual([]); expect(wakeup).not.toHaveBeenCalled(); } finally { releaseSettlement(); await Promise.all([first, duplicate]); transactionSpy.mockRestore(); await service.shutdown(); } }, ); it("cannot publish a runtime whose initialization is overtaken by pause", async () => { const fixture = await seedCompany(); const { endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const [beforeActivation] = await db .select({ setup: chatEndpoints.setup }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)); await db .update(chatEndpoints) .set({ status: "active", setup: { ...beforeActivation!.setup, step: "complete" }, activatedAt: new Date(), }) .where(eq(chatEndpoints.id, endpoint.id)); await runtime.removeEndpoint(endpoint.id); let releaseInitialization!: () => void; let markInitializationEntered!: () => void; const initializationGate = new Promise((resolve) => { releaseInitialization = resolve; }); const initializationEntered = new Promise((resolve) => { markInitializationEntered = resolve; }); runtime.initializeHook = async () => { markInitializationEntered(); await initializationGate; }; const initializingWebhook = service.handleWebhook( endpoint.publicId, "slack", new Request("https://paperclip.example/slack", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ type: "event_callback", event: {} }), }), ); await initializationEntered; await service.configure(endpoint.id, { action: "pause" }, "owner-user"); releaseInitialization(); await expect(initializingWebhook).rejects.toMatchObject({ status: 409, details: { code: "chat_endpoint_runtime_superseded" }, }); expect(runtime.endpoints.has(endpoint.id)).toBe(false); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "paused", }); }); it("acknowledges durable Slack ingress promptly and never replays paused traffic on resume", async () => { const fixture = await seedCompany(); const deferred: Array<() => void> = []; const wakeup = vi.fn(async () => ({ accepted: true })); const service = chatChannelService(db, { deferWebhookProcessing: true, fetch: fakeSlackFetch("U-BOT-PAUSE") as typeof globalThis.fetch, heartbeat: { wakeup: receiptBackedWakeup(wakeup) }, publicBaseUrl: "https://paperclip.example", scheduleDeferredWork: (task) => deferred.push(task), }); const endpoint = await service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); const signingSecret = "pause-ingress-signing-secret"; await service.configure( endpoint.id, { action: "configure", credentials: { botToken: "xoxb-pause-ingress", signingSecret, }, }, "owner-user", ); await db .update(chatEndpoints) .set({ status: "active", setup: { step: "complete" } }) .where(eq(chatEndpoints.id, endpoint.id)); const signedRequest = (input: { eventId: string; messageId: string; text: string; }) => { const body = JSON.stringify({ type: "event_callback", event_id: input.eventId, event_time: Math.floor(Date.now() / 1000), team_id: "T-PAPERCLIP", event: { type: "app_mention", user: "U-PAUSED-SENDER", username: "alex", text: input.text, ts: input.messageId, channel: "C-PAUSED-INGRESS", channel_type: "channel", team: "T-PAPERCLIP", }, }); const timestamp = String(Math.floor(Date.now() / 1000)); const signature = `v0=${createHmac("sha256", signingSecret).update(`v0:${timestamp}:${body}`).digest("hex")}`; return new Request( `https://paperclip.example/api/chat-webhooks/${endpoint.publicId}/slack`, { method: "POST", headers: { "content-type": "application/json", "x-slack-request-timestamp": timestamp, "x-slack-signature": signature, }, body, }, ); }; const activeResponse = await service.handleWebhook( endpoint.publicId, "slack", signedRequest({ eventId: "Ev-before-pause", messageId: "9300.1", text: "@maya queued just before pause", }), ); expect(activeResponse.status).toBe(200); expect(deferred).toHaveLength(1); expect(wakeup).not.toHaveBeenCalled(); await service.configure(endpoint.id, { action: "pause" }, "owner-user"); const pausedResponse = await service.handleWebhook( endpoint.publicId, "slack", signedRequest({ eventId: "Ev-during-pause", messageId: "9300.2", text: "@maya this must stay ignored after resume", }), ); expect(pausedResponse.status).toBe(200); expect(deferred).toHaveLength(1); await service.configure(endpoint.id, { action: "resume" }, "owner-user"); deferred.shift()?.(); await service.shutdown(); const deliveries = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)) .orderBy(asc(chatDeliveries.receivedAt)); expect(deliveries).toHaveLength(1); expect(deliveries.map((delivery) => delivery.state)).toEqual(["filtered"]); expect(deliveries.map((delivery) => delivery.redactedError)).toEqual([ "Connection was paused before processing", ]); expect(deliveries.every((delivery) => delivery.processedAt)).toBe(true); expect(await service.get(endpoint.id)).toMatchObject({ status: "active" }); expect(wakeup).not.toHaveBeenCalled(); expect( await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toHaveLength(0); expect( await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.companyId, fixture.companyId)), ).toHaveLength(0); expect( await db .select() .from(chatPublications) .where(eq(chatPublications.companyId, fixture.companyId)), ).toHaveLength(0); }); it("keeps processing audit truth and rejects stale Slack runtime callbacks across resume", async () => { const fixture = await seedCompany(); const { runtime, service, wakeup } = createService( new FakeChatSdkRuntime(), fakeSlackFetch() as typeof globalThis.fetch, ); const endpoint = await service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await service.configure( endpoint.id, { action: "configure", credentials: { botToken: "xoxb-generation-boundary", signingSecret: "generation-boundary-secret", }, }, "owner-user", ); await recordSlackUrlVerification(service, endpoint.publicId); await service.configure(endpoint.id, { action: "verify" }, "owner-user"); const staleCallbacks = runtime.configurations.get(endpoint.id)?.callbacks; if (!staleCallbacks) throw new Error("Expected endpoint callbacks"); const [configuredSetup] = await db .select({ setup: chatEndpoints.setup }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)); await db .update(chatEndpoints) .set({ status: "active", setup: { ...configuredSetup!.setup, step: "complete" }, activatedAt: new Date(), }) .where(eq(chatEndpoints.id, endpoint.id)); const thread = makeThread({ channelId: "C-PAUSE-RACE", id: "slack:C-PAUSE-RACE:9400.1", name: "pause-race", }); const processingMessage = makeMessage({ id: "9400.1", text: "@maya processing before pause", mentioned: true, }); const providerEventId = `${thread.thread.id}:${processingMessage.id}`; await db.insert(chatDeliveries).values({ companyId: fixture.companyId, endpointId: endpoint.id, providerEventId, deduplicationKey: createHash("sha256") .update(providerEventId) .digest("hex"), eventKind: "mention", normalizedEvent: {}, state: "processing", attempts: 1, }); await service.configure(endpoint.id, { action: "pause" }, "owner-user"); await deliverMessage({ callbacks: staleCallbacks, endpointId: endpoint.id, thread: thread.thread, message: processingMessage, trigger: "mention", }); const [processingDuplicate] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.providerEventId, providerEventId)); expect(processingDuplicate.state).toBe("processing"); expect(processingDuplicate.processedAt).toBeNull(); expect( Number( processingDuplicate.normalizedEvent.deduplication?.duplicateCount ?? 0, ), ).toBe(1); await service.configure(endpoint.id, { action: "resume" }, "owner-user"); const staleMessage = makeMessage({ id: "9400.2", text: "@maya parsed before the pause", mentioned: true, }); await deliverMessage({ callbacks: staleCallbacks, endpointId: endpoint.id, thread: thread.thread, message: staleMessage, trigger: "mention", }); await staleCallbacks.onMessageUpdated?.({ endpointId: endpoint.id, provider: "slack", thread: thread.thread, message: makeMessage({ id: "9400.3", text: "stale edit" }), } as never); await staleCallbacks.onMessageDeleted?.({ endpointId: endpoint.id, provider: "slack", event: { threadId: thread.thread.id, messageId: "9400.4", deletedAt: new Date(), }, } as never); await staleCallbacks.onReaction?.({ endpointId: endpoint.id, provider: "slack", event: {}, } as never); await staleCallbacks.onSlashCommand?.({ endpointId: endpoint.id, provider: "slack", event: {}, } as never); await expect( staleCallbacks.onAction!({ endpointId: endpoint.id, provider: "slack", event: {}, } as never), ).rejects.toMatchObject({ status: 403 }); await expect( staleCallbacks.onModalSubmit!({ endpointId: endpoint.id, provider: "slack", event: { adapter: {} as never, callbackId: "stale-callback", raw: {}, user: { userId: "U-STALE-CALLBACK", userName: "stale-callback", fullName: "Stale Callback", isBot: false, isMe: false, isSystem: false, }, values: {}, viewId: "stale-view", }, } as never), ).resolves.toEqual({ action: "clear" }); const deliveries = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)) .orderBy(asc(chatDeliveries.receivedAt)); expect(deliveries).toHaveLength(3); expect(deliveries[0]).toMatchObject({ state: "processing", processedAt: null, }); expect(deliveries[1]).toMatchObject({ state: "filtered", redactedError: "Connection activation changed before admission", processedAt: expect.any(Date), }); expect(deliveries[2]).toMatchObject({ eventKind: "action", state: "filtered", redactedError: "External chat modal submission denied by Paperclip", processedAt: expect.any(Date), }); expect(JSON.stringify(await service.get(endpoint.id))).not.toContain( "runtimeGeneration", ); expect(wakeup).not.toHaveBeenCalled(); expect( await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toHaveLength(0); const freshCallbacks = runtime.configurations.get(endpoint.id)?.callbacks; if (!freshCallbacks) throw new Error("Expected resumed endpoint callbacks"); const freshThread = makeThread({ channelId: "C-POST-RESUME", id: "slack:C-POST-RESUME:9500.1", isDM: true, name: "post-resume", }); const freshMessage = makeMessage({ id: "9500.1", text: "@maya accepted after resume", mentioned: true, }); await deliverMessage({ callbacks: freshCallbacks, endpointId: endpoint.id, thread: freshThread.thread, message: freshMessage, trigger: "mention", }); await service.processPendingDeliveries(); await vi.waitFor(() => expect(wakeup).toHaveBeenCalledTimes(1)); expect( await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toHaveLength(1); await service.shutdown(); }); it("recovers the visible receipt exactly once after wake acceptance contention", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredSlackEndpoint(fixture); const first = makeThread({ channelId: "C-RECEIPT-RETRY", id: "slack:C-RECEIPT-RETRY:1000.1", name: "receipt-retry", }); const message = makeMessage({ id: "1000.1", text: "@maya acknowledge this after retry", mentioned: true, }); let releaseLock!: () => void; let lockEntered!: () => void; const holdLock = new Promise((resolve) => { releaseLock = resolve; }); const entered = new Promise((resolve) => { lockEntered = resolve; }); let lockTransaction: Promise | undefined; first.subscribe.mockImplementationOnce(async () => { lockTransaction = db.transaction(async (tx) => { await tx .select({ id: chatEndpoints.id }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)) .for("update"); lockEntered(); await holdLock; }); await entered; }); try { await expect( deliverMessage({ callbacks, endpointId: endpoint.id, thread: first.thread, message, trigger: "mention", }), ).rejects.toMatchObject({ cause: { code: "55P03" } }); expect(wakeup).not.toHaveBeenCalled(); expect(runtime.endpoints.get(endpoint.id)?.reactions).toEqual([]); } finally { releaseLock(); await lockTransaction; } await vi.waitFor( async () => { await service.processPendingDeliveries(); expect(wakeup).toHaveBeenCalledTimes(1); expect(runtime.endpoints.get(endpoint.id)?.reactions).toEqual([ { threadId: first.thread.id, messageId: message.id, emoji: "eyes" }, ]); }, { timeout: 10_000 }, ); await service.processPendingDeliveries(); expect(wakeup).toHaveBeenCalledTimes(1); expect(runtime.endpoints.get(endpoint.id)?.reactions).toHaveLength(1); const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(delivery).toMatchObject({ state: "processed" }); expect( await db .select() .from(issueComments) .where(eq(issueComments.companyId, fixture.companyId)), ).toHaveLength(1); await service.shutdown(); }); it("deduplicates inbound events, keeps one task per thread, and requires enablement for newly discovered channels", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredSlackEndpoint(fixture); const serializedEndpointResponses = JSON.stringify({ detail: await service.get(endpoint.id), list: await service.list(fixture.companyId), }); expect(serializedEndpointResponses).not.toContain("xoxb-test-token"); expect(serializedEndpointResponses).not.toContain("test-signing-secret"); const first = makeThread({ channelId: "C-ENGINEERING", id: "slack:C-ENGINEERING:1000.1", name: "engineering", }); const firstMessage = makeMessage({ id: "1000.1", text: "@maya investigate the deploy", mentioned: true, }); const duplicateResults = await Promise.allSettled( Array.from({ length: 12 }, () => deliverMessage({ callbacks, endpointId: endpoint.id, thread: first.thread, message: firstMessage, trigger: "mention", }), ), ); // Concurrent duplicate receipts can briefly hold the endpoint while the // issue-first wake acceptance checks it with NOWAIT. The direct SDK test // callback surfaces that contention; its durable delivery must still drain // to exactly one wake rather than requiring every synchronous call to win. for (const result of duplicateResults) { if (result.status === "rejected") { expect(result.reason).toMatchObject({ cause: { code: "55P03" } }); } } await deliverMessage({ callbacks, endpointId: endpoint.id, thread: first.thread, message: firstMessage, trigger: "mention", }); await vi.waitFor( async () => { await service.processPendingDeliveries(); expect(wakeup).toHaveBeenCalledTimes(1); }, { timeout: 10_000 }, ); let conversations = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); let endpointIssues = await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)); let deliveries = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); let comments = await db .select() .from(issueComments) .where(eq(issueComments.issueId, conversations[0].issueId)) .orderBy(asc(issueComments.createdAt), asc(issueComments.id)); expect(conversations).toHaveLength(1); expect(endpointIssues).toHaveLength(1); expect(deliveries).toHaveLength(1); expect(deliveries[0].normalizedEvent).toMatchObject({ deduplication: { duplicateCount: 12, lastDuplicateAt: expect.any(String), }, }); expect(comments.map((comment) => comment.body)).toEqual([ "@maya investigate the deploy", ]); expect(runtime.endpoints.get(endpoint.id)?.reactions).toContainEqual({ threadId: first.thread.id, messageId: firstMessage.id, emoji: "eyes", }); // Receipt reactions deliberately resolve the current credential-fenced // runtime instead of reusing an inbound callback's potentially stale // adapter object. expect(first.addReaction).not.toHaveBeenCalled(); expect(first.startTyping).not.toHaveBeenCalled(); expect(first.subscribe).toHaveBeenCalledTimes(1); expect(wakeup).toHaveBeenCalledTimes(1); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: first.thread, message: makeMessage({ id: "1000.2", text: "Here is another detail" }), trigger: "subscribed_message", }); conversations = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); endpointIssues = await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)); comments = await db .select() .from(issueComments) .where(eq(issueComments.issueId, conversations[0].issueId)) .orderBy(asc(issueComments.createdAt), asc(issueComments.id)); expect(conversations).toHaveLength(1); expect(endpointIssues).toHaveLength(1); expect(comments.map((comment) => comment.body)).toEqual([ "@maya investigate the deploy", "Here is another detail", ]); const second = makeThread({ channelId: "C-FINANCE", id: "slack:C-FINANCE:2000.1", name: "finance", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: second.thread, message: makeMessage({ id: "2000.1", text: "@maya summarize spend", mentioned: true, }), trigger: "mention", }); const resources = await service.listResources(endpoint.id); expect( resources.map((resource) => ({ label: resource.label, enabled: resource.enabled, })), ).toEqual([ { label: "engineering", enabled: true }, { label: "finance", enabled: false }, ]); deliveries = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect( deliveries.find((delivery) => delivery.providerEventId.includes("2000.1")) ?.state, ).toBe("filtered"); expect( await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toHaveLength(1); const finance = resources.find((resource) => resource.label === "finance"); if (!finance) throw new Error("Expected discovered finance resource"); await service.replaceResources(endpoint.id, [ { id: finance.id, enabled: true }, ]); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: second.thread, message: makeMessage({ id: "2000.2", text: "@maya summarize spend", mentioned: true, }), trigger: "mention", }); expect( await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)), ).toHaveLength(2); expect( await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toHaveLength(2); }); it("acknowledges a provider question when the board resolves after callback validation", async () => { const fixture = await seedCompany(); let enterQuestionResolution!: () => void; let releaseQuestionResolution!: () => void; const questionResolutionEntered = new Promise((resolve) => { enterQuestionResolution = resolve; }); const questionResolutionGate = new Promise((resolve) => { releaseQuestionResolution = resolve; }); const context = await configuredSlackEndpoint(fixture, { questionResolutionPersistBarrier: async () => { enterQuestionResolution(); await questionResolutionGate; }, }); const { callbacks, endpoint, service } = context; if (!callbacks.onAction) throw new Error("Expected Slack action callback"); const externalUserId = `U-QUESTION-RACE-${randomUUID()}`; const channel = makeThread({ channelId: "C-QUESTION-RACE", id: `slack:C-QUESTION-RACE:${randomUUID()}`, name: "question-race", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: randomUUID(), text: "@maya choose during a race", mentioned: true, userId: externalUserId, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const principal = await db .select() .from(chatExternalPrincipals) .where(eq(chatExternalPrincipals.externalId, externalUserId)) .then((rows) => rows[0]); const intent = await service.createLinkIntent( endpoint.id, principal.id, 1_800, ); const token = new URL(intent.confirmationUrl).searchParams.get("token"); if (!token) throw new Error("Question-race identity token was absent"); await service.confirmIdentityLink(token, "owner-user"); const interaction = await issueThreadInteractionService(db).create( { id: conversation.issueId, companyId: fixture.companyId }, { kind: "ask_user_questions", continuationPolicy: "none", title: "Race choice", payload: { version: 1, questions: [ { id: "choice", prompt: "Choose one", selectionMode: "single", required: true, allowOther: false, options: [ { id: "one", label: "One" }, { id: "two", label: "Two" }, ], }, ], }, }, { agentId: fixture.assignedAgentId }, ); await service.processPendingPublications(); const publication = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), eq( chatPublications.idempotencyKey, `interaction:${interaction.id}:${endpoint.id}`, ), ), ) .then((rows) => rows[0]); const action = publication.payload.card?.actions?.find( (candidate) => candidate.type === "callback" && candidate.label === "One", ); if ( !publication.providerMessageId || !action || action.type !== "callback" ) { throw new Error("Question-race callback was not published"); } const callback = callbacks.onAction({ endpointId: endpoint.id, provider: "slack", event: { actionId: action.actionId, adapter: {} as never, messageId: publication.providerMessageId, openModal: async () => undefined, raw: { type: "block_actions" }, thread: channel.thread, threadId: `slack:C-QUESTION-RACE:${publication.providerMessageId}`, user: { userId: externalUserId, userName: "question-racer", fullName: "Question Racer", isBot: false, isMe: false, isSystem: false, }, value: interaction.id, }, }); await questionResolutionEntered; await issueThreadInteractionService(db).answerQuestions( { id: conversation.issueId, companyId: fixture.companyId }, interaction.id, { answers: [{ questionId: "choice", optionIds: ["two"] }] }, { userId: "owner-user" }, ); releaseQuestionResolution(); await expect(callback).resolves.toBeUndefined(); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.providerActionId, action.actionId)), ).resolves.toEqual([ { status: "expired", result: { code: "interaction_resolved_elsewhere" }, }, ]); await expect( db .select() .from(activityLog) .where( and( eq(activityLog.companyId, fixture.companyId), eq(activityLog.action, "issue.thread_interaction_answered"), eq(activityLog.entityId, conversation.issueId), ), ), ).resolves.toHaveLength(0); await service.shutdown(); }); it("cleans only the GitHub final run's receipt and never re-adds it on replay", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredGitHubEndpoint(fixture, { scheduleDeferredWork: () => undefined, }); try { await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, endpoint.id)); const thread = makeThread({ channelId: "github:paperclipai/paperclip", id: "github:paperclipai/paperclip:issue:5", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id: "5603841952", text: "@maya answer this fresh request", mentioned: true, }), trigger: "mention", }); const [conversation] = await service.listConversations(endpoint.id); const runId = randomUUID(); await db .insert(heartbeatRuns) .values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "github", providerMessageId: "5603841952", }), }); await addSelectedChatFinal({ companyId: fixture.companyId, issueId: conversation.issueId, agentId: fixture.assignedAgentId, runId, body: "GITHUB-RECEIPT-FINAL", }); await service.processPendingPublications(); const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); const [removal] = await db .select() .from(chatActions) .where( eq( chatActions.providerActionId, `receipt_reaction_remove:${delivery.id}`, ), ); expect(removal).toBeDefined(); await service.processPendingReceiptReactions(1, removal!.id); await service.processPendingReceiptReactions(1, removal!.id); expect(runtime.endpoints.get(endpoint.id)?.removedReactions).toEqual([ { threadId: thread.thread.id, messageId: "5603841952", emoji: "eyes" }, ]); const [add] = await db .select() .from(chatActions) .where( eq(chatActions.providerActionId, `receipt_reaction:${delivery.id}`), ); await db .update(chatActions) .set({ status: "failed", result: { retryable: true, retryAt: new Date(0).toISOString() }, }) .where(eq(chatActions.id, add.id)); await service.processPendingReceiptReactions(1, add.id); expect(runtime.endpoints.get(endpoint.id)?.reactions).toHaveLength(1); expect( runtime.endpoints .get(endpoint.id) ?.posts.filter((post) => post.text === "GITHUB-RECEIPT-FINAL"), ).toHaveLength(1); } finally { await retirePublicationFixture(service, endpoint.id); } }); it.each([ "ordinary", "final_before_add", "held_token", "held_add", "newer_followup", "restart", "rotation", "remove_retry", "unknown_bot", "incomplete_page", "connection_revoked", ] as const)( "GitHub terminal receipt composes pinned HTTP and durable source authority (%s)", async (mode) => { const fixture = await seedCompany(); let earlyFinal: (() => Promise) | undefined; const { callbacks, endpoint, runtime, service, wakeup, providerFetch } = await configuredGitHubEndpoint(fixture, { scheduleDeferredWork: () => undefined, ...(mode === "final_before_add" ? { wakeup: async () => { await earlyFinal!(); return { accepted: true }; }, } : {}), }); let activeService = service; const pins: ChatSdkEndpointRuntime[] = []; const calls: Array<{ method: string; path: string }> = []; const remote = new Map(); let failRemoval = mode === "remove_retry"; let removing = false; let entered!: () => void; const ready = new Promise((resolve) => { entered = resolve; }); let release!: () => void; const held = new Promise((resolve) => { release = resolve; }); let heldOnce = false; const inFlight: Promise[] = []; const configuration = runtime.configurations.get(endpoint.id)!; if ( configuration.providerConfig.provider !== "github" || !("appId" in configuration.providerConfig.credentials) ) throw new Error("Expected GitHub App runtime"); const appId = Number(configuration.providerConfig.credentials.appId); const requestFetch: typeof fetch = async (input, init = {}) => { const url = new URL(String(input)); calls.push({ method: init.method ?? "GET", path: url.pathname }); if ( !heldOnce && ((mode === "held_token" && url.pathname.includes("/access_tokens")) || (mode === "held_add" && init.method === "POST" && url.pathname.endsWith("/reactions"))) ) { heldOnce = true; entered(); await new Promise((resolve, reject) => { const abort = () => reject(new Error("synthetic held receipt deadline")); init.signal!.addEventListener("abort", abort, { once: true }); void held.then(() => { init.signal!.removeEventListener("abort", abort); resolve(); }); }); } if (url.pathname.includes("/access_tokens")) return Response.json( { token: "synthetic-joined-token" }, { status: 201 }, ); if (url.pathname === "/app") return Response.json({ id: appId, slug: "receipt-fixture" }); if (url.pathname.startsWith("/users/")) return Response.json({ id: removing && mode === "unknown_bot" ? null : 9001, login: "receipt-fixture[bot]", type: "Bot", }); const match = /\/issues\/comments\/(\d+)\/reactions(?:\/(\d+))?$/.exec( url.pathname, ); if (!match) throw new Error("Unexpected synthetic GitHub receipt route"); if (init.method === "POST") { const id = match[1] === "5603841952" ? 700 : 701; remote.set(match[1]!, id); return Response.json( { id, content: "eyes", user: { id: 9001 } }, { status: 201 }, ); } if (init.method === "DELETE") { if (failRemoval) { failRemoval = false; return Response.json( {}, { status: 429, headers: { "retry-after": "60" } }, ); } expect(Number(match[2])).toBe(remote.get(match[1]!)); remote.delete(match[1]!); return new Response(null, { status: 204 }); } if (mode === "connection_revoked") await db .update(toolConnections) .set({ enabled: false }) .where(eq(toolConnections.id, endpoint.connectionId)); if (mode === "incomplete_page") return Response.json( url.searchParams.get("page") === "1" ? Array.from({ length: 100 }, (_, index) => ({ id: 1000 + index, content: "eyes", user: { id: 10000 + index }, })) : null, ); return Response.json([ { id: 702, content: "eyes", user: { id: 9002 } }, ...(remote.has(match[1]!) ? [{ id: remote.get(match[1]!), content: "eyes", user: { id: 9001 } }] : []), ]); }; const attach = (target: FakeEndpointRuntime) => { const pinned = createChatSdkEndpointRuntime({ ...configuration, logger: "silent", }); pins.push(pinned); Object.assign(target, { applyGitHubReceiptReaction: ( input: Parameters[0], check: () => Promise, ) => pinned.applyGitHubReceiptReaction(input, check, requestFetch), }); }; const originalRuntime = runtime.endpoints.get(endpoint.id)!; attach(originalRuntime); const thread = makeThread({ channelId: "github:paperclipai/paperclip", id: "github:paperclipai/paperclip:issue:5", }); const runId = randomUUID(); const publish = async () => { const [conversation] = await service.listConversations(endpoint.id); await db .insert(heartbeatRuns) .values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "github", providerMessageId: "5603841952", }), }); await addSelectedChatFinal({ companyId: fixture.companyId, issueId: conversation.issueId, agentId: fixture.assignedAgentId, runId, body: "PINNED-GITHUB-RECEIPT-FINAL", }); await service.processPendingPublications(); }; earlyFinal = publish; try { await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, endpoint.id)); const deliveryWork = deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id: "5603841952", text: "@maya answer exactly once", mentioned: true, }), trigger: "mention", }); inFlight.push(deliveryWork); if (mode === "held_token" || mode === "held_add") { await ready; const finalWork = publish(); inFlight.push(finalWork); await vi.waitFor(async () => { const rows = await db .select({ state: chatPublications.state }) .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), eq(chatPublications.state, "streaming"), ), ); expect(rows).toHaveLength(1); }); // Both the final and cleanup require the sender's credential lease; // neither can overtake a still-active add HTTP request. expect( originalRuntime.posts.filter( (post) => post.text === "PINNED-GITHUB-RECEIPT-FINAL", ), ).toHaveLength(0); expect(calls.some((call) => call.method === "DELETE")).toBe(false); const pendingRemovals = await db .select({ id: chatActions.id }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), like(chatActions.providerActionId, "receipt_reaction_remove:%"), ), ); expect(pendingRemovals).toHaveLength(0); release(); await Promise.all([deliveryWork, finalWork]); } else { await deliveryWork; if (mode !== "final_before_add") await publish(); } const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); const [removal] = await db .select() .from(chatActions) .where( eq( chatActions.providerActionId, `receipt_reaction_remove:${delivery.id}`, ), ); expect(removal).toBeDefined(); if (mode !== "final_before_add") expect(removal.payload.githubReceipt).toEqual({ botUserId: "9001", reactionId: "700", }); if (mode === "newer_followup") await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id: "5603841953", text: "A separate fresh queued question", }), trigger: "subscribed_message", }); removing = true; if (["restart", "unknown_bot"].includes(mode)) { await service.shutdown(); const next = createService(new FakeChatSdkRuntime(), providerFetch, { scheduleDeferredWork: () => undefined, }); next.runtime.initializeHook = async (id) => { if (id === endpoint.id) attach(next.runtime.endpoints.get(id)!); }; activeService = next.service; } if (mode === "rotation") { const [current] = await db .select() .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)); await db .update(chatEndpoints) .set({ setup: { ...current.setup, runtimeGeneration: Number(current.setup.runtimeGeneration ?? 0) + 1, }, }) .where(eq(chatEndpoints.id, endpoint.id)); } await activeService.processPendingReceiptReactions(1, removal.id); if (mode === "remove_retry") { const [failed] = await db .select() .from(chatActions) .where(eq(chatActions.id, removal.id)); expect(failed).toMatchObject({ status: "failed", result: { retryable: true, attempts: 1 }, }); await db .update(chatActions) .set({ result: { ...failed.result, retryAt: new Date(0).toISOString() }, }) .where(eq(chatActions.id, removal.id)); } await activeService.processPendingReceiptReactions(1, removal.id); const [result] = await db .select() .from(chatActions) .where(eq(chatActions.id, removal.id)); const blocked = [ "unknown_bot", "incomplete_page", "connection_revoked", ].includes(mode); expect(result.status).toBe( mode === "rotation" ? "cancelled" : blocked ? "failed" : "processed", ); expect(remote.has("5603841952")).toBe(blocked || mode === "rotation"); expect(remote.has("5603841953")).toBe(mode === "newer_followup"); expect(calls.filter((call) => call.method === "DELETE")).toHaveLength( blocked || mode === "rotation" || mode === "final_before_add" ? 0 : mode === "remove_retry" ? 2 : 1, ); expect( calls.filter( (call) => call.method === "POST" && call.path.endsWith("/reactions"), ), ).toHaveLength( mode === "final_before_add" ? 0 : mode === "newer_followup" ? 2 : 1, ); expect( originalRuntime.posts.filter( (post) => post.text === "PINNED-GITHUB-RECEIPT-FINAL", ), ).toHaveLength(1); expect(wakeup).toHaveBeenCalledTimes(mode === "newer_followup" ? 2 : 1); } finally { release(); await Promise.allSettled(inFlight); try { await retirePublicationFixture(activeService, endpoint.id); } finally { if (activeService !== service) await service.shutdown(); await Promise.all(pins.map((pin) => pin.shutdown())); } } }, ); it.each([ { label: "records a successful GitHub receipt reaction once", failures: [] as Error[], terminalFailure: null as Error | null, expectedAttempts: 1, expectedDiagnostic: null as RegExp | null, }, { label: "keeps an accepted task authoritative after a GitHub 403", failures: [] as Error[], terminalFailure: Object.assign(new Error("GitHub reaction forbidden"), { status: 403, }), expectedAttempts: 1, expectedDiagnostic: /Receipt reaction failed after 1 attempt \(endpoint_attention\): GitHub reaction forbidden/, }, { label: "retries a GitHub 429 receipt reaction with bounded backoff", failures: [ Object.assign(new Error("GitHub reaction rate limited"), { status: 429, retryAfterMs: 1, }), ], terminalFailure: null as Error | null, expectedAttempts: 2, expectedDiagnostic: null as RegExp | null, }, { label: "retries idempotent GitHub receipt reactions after network errors", failures: [ Object.assign(new Error("connection reset before response"), { code: "ECONNRESET", name: "NetworkError", }), Object.assign(new Error("temporary network timeout"), { code: "ETIMEDOUT", name: "NetworkError", }), ], terminalFailure: null as Error | null, expectedAttempts: 3, expectedDiagnostic: null as RegExp | null, }, ])( "$label", async ({ expectedAttempts, expectedDiagnostic, failures, terminalFailure, }) => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredGitHubEndpoint(fixture); const endpointRuntime = runtime.endpoints.get(endpoint.id); if (!endpointRuntime) throw new Error("Expected GitHub runtime"); const github = makeThread({ channelId: "github:paperclipai/paperclip", id: "github:paperclipai/paperclip:issue:receipt-reaction", name: "paperclipai/paperclip", }); for (const failure of failures) { endpointRuntime.reactionErrors.push(failure); } if (terminalFailure) { endpointRuntime.reactionErrors.push(terminalFailure); } await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: github.thread, message: makeMessage({ id: "880011", text: "@maya verify receipt reaction reliability", mentioned: true, }), trigger: "mention", }); for (let attempt = 1; attempt < expectedAttempts; attempt += 1) { const pending = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "receipt_reaction"), ), ) .then((rows) => rows[0]); if (!pending) throw new Error("Expected durable receipt reaction"); await db .update(chatActions) .set({ result: { ...pending.result, retryAt: new Date(0).toISOString() }, updatedAt: new Date(), }) .where(eq(chatActions.id, pending.id)); await service.processPendingReceiptReactions(1, pending.id); } expect(endpointRuntime.reactionErrors).toHaveLength(0); expect(endpointRuntime.reactions).toEqual( terminalFailure ? [] : [ { threadId: github.thread.id, messageId: "880011", emoji: "eyes", }, ], ); const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(delivery).toMatchObject({ state: "processed", attempts: 1 }); expect(wakeup).toHaveBeenCalledTimes(1); expect( await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toHaveLength(1); const activity = (await service.listActivity(endpoint.id)).find( (item) => item.id === delivery.id, ); await expect( db .select({ result: chatActions.result, status: chatActions.status }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "receipt_reaction"), ), ), ).resolves.toEqual([ { status: terminalFailure ? "failed" : "processed", result: expect.objectContaining({ attempts: expectedAttempts }), }, ]); if (expectedDiagnostic) { expect(delivery.redactedError).toMatch(expectedDiagnostic); expect(activity?.detail).toMatch(expectedDiagnostic); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "verifying", }); } else { expect(delivery.redactedError).toBeNull(); expect(activity?.detail).toBeNull(); } await service.shutdown(); }, ); it.each([ "ordinary", "final_before_add", "newer_followup", "same_source_retry", ] as const)( "retires only the exact Slack final's receipt (%s)", async (mode) => { const fixture = await seedCompany(); const statusCalls: string[] = []; const ordinaryFetch = fakeSlackFetch(); let earlyFinal: | (( agentId: string, opts: Parameters< ChatChannelServiceOptions["heartbeat"]["wakeup"] >[1], ) => Promise) | undefined; const context = await configuredSlackEndpoint(fixture, { scheduleDeferredWork: () => undefined, fetch: async (input, init) => { if ( String(input) === "https://slack.com/api/agents.sessions.setStatus" ) statusCalls.push(JSON.parse(String(init?.body)).status); return await ordinaryFetch(input); }, ...(mode === "final_before_add" ? { wakeup: async (agentId, opts) => await earlyFinal!(agentId, opts), } : {}), }); const { callbacks, endpoint, runtime, service, wakeup } = context; const providerRuntime = runtime.endpoints.get(endpoint.id)!; const thread = makeThread({ channelId: "CCLEANUP", id: "slack:CCLEANUP:1740000021.1", }); const runId = randomUUID(); const publishFinal = async () => { const [conversation] = await service.listConversations(endpoint.id); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "slack", providerMessageId: "1740000021.1", }), }); await addSelectedChatFinal({ agentId: fixture.assignedAgentId, body: "SLACK-RECEIPT-FINAL", companyId: fixture.companyId, issueId: conversation.issueId, runId, }); await service.processPendingPublications(); }; earlyFinal = async (agentId, opts) => { const request = opts.durableChatRequest!; await db.transaction(async (tx) => { await request.authorize(tx as never); await tx.insert(agentWakeupRequests).values({ id: request.id, companyId: request.companyId, agentId, source: opts.source!, triggerDetail: opts.triggerDetail, reason: opts.reason, payload: opts.payload, requestedByActorType: opts.requestedByActorType, requestedByActorId: opts.requestedByActorId, idempotencyKey: request.idempotencyKey, requestedAt: request.requestedAt, status: "queued", }); }); await expect( db .select({ id: chatActions.id }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "receipt_reaction"), ), ), ).resolves.toHaveLength(0); await publishFinal(); return { runId }; }; try { await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: "1740000021.1", text: "@maya reply exactly", mentioned: true, }), trigger: "mention", }); if (mode !== "final_before_add") await publishFinal(); const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); const [removal] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq( chatActions.providerActionId, `receipt_reaction_remove:${delivery.id}`, ), ), ); expect(removal).toBeDefined(); if (mode === "newer_followup") { await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: "1740000022.2", text: "Newer follow-up must retain its own receipt", }), trigger: "subscribed_message", }); } await service.processPendingReceiptReactions(1, removal!.id); await service.processPendingReceiptReactions(1, removal!.id); expect(providerRuntime.removedReactions).toEqual([ { threadId: thread.thread.id, messageId: "1740000021.1", emoji: "eyes", }, ]); const expectedAdds = mode === "final_before_add" ? [] : [ { threadId: thread.thread.id, messageId: "1740000021.1", emoji: "eyes", }, ]; if (mode === "newer_followup") expectedAdds.push({ threadId: thread.thread.id, messageId: "1740000022.2", emoji: "eyes", }); expect(providerRuntime.reactions).toEqual(expectedAdds); const [add] = await db .select() .from(chatActions) .where( eq(chatActions.providerActionId, `receipt_reaction:${delivery.id}`), ); await db .update(chatActions) .set({ status: "failed", result: { retryable: true, retryAt: new Date(0).toISOString() }, }) .where(eq(chatActions.id, add.id)); await service.processPendingReceiptReactions(1, add.id); expect(providerRuntime.reactions).toEqual(expectedAdds); if (mode === "same_source_retry") { const [conversation] = await service.listConversations(endpoint.id); const retryRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: retryRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, runtimeMode: "native", status: "running", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "slack", providerMessageId: "1740000021.1", }), }); await enqueueChatRunMilestones(db); await service.processPendingPublications(); const [statusAction] = await db .select({ id: chatActions.id }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slack_session_sync"), ), ); await service.processPendingSlackSessionSyncs(1, statusAction.id); expect(statusCalls.at(-1)).toBe("processing"); // The retry has native working status, not a reminted acknowledgement // for the same original provider message. Late add replay stays inert. await service.processPendingDeliveries(25, delivery.id); expect(providerRuntime.reactions).toEqual(expectedAdds); expect(providerRuntime.removedReactions).toHaveLength(1); } expect( [...providerRuntime.posts, ...providerRuntime.edits].filter( (entry) => entry.text === "SLACK-RECEIPT-FINAL", ), ).toHaveLength(1); expect(wakeup).toHaveBeenCalledTimes(mode === "newer_followup" ? 2 : 1); } finally { // Retire the synthetic still-running retry's conversation so later // globally-scanned milestone fixtures cannot inherit this test's work. if (mode === "same_source_retry") await db .update(chatConversations) .set({ state: "completed" }) .where(eq(chatConversations.endpointId, endpoint.id)); await retirePublicationFixture(service, endpoint.id); } }, ); it.each([429, 503])( "does not accept a contradictory HTTP%s Slack duplicate receipt", async (status) => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture, { scheduleDeferredWork: () => undefined, }); const pinned = createChatSdkEndpointRuntime({ ...runtime.configurations.get(endpoint.id)!, logger: "silent", }); Object.assign(runtime.endpoints.get(endpoint.id)!, { applySlackReceiptReaction: ( input: Parameters[0], ) => pinned.applySlackReceiptReaction(input, async () => Response.json( { ok: false, error: "already_reacted" }, { status, headers: { "retry-after": "60" } }, ), ), }); try { const thread = makeThread({ channelId: "CRECEIPTDENY", id: "slack:CRECEIPTDENY:1740000041.1", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: "1740000041.1", text: "@maya receipt must be coherent", mentioned: true, }), trigger: "mention", }); const [action] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "receipt_reaction"), ), ); expect(action.status).toBe("failed"); expect(action.result?.code).not.toBe( "receipt_reaction_already_applied", ); expect(action.result?.retryable).toBe(true); } finally { try { await pinned.shutdown(); } finally { await retirePublicationFixture(service, endpoint.id); } } }, ); it("bounds a held Slack receipt before delivering an already-ready same-endpoint final", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture, { scheduleDeferredWork: () => undefined, }); const providerRuntime = runtime.endpoints.get(endpoint.id)!; const pinned = createChatSdkEndpointRuntime({ ...runtime.configurations.get(endpoint.id)!, logger: "silent", }); let started!: () => void; const start = new Promise((resolve) => { started = resolve; }); let release!: () => void; let localTransportSettled = false; let aborted = false; const hold = async (signal?: AbortSignal | null) => { await new Promise((resolve, reject) => { const stop = () => { aborted = true; localTransportSettled = true; reject(new Error("synthetic local transport aborted")); }; release = () => { signal?.removeEventListener("abort", stop); localTransportSettled = true; resolve(); }; signal?.addEventListener("abort", stop, { once: true }); started(); }); }; const originalThread = providerRuntime.thread.bind(providerRuntime); vi.spyOn(providerRuntime, "thread").mockImplementation((id) => { const thread = originalThread(id); return { ...thread, adapter: { ...thread.adapter, addReaction: async () => hold() }, }; }); // Before the repair, the real service calls its unbounded adapter method. // Afterward, forward only the new runtime transport into fake provider HTTP. const bounded = ( pinned as unknown as { applySlackReceiptReaction?: (...args: unknown[]) => Promise; } ).applySlackReceiptReaction; if (bounded) Object.assign(providerRuntime, { applySlackReceiptReaction: (...args: unknown[]) => bounded.call( pinned, ...args.slice(0, 1), async (_url: unknown, init?: RequestInit) => { await hold(init?.signal); return Response.json({ ok: true }); }, ), }); const thread = makeThread({ channelId: "CRECEIPTBOUND", id: "slack:CRECEIPTBOUND:1740000011.1", }); let inbound: Promise | undefined; let publication: Promise | undefined; try { inbound = deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: "1740000011.1", text: "@maya a fast final", mentioned: true, }), trigger: "mention", }); await start; const [lease] = await db .select() .from(chatEndpointLeases) .where( and( eq(chatEndpointLeases.endpointId, endpoint.id), eq(chatEndpointLeases.leaseKey, "credentials"), ), ); expect(lease).toBeDefined(); const [conversation] = await service.listConversations(endpoint.id); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "slack", providerMessageId: "1740000011.1", }), }); await addSelectedChatFinal({ agentId: fixture.assignedAgentId, body: "SLACK-FAST-FINAL", companyId: fixture.companyId, issueId: conversation.issueId, runId, }); publication = service.processPendingPublications(); await vi.waitFor( () => expect( [...providerRuntime.posts, ...providerRuntime.edits].filter( (entry) => entry.text === "SLACK-FAST-FINAL", ), ).toHaveLength(1), { timeout: 3_500, interval: 25 }, ); expect(aborted).toBe(true); expect(localTransportSettled).toBe(true); await Promise.all([inbound, publication]); } finally { release?.(); await Promise.allSettled([inbound, publication]); try { await pinned.shutdown(); } finally { await retirePublicationFixture(service, endpoint.id); } } }); it("settles a retried Slack receipt reaction when the provider reports already_reacted", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const endpointRuntime = runtime.endpoints.get(endpoint.id); if (!endpointRuntime) throw new Error("Expected Slack runtime"); endpointRuntime.reactionErrors.push( Object.assign(new Error("response lost after provider acceptance"), { code: "ECONNRESET", name: "NetworkError", }), Object.assign(new Error("reaction already exists"), { code: "slack_webapi_platform_error", data: { error: "already_reacted" }, }), ); const slack = makeThread({ channelId: "C-RECEIPT-IDEMPOTENT", id: "slack:C-RECEIPT-IDEMPOTENT:880012", name: "receipt-idempotent", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: slack.thread, message: makeMessage({ id: "880012", text: "@maya verify idempotent Slack receipt recovery", mentioned: true, }), trigger: "mention", }); const action = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "receipt_reaction"), ), ) .then((rows) => rows[0]!); expect(action).toMatchObject({ status: "failed" }); await db .update(chatActions) .set({ result: { ...(action.result ?? {}), retryAt: new Date(0).toISOString(), }, updatedAt: new Date(), }) .where(eq(chatActions.id, action.id)); await service.processPendingDeliveries(); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, action.id)), ).resolves.toEqual([ { status: "processed", result: { attempts: 2, code: "receipt_reaction_already_applied", }, }, ]); await expect( db .select({ redactedError: chatDeliveries.redactedError }) .from(chatDeliveries) .where(eq(chatDeliveries.id, action.deliveryId!)), ).resolves.toEqual([{ redactedError: null }]); expect(endpointRuntime.reactionErrors).toEqual([]); await service.shutdown(); }); it("canonicalizes prefixed Chat SDK channel ids onto provider inventory resources", async () => { const fixture = await seedCompany(); const providerFetch = vi.fn(async (input: string | URL | Request) => { const url = String(input); if (url === "https://slack.com/api/auth.test") { return new Response( JSON.stringify({ ok: true, team_id: "T-PREFIXED", team: "Prefixed Workspace", user_id: "U-PREFIXED-BOT", user: "maya-prefixed", }), { status: 200, headers: { "content-type": "application/json", "x-oauth-scopes": TEST_SLACK_BOT_SCOPES, }, }, ); } if (url.startsWith("https://slack.com/api/conversations.list")) { return new Response( JSON.stringify({ ok: true, channels: [ { id: "C-PREFIXED", name: "prefixed-channel", is_member: true, is_archived: false, }, ], response_metadata: { next_cursor: "" }, }), { status: 200, headers: { "content-type": "application/json" } }, ); } throw new Error(`Unexpected provider request: ${url}`); }) as typeof globalThis.fetch; const runtime = new FakeChatSdkRuntime(); const { service } = createService(runtime, providerFetch); const endpoint = await service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await service.configure( endpoint.id, { action: "configure", credentials: { botToken: "xoxb-prefixed-channel", signingSecret: "prefixed-channel-secret", }, }, "owner-user", ); const [inventoryResource] = await service.listResources(endpoint.id); await service.replaceResources(endpoint.id, [ { id: inventoryResource!.id, enabled: true }, ]); const callbacks = runtime.configurations.get(endpoint.id)?.callbacks; if (!callbacks) throw new Error("Expected Slack callbacks"); const thread = makeThread({ channelId: "slack:C-PREFIXED", id: "slack:C-PREFIXED:2200.1", name: "prefixed-channel", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: "2200.1", text: "@maya use the inventoried destination", mentioned: true, }), trigger: "mention", }); const resources = await service.listResources(endpoint.id); expect(resources).toHaveLength(1); expect(resources[0]).toMatchObject({ id: inventoryResource!.id, providerResourceId: "C-PREFIXED", availability: "available", enabled: true, }); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ externalThreadId: thread.thread.id }), ]); }); it("refuses to enable a destination the provider no longer exposes", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service } = await configuredSlackEndpoint(fixture); const channel = makeThread({ channelId: "C-REMOVED", id: "slack:C-REMOVED:2100.1", name: "removed-channel", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "2100.1", text: "This unaddressed event only discovers the channel", }), trigger: "unaddressed_message", }); const [resource] = await db .select() .from(chatEndpointResources) .where(eq(chatEndpointResources.endpointId, endpoint.id)); await db .update(chatEndpointResources) .set({ availability: "removed", updatedAt: new Date() }) .where(eq(chatEndpointResources.id, resource.id)); await expect( service.replaceResources(endpoint.id, [ { id: resource.id, enabled: true }, ]), ).rejects.toMatchObject({ status: 409, details: { code: "chat_resource_unavailable", resourceId: resource.id, }, }); }); it("filters unlinked people when configured, then honors a confirmed active-member identity link", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service } = await configuredSlackEndpoint( fixture, { allowUnlinkedPeople: false }, ); const channel = makeThread({ channelId: "C-PRIVATE", id: "slack:C-PRIVATE:3000.1", name: "private", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "3000.1", text: "@maya private task", mentioned: true, userId: "U-LINK-ME", }), trigger: "mention", }); expect( await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)), ).toHaveLength(0); const [filtered] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(filtered.state).toBe("filtered"); const now = new Date(); await db.insert(authUsers).values({ id: "linked-paperclip-user", name: "Linked User", email: `linked-${fixture.companyId}@example.com`, emailVerified: true, createdAt: now, updatedAt: now, }); await db.insert(companyMemberships).values({ companyId: fixture.companyId, principalType: "user", principalId: "linked-paperclip-user", status: "active", membershipRole: "viewer", }); const [principal] = await db .select() .from(chatExternalPrincipals) .where(eq(chatExternalPrincipals.externalId, "U-LINK-ME")); const intent = await service.createLinkIntent( endpoint.id, principal.id, 1_800, ); const token = new URL(intent.confirmationUrl).searchParams.get("token"); if (!token) throw new Error("Identity-link confirmation token was absent"); await expect(service.previewIdentityLink(token)).resolves.toMatchObject({ endpointId: endpoint.id, companyId: fixture.companyId, provider: "slack", externalLabel: "Alex External", }); await expect( service.confirmIdentityLink(token, "linked-paperclip-user"), ).resolves.toEqual({ ok: true, endpointId: endpoint.id, }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "3000.2", text: "@maya private task", mentioned: true, userId: "U-LINK-ME", }), trigger: "mention", }); expect( await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)), ).toHaveLength(0); const viewerDelivery = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)) .then((rows) => rows.find((delivery) => delivery.providerEventId.includes("3000.2")), ); expect(viewerDelivery?.state).toBe("filtered"); await db .update(companyMemberships) .set({ membershipRole: "operator" }) .where(eq(companyMemberships.principalId, "linked-paperclip-user")); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "3000.3", text: "@maya private task", mentioned: true, userId: "U-LINK-ME", }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id, "U-LINK-ME"); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); expect(conversation).toBeDefined(); const [comment] = await db .select() .from(issueComments) .where( and( eq(issueComments.issueId, conversation.issueId), eq(issueComments.authorType, "user"), ), ); expect(comment.authorType).toBe("user"); expect(comment.authorUserId).toBe("linked-paperclip-user"); const userCommentCountBeforeSuspend = ( await db .select() .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)) ).filter((candidate) => candidate.authorType === "user").length; await db .update(companyMemberships) .set({ status: "suspended" }) .where(eq(companyMemberships.principalId, "linked-paperclip-user")); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "3000.4", text: "This must no longer pass", userId: "U-LINK-ME", }), trigger: "subscribed_message", }); const allComments = await db .select() .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)); expect( allComments.filter((candidate) => candidate.authorType === "user"), ).toHaveLength(userCommentCountBeforeSuspend); const allDeliveries = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect( allDeliveries.find((delivery) => delivery.providerEventId.includes("3000.4"), )?.state, ).toBe("filtered"); }); it("filters an unlinked guest when the endpoint sponsor is suspended and exposes the reason and duplicate count", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service } = await configuredSlackEndpoint(fixture); await db .update(companyMemberships) .set({ status: "suspended" }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalType, "user"), eq(companyMemberships.principalId, "owner-user"), ), ); const channel = makeThread({ channelId: "C-SPONSOR-SUSPENDED", id: "slack:C-SPONSOR-SUSPENDED:3100.1", name: "sponsor-suspended", }); const message = makeMessage({ id: "3100.1", text: "@maya this guest no longer has a sponsor", mentioned: true, userId: `U-UNLINKED-${randomUUID()}`, }); for (let attempt = 0; attempt < 3; attempt += 1) { await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message, trigger: "mention", }); } expect(await service.listConversations(endpoint.id)).toEqual([]); const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(delivery).toMatchObject({ state: "filtered", redactedError: "Endpoint sponsor can no longer authorize external guests", normalizedEvent: { deduplication: { duplicateCount: 2 }, }, }); await expect(service.listActivity(endpoint.id)).resolves.toEqual([ expect.objectContaining({ id: delivery.id, kind: "delivery", status: "filtered", summary: "mention ignored · 2 duplicates ignored", detail: "Endpoint sponsor can no longer authorize external guests", replayable: false, }), ]); }); it("guides empty Slack mentions once without creating a task or run", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredSlackEndpoint(fixture); const configured = await service.get(endpoint.id); const variants = [ "", " \t\n", `<@${configured.botExternalId}>`, ` \u200b <@${configured.botExternalId}> @maya !!! `, ]; for (const [index, text] of variants.entries()) { const root = makeThread({ channelId: "C-EMPTY-MENTION", id: `slack:C-EMPTY-MENTION:3200.${index}`, name: "empty-mention", }); const message = makeMessage({ id: `3200.${index}`, text, mentioned: true, userId: "U-EMPTY-MENTION", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: root.thread, message, trigger: "mention", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: root.thread, message, trigger: "mention", }); expect(root.post).toHaveBeenCalledTimes(1); expect(root.post).toHaveBeenCalledWith( "Please include a request after mentioning me.", ); expect(runtime.endpoints.get(endpoint.id)?.reactions).toContainEqual({ threadId: root.thread.id, messageId: message.id, emoji: "eyes", }); expect(root.addReaction).not.toHaveBeenCalled(); } expect(await service.listConversations(endpoint.id)).toEqual([]); expect( await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toHaveLength(0); expect( await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.companyId, fixture.companyId)), ).toHaveLength(0); expect( await db .select() .from(chatPublications) .where(eq(chatPublications.companyId, fixture.companyId)), ).toHaveLength(0); const deliveries = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(deliveries).toHaveLength(variants.length); expect( deliveries.every( (delivery) => delivery.state === "processed" && delivery.normalizedEvent.deduplication?.duplicateCount === 1, ), ).toBe(true); expect(wakeup).not.toHaveBeenCalled(); }); it("suppresses a queued provider reply when destination reach is revoked before transport", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const root = makeThread({ channelId: "C-EFFECT-REVOKE", id: "slack:C-EFFECT-REVOKE:3210.1", name: "effect-revoke", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: root.thread, message: makeMessage({ id: "3210.1", text: "@maya", mentioned: true, userId: "U-EFFECT-REVOKE", }), trigger: "mention", }); const seedEffect = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "provider_effect"), ), ) .then((rows) => rows[0]); if (!seedEffect?.principalId) throw new Error("Expected an authorized provider-effect seed"); const resource = await db .select() .from(chatEndpointResources) .where( and( eq(chatEndpointResources.endpointId, endpoint.id), eq(chatEndpointResources.providerResourceId, "C-EFFECT-REVOKE"), ), ) .then((rows) => rows[0]); if (!resource) throw new Error("Expected the discovered Slack resource"); const [queuedDelivery] = await db .insert(chatDeliveries) .values({ companyId: fixture.companyId, endpointId: endpoint.id, principalId: seedEffect.principalId, providerEventId: "queued-provider-effect-reach-revoke", deduplicationKey: "queued-provider-effect-reach-revoke", eventKind: "mention", normalizedEvent: {}, state: "processing", attempts: 1, }) .returning(); const [queuedEffect] = await db .insert(chatActions) .values({ companyId: fixture.companyId, endpointId: endpoint.id, deliveryId: queuedDelivery!.id, principalId: seedEffect.principalId, kind: "provider_effect", providerActionId: "provider_effect:queued-reach-revoke", payload: seedEffect.payload, status: "received", }) .returning(); await service.replaceResources(endpoint.id, [ { id: resource.id, enabled: false }, ]); await service.processPendingProviderEffects(); expect(runtime.endpoints.get(endpoint.id)?.posts).toEqual([]); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, queuedEffect!.id)), ).resolves.toEqual([ { status: "cancelled", result: { attempts: 1, code: "provider_effect_no_longer_authorized", }, }, ]); await expect( db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where(eq(chatDeliveries.id, queuedDelivery!.id)), ).resolves.toEqual([{ state: "filtered" }]); }); it("keeps one Paperclip account mapping for the same provider principal across endpoints", async () => { const fixture = await seedCompany(); const firstContext = createService( new FakeChatSdkRuntime(), fakeSlackFetch("U-BOT-IDENTITY-A") as typeof globalThis.fetch, ); const secondContext = createService( new FakeChatSdkRuntime(), fakeSlackFetch("U-BOT-IDENTITY-B") as typeof globalThis.fetch, ); const first = await firstContext.service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId, }, "owner-user", ); const second = await secondContext.service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.replacementAgentId, }, "owner-user", ); for (const [context, endpoint, suffix] of [ [firstContext, first, "a"], [secondContext, second, "b"], ] as const) { await context.service.configure( endpoint.id, { action: "configure", credentials: { botToken: `xoxb-identity-${suffix}`, signingSecret: `identity-secret-${suffix}`, }, }, "owner-user", ); await recordSlackUrlVerification(context.service, endpoint.publicId); await context.service.configure( endpoint.id, { action: "verify" }, "owner-user", ); const callbacks = context.runtime.configurations.get( endpoint.id, )?.callbacks; if (!callbacks) throw new Error("Expected endpoint callbacks"); const thread = makeThread({ channelId: `C-IDENTITY-${suffix.toUpperCase()}`, id: `slack:C-IDENTITY-${suffix.toUpperCase()}:1`, name: `identity-${suffix}`, }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: `identity-${suffix}`, text: `@bot identify ${suffix}`, mentioned: true, userId: "U-SHARED-HUMAN", }), trigger: "mention", }); } const now = new Date(); await db.insert(authUsers).values([ { id: "paperclip-user-a", name: "Paperclip User A", email: `identity-a-${fixture.companyId}@example.com`, emailVerified: true, createdAt: now, updatedAt: now, }, { id: "paperclip-user-b", name: "Paperclip User B", email: `identity-b-${fixture.companyId}@example.com`, emailVerified: true, createdAt: now, updatedAt: now, }, ]); await db.insert(companyMemberships).values([ { companyId: fixture.companyId, principalType: "user", principalId: "paperclip-user-a", status: "active", membershipRole: "operator", }, { companyId: fixture.companyId, principalType: "user", principalId: "paperclip-user-b", status: "active", membershipRole: "operator", }, ]); const principal = await db .select() .from(chatExternalPrincipals) .where(eq(chatExternalPrincipals.externalId, "U-SHARED-HUMAN")) .then((rows) => rows[0]); const firstIntent = await firstContext.service.createLinkIntent( first.id, principal.id, 1_800, ); const firstToken = new URL(firstIntent.confirmationUrl).searchParams.get( "token", ); if (!firstToken) throw new Error("First identity token was absent"); await firstContext.service.confirmIdentityLink( firstToken, "paperclip-user-a", ); const secondIntent = await secondContext.service.createLinkIntent( second.id, principal.id, 1_800, ); const secondToken = new URL(secondIntent.confirmationUrl).searchParams.get( "token", ); if (!secondToken) throw new Error("Second identity token was absent"); await expect( secondContext.service.confirmIdentityLink( secondToken, "paperclip-user-b", ), ).rejects.toMatchObject({ status: 409, details: { code: "chat_identity_link_conflict" }, }); }); it("publishes only the safe projection once and locks reassignment of a bound task", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const channel = makeThread({ channelId: "C-SAFE", id: "slack:C-SAFE:4000.1", name: "safe-output", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "4000.1", text: "@maya give me the public result", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const comment = await issueService(db).addComment( conversation.issueId, "Visible answer. never expose this reasoning", { userId: "owner-user" }, { authorType: "user" }, ); const first = await service.publishComment( endpoint.id, conversation.id, comment.id, ); const second = await service.publishComment( endpoint.id, conversation.id, comment.id, ); expect(first.id).toBe(second.id); // Both calls return the same durable row after the synchronous delivery // attempt; callers never observe a stale pre-send snapshot. expect(first.state).toBe("published"); expect(second.state).toBe("published"); const providerRuntime = runtime.endpoints.get(endpoint.id); expect(providerRuntime?.posts).toEqual([ { threadId: channel.thread.id, text: "Visible answer." }, ]); const publications = await db .select() .from(chatPublications) .where(eq(chatPublications.endpointId, endpoint.id)); const links = await db .select() .from(chatMessageLinks) .where(eq(chatMessageLinks.endpointId, endpoint.id)); const explicitPublications = publications.filter( (publication) => publication.commentId === comment.id, ); expect(explicitPublications).toHaveLength(1); expect(explicitPublications[0]).toMatchObject({ state: "published", providerMessageId: expect.stringMatching(/^outbound-\d+$/), }); expect( links.filter( (link) => link.direction === "inbound" || link.commentId === comment.id, ), ).toHaveLength(3); expect( links.find( (link) => link.direction === "outbound" && link.commentId === comment.id, )?.providerMessageId, ).toBe(explicitPublications[0]?.providerMessageId); await expect( issueService(db).update(conversation.issueId, { assigneeAgentId: fixture.replacementAgentId, actorUserId: "owner-user", }), ).rejects.toMatchObject({ status: 409, details: { code: "chat_binding_agent_locked", conversationId: conversation.id, }, }); const [boundIssue] = await db .select() .from(issues) .where(eq(issues.id, conversation.issueId)); expect(boundIssue.assigneeAgentId).toBe(fixture.assignedAgentId); await service.configure(endpoint.id, { action: "remove" }, "owner-user"); await expect( issueService(db).update(conversation.issueId, { assigneeAgentId: fixture.replacementAgentId, actorUserId: "owner-user", }), ).rejects.toMatchObject({ status: 409, details: { code: "chat_binding_agent_locked", conversationId: conversation.id, }, }); await expect( service.getIssueBinding(conversation.issueId), ).resolves.toMatchObject({ endpointId: endpoint.id, conversationId: conversation.id, assignedAgentLocked: true, }); }); it("keeps external-continuation lifecycle comments internal and publishes only the selected final", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const channel = makeThread({ channelId: "C-CONTINUATION-FINAL", id: "slack:C-CONTINUATION-FINAL:4005.1", name: "continuation-final", }); const providerMessageId = "4005.1"; await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: providerMessageId, text: "@maya return the exact continuation result", mentioned: true, }), trigger: "mention", }); const conversation = await db .select() .from(chatConversations) .where( and( eq(chatConversations.endpointId, endpoint.id), eq(chatConversations.externalThreadId, channel.thread.id), ), ) .then((rows) => rows[0]); if (!conversation) throw new Error("Expected Slack conversation"); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: { ...(await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "slack", providerMessageId, })), externalChatContinuation: true, paperclipWake: { externalInteractionContinuation: true }, }, }); const lifecycleComment = await issueService(db).addComment( conversation.issueId, "Answer received for the interaction. Closing issue.", { agentId: fixture.assignedAgentId, runId }, { authorType: "agent", authorizationReason: "paperclip_runner_protocol", }, ); await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.id, lifecycleComment.id)), ).resolves.toEqual([ { body: "Answer received for the interaction. Closing issue." }, ]); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.commentId, lifecycleComment.id)), ).resolves.toHaveLength(0); // A continuation agent may write its final prose before the heartbeat // presentation resolver selects the identical adapter response. That // first write is intentionally internal; resolving the same text must // upgrade the one durable comment instead of falling through to a generic // "completed this turn" provider milestone. const provisionalFinalComment = await issueService(db).addComment( conversation.issueId, "SLACK-CONTINUATION-Onyx", { agentId: fixture.assignedAgentId, runId }, { authorType: "agent", authorizationReason: "allow_self" }, ); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.commentId, provisionalFinalComment.id)), ).resolves.toHaveLength(0); const authorizationReason = await resolveChatRunPresentationAuthorizationReason(db, { companyId: fixture.companyId, issueId: conversation.issueId, runId, }); expect(authorizationReason).toBe("allow_chat_run_presentation"); const finalComment = await issueService(db).addComment( conversation.issueId, "SLACK-CONTINUATION-Onyx", { agentId: fixture.assignedAgentId, runId }, { authorType: "agent", authorizationReason }, ); expect(finalComment.id).toBe(provisionalFinalComment.id); await expect( db .select({ id: issueComments.id }) .from(issueComments) .where( and( eq(issueComments.issueId, conversation.issueId), eq(issueComments.createdByRunId, runId), eq(issueComments.body, "SLACK-CONTINUATION-Onyx"), ), ), ).resolves.toHaveLength(1); await expect( db .select({ metadata: issueComments.metadata }) .from(issueComments) .where(eq(issueComments.id, finalComment.id)), ).resolves.toEqual([ expect.objectContaining({ metadata: expect.objectContaining({ authorizationReason: "allow_chat_run_presentation", }), }), ]); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.commentId, finalComment.id)), ).resolves.toEqual([{ state: "pending" }]); await service.processPendingPublications(); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.commentId, finalComment.id)), ).resolves.toEqual([{ state: "published" }]); const posts = runtime.endpoints.get(endpoint.id)?.posts ?? []; expect(posts).toEqual( expect.arrayContaining([ expect.objectContaining({ threadId: channel.thread.id, text: "SLACK-CONTINUATION-Onyx", }), ]), ); expect(JSON.stringify(posts)).not.toContain("Answer received"); }); it("keeps root-chat bookkeeping comments internal and publishes only the selected final", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const channel = makeThread({ channelId: "C-ROOT-FINAL", id: "slack:C-ROOT-FINAL:4006.1", name: "root-final", }); const providerMessageId = "4006.1"; await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: providerMessageId, text: "@maya return only ROOT-CHAT-FINAL", mentioned: true, }), trigger: "mention", }); const conversation = await db .select() .from(chatConversations) .where( and( eq(chatConversations.endpointId, endpoint.id), eq(chatConversations.externalThreadId, channel.thread.id), ), ) .then((rows) => rows[0]); if (!conversation) throw new Error("Expected Slack conversation"); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "slack", providerMessageId, }), }); const bookkeepingComment = await issueService(db).addComment( conversation.issueId, "Acknowledged the latest comment; it changes my next action.", { agentId: fixture.assignedAgentId, runId }, { authorType: "agent", authorizationReason: "paperclip_runner_protocol", }, ); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.commentId, bookkeepingComment.id)), ).resolves.toHaveLength(0); const authorizationReason = await resolveChatRunPresentationAuthorizationReason(db, { companyId: fixture.companyId, issueId: conversation.issueId, runId, }); expect(authorizationReason).toBe("allow_chat_run_presentation"); const finalComment = await issueService(db).addComment( conversation.issueId, "ROOT-CHAT-FINAL", { agentId: fixture.assignedAgentId, runId }, { authorType: "agent", authorizationReason }, ); await service.processPendingPublications(1_000); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.commentId, finalComment.id)), ).resolves.toEqual([{ state: "published" }]); const posts = runtime.endpoints.get(endpoint.id)?.posts ?? []; expect( posts.filter( (post) => post.threadId === channel.thread.id && post.text === "ROOT-CHAT-FINAL", ), ).toHaveLength(1); expect(JSON.stringify(posts)).not.toContain("Acknowledged"); }); it("materializes direct external-chat finals once and accepts the next turn without agent bookkeeping writes", async () => { const fixture = await seedCompany(); const adapterType = `chat-shortcut-test-${randomUUID()}`; const directFinals = ["SHORTCUT-FIRST", "SHORTCUT-SECOND"]; const execute = vi.fn(async (_input: unknown) => ({ exitCode: 0, signal: null, timedOut: false, errorMessage: null, summary: directFinals[execute.mock.calls.length - 1]!, provider: "test", model: "test-model", })); registerServerAdapter({ type: adapterType, supportsLocalAgentJwt: false, execute, testEnvironment: async () => ({ adapterType, status: "pass", checks: [], testedAt: new Date().toISOString(), }), }); const heartbeat = heartbeatService(db); try { await db .update(companies) .set({ defaultResponsibleUserId: "owner-user" }) .where(eq(companies.id, fixture.companyId)); await db .update(agents) .set({ adapterType, runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 }, }, }) .where(eq(agents.id, fixture.assignedAgentId)); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture, { wakeup: heartbeat.wakeup, }); const configuredEndpoint = await service.get(endpoint.id); const [principal] = await db .insert(chatExternalPrincipals) .values({ companyId: fixture.companyId, provider: "slack", providerAccountId: configuredEndpoint.providerAccountId!, externalId: "U-SHORTCUT-LINKED", kind: "user", displayName: "Linked Chat User", }) .returning(); await db.insert(chatIdentityLinks).values({ companyId: fixture.companyId, endpointId: endpoint.id, principalId: principal!.id, paperclipUserId: "owner-user", status: "linked", confirmedAt: new Date(), }); const channel = makeThread({ channelId: "C-SHORTCUT", id: "slack:C-SHORTCUT:shortcut-root", name: "shortcut", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "shortcut-root", text: "@maya answer the first self-contained question", mentioned: true, userId: "U-SHORTCUT-LINKED", }), trigger: "mention", }); await heartbeat.drainActiveRunExecutions(); const [conversation] = await db .select() .from(chatConversations) .where( and( eq(chatConversations.endpointId, endpoint.id), eq(chatConversations.externalThreadId, channel.thread.id), ), ); expect(conversation).toBeDefined(); const firstRun = await db .select() .from(heartbeatRuns) .where( and( eq(heartbeatRuns.companyId, fixture.companyId), eq(heartbeatRuns.agentId, fixture.assignedAgentId), ), ) .then((rows) => rows.find((row) => row.status === "succeeded")); expect(firstRun).toBeDefined(); const firstAdapterInput = execute.mock.calls[0]?.[0] as { context?: { paperclipWake?: unknown } } | undefined; expect( isPaperclipExternalChatTurn(firstAdapterInput?.context?.paperclipWake), ).toBe(true); const [firstInboundLink] = await db .select({ commentId: chatMessageLinks.commentId }) .from(chatMessageLinks) .where( and( eq(chatMessageLinks.endpointId, endpoint.id), eq(chatMessageLinks.providerMessageId, "shortcut-root"), eq(chatMessageLinks.direction, "inbound"), ), ); const trustedWakeContext = { source: "chat:slack", wakeCommentIds: [firstInboundLink!.commentId!], paperclipHarnessCheckedOut: true, }; await expect( resolveExternalChatWakeProvider({ db, companyId: fixture.companyId, agentId: fixture.assignedAgentId, issueId: conversation!.issueId, contextSnapshot: trustedWakeContext, }), ).resolves.toBe("slack"); await expect( resolveExternalChatWakeProvider({ db, companyId: fixture.companyId, agentId: fixture.assignedAgentId, issueId: conversation!.issueId, contextSnapshot: { ...trustedWakeContext, source: "chat:slack:recovery", }, }), ).resolves.toBe("slack"); await expect( resolveExternalChatWakeProvider({ db, companyId: fixture.companyId, agentId: fixture.assignedAgentId, issueId: conversation!.issueId, contextSnapshot: { ...trustedWakeContext, source: "chat:slack:recovery:unexpected", }, }), ).resolves.toBeNull(); await expect( resolveExternalChatWakeProvider({ db, companyId: fixture.companyId, agentId: fixture.replacementAgentId, issueId: conversation!.issueId, contextSnapshot: trustedWakeContext, }), ).resolves.toBeNull(); await expect( resolveExternalChatWakeProvider({ db, companyId: fixture.companyId, agentId: fixture.assignedAgentId, issueId: conversation!.issueId, contextSnapshot: { ...trustedWakeContext, source: "automation", }, }), ).resolves.toBeNull(); await expect( resolveExternalChatWakeProvider({ db, companyId: randomUUID(), agentId: fixture.assignedAgentId, issueId: conversation!.issueId, contextSnapshot: trustedWakeContext, }), ).resolves.toBeNull(); await expect( resolveExternalChatWakeProvider({ db, companyId: fixture.companyId, agentId: fixture.assignedAgentId, issueId: randomUUID(), contextSnapshot: trustedWakeContext, }), ).resolves.toBeNull(); await expect( resolveExternalChatWakeProvider({ db, companyId: fixture.companyId, agentId: fixture.assignedAgentId, issueId: conversation!.issueId, contextSnapshot: { ...trustedWakeContext, paperclipHarnessCheckedOut: false, }, }), ).resolves.toBeNull(); await expect( resolveExternalChatWakeProvider({ db, companyId: fixture.companyId, agentId: fixture.assignedAgentId, issueId: conversation!.issueId, contextSnapshot: { ...trustedWakeContext, source: "chat:discord", }, }), ).resolves.toBeNull(); const [internalComment] = await db .insert(issueComments) .values({ companyId: fixture.companyId, issueId: conversation!.issueId, authorType: "user", authorUserId: "owner-user", body: "Internal board note, not external chat input.", }) .returning(); await db.insert(chatMessageLinks).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation!.id, commentId: internalComment!.id, providerMessageId: "shortcut-outbound-only", direction: "outbound", }); await expect( resolveExternalChatWakeProvider({ db, companyId: fixture.companyId, agentId: fixture.assignedAgentId, issueId: conversation!.issueId, contextSnapshot: { ...trustedWakeContext, wakeCommentIds: [firstInboundLink!.commentId!, internalComment!.id], }, }), ).resolves.toBeNull(); await expect( resolveExternalChatWakeProvider({ db, companyId: fixture.companyId, agentId: fixture.assignedAgentId, issueId: conversation!.issueId, contextSnapshot: { ...trustedWakeContext, wakeCommentIds: [internalComment!.id], }, }), ).resolves.toBeNull(); const firstFinal = await db .select() .from(issueComments) .where(eq(issueComments.createdByRunId, firstRun!.id)); expect(firstFinal).toHaveLength(1); expect(firstFinal[0]).toMatchObject({ issueId: conversation!.issueId, body: "SHORTCUT-FIRST", metadata: expect.objectContaining({ authorizationReason: "allow_chat_run_presentation", }), }); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.commentId, firstFinal[0]!.id)), ).resolves.toHaveLength(1); await service.processPendingPublications(1_000); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "shortcut-followup", text: "answer the second self-contained question", userId: "U-SHORTCUT-LINKED", }), trigger: "subscribed_message", }); await heartbeat.drainActiveRunExecutions(); const runs = await db .select() .from(heartbeatRuns) .where( and( eq(heartbeatRuns.companyId, fixture.companyId), eq(heartbeatRuns.agentId, fixture.assignedAgentId), ), ); const succeededRuns = runs.filter((row) => row.status === "succeeded"); expect(succeededRuns).toHaveLength(2); const secondRun = succeededRuns.find((row) => row.id !== firstRun!.id); expect(secondRun).toBeDefined(); const secondAdapterInput = execute.mock.calls[1]?.[0] as { context?: { paperclipWake?: unknown } } | undefined; expect( isPaperclipExternalChatTurn(secondAdapterInput?.context?.paperclipWake), ).toBe(true); const secondFinal = await db .select() .from(issueComments) .where(eq(issueComments.createdByRunId, secondRun!.id)); expect(secondFinal).toHaveLength(1); expect(secondFinal[0]).toMatchObject({ issueId: conversation!.issueId, body: "SHORTCUT-SECOND", metadata: expect.objectContaining({ authorizationReason: "allow_chat_run_presentation", }), }); await service.processPendingPublications(1_000); expect(execute).toHaveBeenCalledTimes(2); expect( (runtime.endpoints.get(endpoint.id)?.posts ?? []).filter( (post) => post.threadId === channel.thread.id && directFinals.includes(post.text), ), ).toEqual([ expect.objectContaining({ text: "SHORTCUT-FIRST" }), expect.objectContaining({ text: "SHORTCUT-SECOND" }), ]); const providerFacingComments = await db .select({ body: issueComments.body }) .from(issueComments) .where( and( eq(issueComments.issueId, conversation!.issueId), eq(issueComments.authorType, "agent"), ), ); expect(providerFacingComments).toHaveLength(2); expect(providerFacingComments).toEqual( expect.arrayContaining([ { body: "SHORTCUT-FIRST" }, { body: "SHORTCUT-SECOND" }, ]), ); } finally { await heartbeat.drainActiveRunExecutions(); unregisterServerAdapter(adapterType); } }); it("coalesces one GitHub run's progress and final response into one provider comment", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredGitHubEndpoint(fixture); const thread = makeThread({ channelId: "github:paperclipai/paperclip", id: "github:paperclipai/paperclip:issue:417", name: "paperclipai/paperclip", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id: "41701", text: "@maya produce one quiet GitHub response", mentioned: true, }), trigger: "mention", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "github", providerMessageId: "41701", }), }); await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${runId}:queued:${endpoint.id}`, payload: { text: "Maya is queued.", progressState: "queued" }, state: "pending", }); await service.processPendingPublications(); await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${runId}:working:${endpoint.id}`, payload: { text: "Maya is working…", progressState: "working" }, state: "pending", }); await service.processPendingPublications(); const finalComment = await addSelectedChatFinal({ agentId: fixture.assignedAgentId, body: "Final GitHub result", companyId: fixture.companyId, issueId: conversation.issueId, runId, }); await service.processPendingPublications(); const providerRuntime = runtime.endpoints.get(endpoint.id); expect(providerRuntime?.posts).toEqual([ { threadId: thread.thread.id, text: "Maya is queued.", }, ]); expect(providerRuntime?.edits).toEqual([ { threadId: thread.thread.id, messageId: "outbound-1", text: "Maya is working…", }, { threadId: thread.thread.id, messageId: "outbound-1", text: "Final GitHub result", }, ]); const publications = await db .select() .from(chatPublications) .where(eq(chatPublications.conversationId, conversation.id)); expect(publications).toHaveLength(3); expect( publications.every( (publication) => publication.state === "published" && publication.providerMessageId === "outbound-1", ), ).toBe(true); const [providerLink] = await db .select() .from(chatMessageLinks) .where( and( eq(chatMessageLinks.conversationId, conversation.id), eq(chatMessageLinks.direction, "outbound"), ), ); expect(providerLink).toMatchObject({ providerMessageId: "outbound-1", commentId: finalComment.id, }); await service.processPendingPublications(); expect(providerRuntime?.posts).toHaveLength(1); expect(providerRuntime?.edits).toHaveLength(2); const replacementRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: replacementRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", contextSnapshot: { issueId: conversation.issueId }, }); await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${replacementRunId}:queued:${endpoint.id}`, payload: { text: "A replacement run is queued.", progressState: "queued", }, state: "pending", }); await service.processPendingPublications(); if (!providerRuntime) throw new Error("Expected GitHub provider runtime"); providerRuntime.editError = Object.assign(new Error("comment gone"), { status: 404, }); await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${replacementRunId}:working:${endpoint.id}`, payload: { text: "A replacement run is working…", progressState: "working", }, state: "pending", }); await service.processPendingPublications(); const replacementEdit = await db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `run:${replacementRunId}:working:${endpoint.id}`, ), ) .then((rows) => rows[0]); expect(replacementEdit).toMatchObject({ state: "published", providerMessageId: "outbound-3", attempts: 1, }); expect(providerRuntime.editAttempts.at(-1)).toEqual({ threadId: thread.thread.id, messageId: "outbound-2", }); expect(providerRuntime.posts.at(-1)).toEqual({ threadId: thread.thread.id, text: "A replacement run is working…", }); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "verifying", }); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ id: conversation.id, state: "active" }), ]); providerRuntime.editError = null; const ambiguousRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: ambiguousRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", contextSnapshot: { issueId: conversation.issueId }, }); await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${ambiguousRunId}:queued:${endpoint.id}`, payload: { text: "A second run is queued.", progressState: "queued" }, state: "pending", }); await service.processPendingPublications(); providerRuntime.postError = new Error("socket reset after write"); await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${ambiguousRunId}:working:${endpoint.id}`, payload: { text: "A second run is working…", progressState: "working", }, state: "pending", }); await service.processPendingPublications(); const ambiguousEdit = await db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `run:${ambiguousRunId}:working:${endpoint.id}`, ), ) .then((rows) => rows[0]); expect(ambiguousEdit).toMatchObject({ state: "delivery_unknown", providerMessageId: null, attempts: 1, }); expect(providerRuntime.posts).toHaveLength(4); expect(providerRuntime.edits).toHaveLength(2); await service.processPendingPublications(); expect(providerRuntime.posts).toHaveLength(4); expect(providerRuntime.edits).toHaveLength(2); }); describe("Telegram callback-only native private responses", () => { async function nativePrivateFixture(linked = false, privateChat = false) { const fixture = await seedCompany(); const botId = Number.parseInt( randomUUID().replaceAll("-", "").slice(0, 12), 16, ); const scheduled: Array<() => void> = []; const context = createService( new FakeChatSdkRuntime(), fakeTelegramFetch(botId) as typeof fetch, { scheduleDeferredWork: (task) => { scheduled.push(task); }, }, ); const endpoint = await context.service.create( fixture.companyId, { provider: "telegram", assignedAgentId: fixture.assignedAgentId, name: "Native private callback fixture", }, "owner-user", ); await context.service.configure( endpoint.id, { action: "configure", credentials: { botToken: `${botId}:synthetic-private-token` }, }, "owner-user", ); await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, endpoint.id)); const chatId = privateChat ? 456 : -100123; await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "group", providerResourceId: String(chatId), label: "Private callback source", enabled: true, }); if (linked) { const [stored] = await db .select() .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)); const [principal] = await db .insert(chatExternalPrincipals) .values({ companyId: fixture.companyId, provider: "telegram", providerAccountId: stored.providerAccountId!, externalId: "456", kind: "user", isBot: false, }) .returning(); await db.insert(chatIdentityLinks).values({ companyId: fixture.companyId, endpointId: endpoint.id, principalId: principal.id, status: "linked", paperclipUserId: "owner-user", confirmedAt: new Date(), }); } const providerRequests: Array<{ method: string; body: Record; }> = []; let responseMode: "accepted" | "missing_receipt" | "rate_limited" = "accepted"; const originalFetch = globalThis.fetch; globalThis.fetch = vi.fn(async (input, init) => { const method = new URL(String(input)).pathname.split("/").at(-1)!; if (method === "getMe") return Response.json({ ok: true, result: { id: botId, is_bot: true, first_name: "Synthetic", username: `paperclip_${botId}_bot`, }, }); const body = JSON.parse(String(init?.body ?? "{}")) as Record< string, unknown >; providerRequests.push({ method, body }); if (method === "answerCallbackQuery") return Response.json({ ok: true, result: true }); if (method !== "sendMessage") throw new Error("Unexpected synthetic Telegram request"); if (responseMode === "missing_receipt") return Response.json({ ok: true, result: true }); if (responseMode === "rate_limited") return Response.json( { ok: false, error_code: 429, parameters: { retry_after: 20 } }, { status: 429 }, ); if (privateChat) return Response.json({ ok: true, result: { message_id: 908, chat: { id: Number(body.chat_id), type: "private" }, from: { id: botId, is_bot: true }, }, }); const parameters = body.ephemeral_message_parameters as { receiver_user_id: number; }; return Response.json({ ok: true, result: { message_id: 0, ephemeral_message_id: 908, chat: { id: Number(body.chat_id), type: "supergroup" }, from: { id: botId, is_bot: true }, receiver_user: { id: parameters.receiver_user_id, is_bot: false }, }, }); }); let pinned: ReturnType; const install = (runtime: FakeChatSdkRuntime) => { const configuration = runtime.configurations.get(endpoint.id)!; pinned = createChatSdkEndpointRuntime({ ...configuration, logger: "silent", }); Object.assign(runtime.endpoints.get(endpoint.id)!, { handleWebhook: (...args: Parameters) => pinned.handleWebhook(...args), sendTelegramCallbackNotice: ( ...args: Parameters ) => pinned.sendTelegramCallbackNotice(...args), }); }; install(context.runtime); const payload = ( callbackId = "native-private-callback-1", actorId = 456, sourceChatId = chatId, ) => ({ update_id: 901, callback_query: { id: callbackId, from: { id: actorId, is_bot: false, first_name: "Synthetic actor" }, chat_instance: "synthetic-instance", data: "pcq:unavailable", message: { message_id: 71, date: Math.floor(Date.now() / 1000), chat: { id: sourceChatId, type: privateChat ? "private" : "supergroup", title: "Source", }, from: { id: botId, is_bot: true, first_name: "Synthetic" }, text: "Choose an option", }, }, }); const deliver = async (update = payload()) => { const configuration = context.runtime.configurations.get(endpoint.id)!; if (configuration.providerConfig.provider !== "telegram") throw new Error("Expected Telegram configuration"); return context.service.handleWebhook( endpoint.publicId, "telegram", new Request("https://paperclip.example/synthetic-telegram-webhook", { method: "POST", headers: { "content-type": "application/json", "x-telegram-bot-api-secret-token": configuration.providerConfig.credentials.secretToken, }, body: JSON.stringify(update), }), ); }; const actions = () => db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "provider_effect"), ), ); const deliveries = () => db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); let resumed: ReturnType | undefined; return { ...context, fixture, endpoint, scheduled, providerRequests, payload, deliver, actions, deliveries, setResponseMode(mode: typeof responseMode) { responseMode = mode; }, async restart() { await pinned.shutdown(); await context.service.shutdown(); const resumedRuntime = new FakeChatSdkRuntime(); const replace = resumedRuntime.replaceEndpoint.bind(resumedRuntime); resumedRuntime.replaceEndpoint = async (options) => { const instance = await replace(options); install(resumedRuntime); return instance; }; resumed = createService( resumedRuntime, fakeTelegramFetch(botId) as typeof fetch, { scheduleDeferredWork: (task) => { scheduled.push(task); }, }, ); return resumed; }, async close() { try { await pinned.shutdown(); } finally { try { await retirePublicationFixture( resumed?.service ?? context.service, endpoint.id, ); } finally { globalThis.fetch = originalFetch; } } }, }; } it("records an authenticated denial before its recipient-bound send and deduplicates the exact callback", async () => { const context = await nativePrivateFixture(); try { expect((await context.deliver()).status).toBe(200); expect( context.providerRequests.filter( (request) => request.method === "sendMessage", ), ).toHaveLength(0); const [delivery] = await context.deliveries(); const [effect] = await context.actions(); expect(delivery).toMatchObject({ state: "filtered", normalizedEvent: { telegramCallback: { callbackId: "native-private-callback-1", receiverUserId: "456", }, }, }); expect(effect).toMatchObject({ status: "received", payload: { effect: "telegram_callback_notice", settleDelivery: false, }, }); expect(context.scheduled).toHaveLength(1); context.scheduled.shift()!(); await expect .poll(async () => (await context.actions())[0]?.status) .toBe("processed"); expect(await context.actions()).toEqual([ expect.objectContaining({ status: "processed", result: expect.objectContaining({ code: "telegram_ephemeral_api_accepted", }), }), ]); expect( context.providerRequests.filter( (request) => request.method === "sendMessage", ), ).toEqual([ { method: "sendMessage", body: { chat_id: "-100123", text: "This Paperclip action is no longer available. Open the linked task or ask an operator to link this account.", ephemeral_message_parameters: { receiver_user_id: 456, callback_query_id: "native-private-callback-1", }, }, }, ]); const after = await context.actions(); expect((await context.deliver()).status).toBe(200); expect(await context.deliveries()).toEqual([delivery]); expect(await context.actions()).toEqual(after); expect(context.scheduled).toHaveLength(0); expect( context.runtime.endpoints.get(context.endpoint.id)!.posts, ).toHaveLength(0); expect(context.wakeup).not.toHaveBeenCalled(); expect( await db .select() .from(chatPublications) .where(eq(chatPublications.endpointId, context.endpoint.id)), ).toEqual([]); } finally { await context.close(); } }); it.each(["actor", "chat", "data"])( "does not rebind a retained callback denial to another %s", async (changed) => { const context = await nativePrivateFixture(); try { expect((await context.deliver()).status).toBe(200); const before = await context.deliveries(); const effects = await context.actions(); const update = context.payload(); if (changed === "actor") update.callback_query.from.id = 457; if (changed === "chat") update.callback_query.message.chat.id = -100124; if (changed === "data") update.callback_query.data = "pcq:other"; expect((await context.deliver(update)).status).toBe(200); expect(await context.deliveries()).toEqual(before); expect(await context.actions()).toEqual(effects); expect(context.scheduled).toHaveLength(1); await context.service.processPendingProviderEffects(); expect((await context.actions())[0]).toMatchObject({ status: "processed", }); expect( context.providerRequests.filter( (entry) => entry.method === "sendMessage", ), ).toEqual([ expect.objectContaining({ body: expect.objectContaining({ chat_id: "-100123", ephemeral_message_parameters: { receiver_user_id: 456, callback_query_id: "native-private-callback-1", }, }), }), ]); } finally { await context.close(); } }, ); it.each(["expired", "reach", "actor", "generation", "credentials"])( "cancels a queued private callback notice after %s changes without public fallback", async (change) => { const context = await nativePrivateFixture(true); let clock: ReturnType | undefined; try { expect((await context.deliver()).status).toBe(200); const [before] = await context.actions(); expect(before.status).toBe("received"); if (change === "expired") { const receipt = before.payload.telegramCallback as { deadlineAtMs: number; }; clock = vi.spyOn(Date, "now").mockReturnValue(receipt.deadlineAtMs); expect((await context.deliver()).status).toBe(200); expect((await context.actions())[0].payload).toEqual( before.payload, ); } else if (change === "reach") { await db .update(chatEndpointResources) .set({ enabled: false }) .where(eq(chatEndpointResources.endpointId, context.endpoint.id)); } else if (change === "actor") { await context.service.revokeLink( context.endpoint.id, before.principalId!, ); } else if (change === "generation") { const [endpoint] = await db .select() .from(chatEndpoints) .where(eq(chatEndpoints.id, context.endpoint.id)); await db .update(chatEndpoints) .set({ setup: { ...endpoint.setup, runtimeGeneration: Number(endpoint.setup.runtimeGeneration) + 1, }, }) .where(eq(chatEndpoints.id, context.endpoint.id)); } else { await db .update(toolConnections) .set({ credentialSecretRefs: [] }) .where(eq(toolConnections.id, context.endpoint.connectionId)); } await context.service.processPendingProviderEffects(); expect((await context.actions())[0]).toMatchObject({ status: "cancelled", result: { code: "provider_effect_no_longer_authorized" }, }); expect( context.providerRequests.filter( (entry) => entry.method === "sendMessage", ), ).toHaveLength(0); expect( context.runtime.endpoints.get(context.endpoint.id)?.posts ?? [], ).toHaveLength(0); expect(context.wakeup).not.toHaveBeenCalled(); } finally { clock?.mockRestore(); await context.close(); } }, ); it.each([false, true])( "retains the original private callback deadline across restart (expired=%s)", async (expired) => { const context = await nativePrivateFixture(); let clock: ReturnType | undefined; try { expect((await context.deliver()).status).toBe(200); const [before] = await context.actions(); const receipt = before.payload.telegramCallback as { deadlineAtMs: number; }; const resumed = await context.restart(); if (expired) clock = vi.spyOn(Date, "now").mockReturnValue(receipt.deadlineAtMs); await resumed.service.processPendingProviderEffects(); expect((await context.actions())[0]).toMatchObject({ status: expired ? "cancelled" : "processed", payload: before.payload, }); expect( context.providerRequests.filter( (entry) => entry.method === "sendMessage", ), ).toHaveLength(expired ? 0 : 1); expect(resumed.wakeup).not.toHaveBeenCalled(); } finally { clock?.mockRestore(); await context.close(); } }, ); it("quarantines missing private acceptance receipts and never retries them publicly", async () => { const context = await nativePrivateFixture(); try { context.setResponseMode("missing_receipt"); expect((await context.deliver()).status).toBe(200); await context.service.processPendingProviderEffects(); expect((await context.actions())[0]).toMatchObject({ status: "delivery_unknown", }); await context.service.processPendingProviderEffects(); expect((await context.deliver()).status).toBe(200); expect( context.providerRequests.filter( (entry) => entry.method === "sendMessage", ), ).toHaveLength(1); expect( context.runtime.endpoints.get(context.endpoint.id)!.posts, ).toHaveLength(0); } finally { await context.close(); } }); it("preserves an authenticated exact-actor DM denial without native ephemeral or public fallback", async () => { const context = await nativePrivateFixture(false, true); try { expect((await context.deliver()).status).toBe(200); expect(await context.actions()).toHaveLength(1); await context.service.processPendingProviderEffects(); expect((await context.actions())[0]).toMatchObject({ status: "processed", result: { code: "telegram_private_api_accepted" }, }); expect( context.providerRequests.filter( (entry) => entry.method === "sendMessage", ), ).toEqual([ { method: "sendMessage", body: { chat_id: "456", text: "This Paperclip action is no longer available. Open the linked task or ask an operator to link this account.", }, }, ]); expect( context.runtime.endpoints.get(context.endpoint.id)!.posts, ).toHaveLength(0); } finally { await context.close(); } }); it("does not treat another actor's private chat as a reply destination", async () => { const context = await nativePrivateFixture(false, true); try { expect( (await context.deliver(context.payload("dm-cross-actor", 457))) .status, ).toBe(200); expect(await context.actions()).toEqual([]); expect(await context.deliveries()).toEqual([ expect.objectContaining({ state: "filtered" }), ]); expect( context.providerRequests.filter( (entry) => entry.method === "sendMessage", ), ).toHaveLength(0); } finally { await context.close(); } }); it("does not refresh the callback deadline after service runtime-readiness waits", async () => { const context = await nativePrivateFixture(); const enteredAt = Date.now(); let now = enteredAt; const clock = vi.spyOn(Date, "now").mockImplementation(() => now); const get = context.runtime.get.bind(context.runtime); const readiness = vi .spyOn(context.runtime, "get") .mockImplementation((endpointId) => { now = enteredAt + 16_000; return get(endpointId); }); try { expect((await context.deliver()).status).toBe(200); expect(readiness).toHaveBeenCalled(); expect((await context.deliveries())[0]).toMatchObject({ normalizedEvent: { telegramCallback: { receivedAtMs: enteredAt, deadlineAtMs: enteredAt + 15_000, }, }, }); expect(await context.actions()).toEqual([]); expect( context.providerRequests.filter( (entry) => entry.method === "sendMessage", ), ).toHaveLength(0); } finally { readiness.mockRestore(); clock.mockRestore(); await context.close(); } }); it.each([false, true])( "retains an exact-actor DM proof across restart (expired=%s)", async (expired) => { const context = await nativePrivateFixture(false, true); let clock: ReturnType | undefined; try { expect((await context.deliver()).status).toBe(200); const [before] = await context.actions(); const resumed = await context.restart(); if (expired) clock = vi .spyOn(Date, "now") .mockReturnValue( (before.payload.telegramCallback as { deadlineAtMs: number }) .deadlineAtMs, ); await resumed.service.processPendingProviderEffects(); expect((await context.actions())[0]).toMatchObject({ status: expired ? "cancelled" : "processed", payload: before.payload, }); expect( context.providerRequests.filter( (entry) => entry.method === "sendMessage", ), ).toHaveLength(expired ? 0 : 1); } finally { clock?.mockRestore(); await context.close(); } }, ); }); it("coalesces Telegram status and final output into one run-scoped provider message", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture); if (!callbacks.onSlashCommand) { throw new Error("Telegram slash command callback was not registered"); } const chatId = "77118899"; const thread = makeThread({ channelId: chatId, id: `telegram:${chatId}`, isDM: true, name: "Telegram status ordering", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: thread.thread, message: makeMessage({ id: `${chatId}:901`, raw: { message_id: 901 }, text: "Return one delayed answer", userId: chatId, }), trigger: "direct_message", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); await db .update(issues) .set({ status: "in_progress", updatedAt: new Date() }) .where(eq(issues.id, conversation.issueId)); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "telegram", providerMessageId: `${chatId}:901`, }), }); await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${runId}:working:${endpoint.id}`, payload: { text: "Maya is working…", progressState: "working" }, state: "pending", }); await service.processPendingPublications(); await callbacks.onSlashCommand({ endpointId: endpoint.id, provider: "telegram", event: { channel: { id: thread.thread.id, name: "Telegram status ordering", isDM: true, post: vi.fn(), } as never, command: "/status", text: "", user: { userId: chatId, userName: "telegram-status-user", fullName: "Telegram Status User", isBot: false, isMe: false, isSystem: false, }, raw: { message_id: 902, date: Math.floor(Date.now() / 1_000), chat: { id: Number(chatId), type: "private" }, from: { id: Number(chatId), is_bot: false }, text: "/status", entities: [{ offset: 0, length: 7, type: "bot_command" }], }, adapter: {} as never, openModal: async () => undefined, }, }); const providerRuntime = runtime.endpoints.get(endpoint.id); expect(providerRuntime?.posts.map((post) => post.text)).toEqual([ "Maya is working…", ]); const statusPublication = ( await db .select() .from(chatPublications) .where(eq(chatPublications.endpointId, endpoint.id)) ).find((publication) => publication.idempotencyKey.startsWith("control:status:"), ); expect(statusPublication).toMatchObject({ conversationId: conversation.id, state: "pending", payload: { text: expect.stringMatching(/— in_progress$/) }, }); // Sample again at send time so a status waiting behind an older provider // operation cannot report a state Paperclip has already left. await db .update(issues) .set({ status: "done", completedAt: new Date(), updatedAt: new Date() }) .where(eq(issues.id, conversation.issueId)); await service.processPendingPublications(); expect(providerRuntime?.posts.map((post) => post.text)).toEqual([ "Maya is working…", ]); expect(providerRuntime?.edits).toEqual([ { threadId: thread.thread.id, messageId: "outbound-1", text: expect.stringMatching(/— done$/), }, ]); await addSelectedChatFinal({ agentId: fixture.assignedAgentId, body: "telegram-status-race-final", companyId: fixture.companyId, issueId: conversation.issueId, runId, }); await service.processPendingPublications(); // Both updates own the run's existing provider message. Applying the edit // log yields one terminal message with no stale working/status sibling. expect(providerRuntime?.posts.map((post) => post.text)).toEqual([ "Maya is working…", ]); expect(providerRuntime?.edits).toEqual([ { threadId: thread.thread.id, messageId: "outbound-1", text: expect.stringMatching(/— done$/), }, { threadId: thread.thread.id, messageId: "outbound-1", text: "telegram-status-race-final", }, ]); const renderedTelegramMessages = new Map( providerRuntime!.posts.map((post, index) => [ `outbound-${index + 1}`, post.text, ]), ); for (const edit of providerRuntime!.edits) renderedTelegramMessages.set(edit.messageId, edit.text); expect([...renderedTelegramMessages.values()]).toEqual([ "telegram-status-race-final", ]); const publishedStatus = await db .select() .from(chatPublications) .where(eq(chatPublications.id, statusPublication!.id)) .then((rows) => rows[0]); expect(publishedStatus).toMatchObject({ state: "published", payload: { text: expect.stringMatching(/— done$/) }, providerMessageId: "outbound-1", }); await service.shutdown(); }); it("publishes the old final before /new and suppresses later output from the completed generation", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture); if (!callbacks.onSlashCommand) { throw new Error("Telegram slash command callback was not registered"); } const chatId = "77118898"; const thread = makeThread({ channelId: chatId, id: `telegram:${chatId}`, isDM: true, name: "Telegram generation ordering", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: thread.thread, message: makeMessage({ id: `${chatId}:910`, raw: { message_id: 910, date: 1_788_622_910 }, text: "Finish the old generation before starting another", userId: chatId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, chatId); await service.test(endpoint.id, "owner-user"); const [oldConversation] = await service.listConversations(endpoint.id); const oldRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: oldRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: oldConversation.issueId, provider: "telegram", providerMessageId: `${chatId}:910`, }), }); await addSelectedChatFinal({ agentId: fixture.assignedAgentId, body: "old-generation-final", companyId: fixture.companyId, issueId: oldConversation.issueId, runId: oldRunId, }); const newEvent = { endpointId: endpoint.id, provider: "telegram" as const, event: { channel: { id: thread.thread.id, name: "Telegram generation ordering", isDM: true, } as never, command: "/new", text: "", user: { userId: chatId, userName: "telegram-generation-user", fullName: "Telegram Generation User", isBot: false, isMe: false, isSystem: false, }, raw: { message_id: 911, date: 1_788_622_911, chat: { id: Number(chatId), type: "private" }, from: { id: Number(chatId), is_bot: false }, text: "/new", entities: [{ offset: 0, length: 4, type: "bot_command" }], }, adapter: {} as never, openModal: async () => undefined, }, }; await callbacks.onSlashCommand(newEvent); expect(runtime.endpoints.get(endpoint.id)?.posts).toEqual([]); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ id: oldConversation.id, state: "active" }), ]); await service.processPendingPublications(); expect( runtime.endpoints.get(endpoint.id)?.posts.map((post) => post.text), ).toEqual([ "old-generation-final", "Send your request to start a new Paperclip task.", ]); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ id: oldConversation.id, state: "completed" }), ]); const lateOldFinal = await issueService(db).addComment( oldConversation.issueId, "late-old-generation-final-stays-in-paperclip", { agentId: fixture.assignedAgentId, runId: oldRunId }, { authorType: "agent", authorizationReason: "paperclip_runner_protocol" }, ); await expect( db .select({ id: chatPublications.id }) .from(chatPublications) .where(eq(chatPublications.commentId, lateOldFinal.id)), ).resolves.toEqual([]); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: thread.thread, message: makeMessage({ id: `${chatId}:912`, raw: { message_id: 912, date: 1_788_622_912 }, text: "Start the new generation", userId: chatId, }), trigger: "direct_message", }); const conversations = await service.listConversations(endpoint.id); const newConversation = conversations.find( (conversation) => conversation.id !== oldConversation.id, ); if (!newConversation) throw new Error("Expected a new chat generation"); expect(newConversation.sessionGeneration).toBe( oldConversation.sessionGeneration + 1, ); const newRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: newRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: newConversation.issueId, provider: "telegram", providerMessageId: `${chatId}:912`, }), }); await addSelectedChatFinal({ agentId: fixture.assignedAgentId, body: "new-generation-final", companyId: fixture.companyId, issueId: newConversation.issueId, runId: newRunId, }); await service.processPendingPublications(); expect( runtime.endpoints.get(endpoint.id)?.posts.map((post) => post.text), ).toEqual([ "old-generation-final", "Send your request to start a new Paperclip task.", "new-generation-final", ]); expect( await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toHaveLength(2); }); it("keeps task-bound /close active after an ambiguous publication commit until explicit confirmation", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture); if (!callbacks.onSlashCommand) { throw new Error("Telegram slash command callback was not registered"); } const chatId = "77118897"; const thread = makeThread({ channelId: chatId, id: `telegram:${chatId}`, isDM: true, name: "Telegram close commit crash", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: thread.thread, message: makeMessage({ id: `${chatId}:920`, raw: { message_id: 920 }, text: "Keep this task active until close is visible", userId: chatId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, chatId); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); await callbacks.onSlashCommand({ endpointId: endpoint.id, provider: "telegram", event: { channel: { id: thread.thread.id, name: "Telegram close commit crash", isDM: true, } as never, command: "/close", text: "", user: { userId: chatId, userName: "telegram-close-user", fullName: "Telegram Close User", isBot: false, isMe: false, isSystem: false, }, raw: { message_id: 921, date: 1_788_622_921, chat: { id: Number(chatId), type: "private" }, from: { id: Number(chatId), is_bot: false }, text: "/close", entities: [{ offset: 0, length: 6, type: "bot_command" }], }, adapter: {} as never, openModal: async () => undefined, }, }); const closePublication = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), like(chatPublications.idempotencyKey, "control:close:%"), ), ) .then((rows) => rows[0]); expect(closePublication).toMatchObject({ state: "pending" }); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ id: conversation.id, state: "active" }), ]); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Telegram runtime"); const originalTransaction = db.transaction.bind(db); let injectedCrash = false; const transactionSpy = vi .spyOn(db, "transaction") .mockImplementation((async ( ...args: Parameters ) => { if (!injectedCrash && providerRuntime.posts.length === 1) { injectedCrash = true; throw new Error("injected task-control publication commit crash"); } return originalTransaction(...args); }) as typeof db.transaction); try { await expect(service.processPendingPublications()).resolves.toBe(1); } finally { transactionSpy.mockRestore(); } expect(injectedCrash).toBe(true); expect(providerRuntime.posts.map((post) => post.text)).toEqual([ "This chat conversation is closed. Send another message to start a new task.", ]); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ id: conversation.id, state: "active" }), ]); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.id, closePublication!.id)), ).resolves.toEqual([{ state: "delivery_unknown" }]); await service.processPendingPublications(); expect(providerRuntime.posts).toHaveLength(1); await service.resolvePublication( endpoint.id, closePublication!.id, "mark_delivered", "owner-user", ); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ id: conversation.id, state: "completed" }), ]); expect(providerRuntime.posts).toHaveLength(1); }); async function waitForProcessedReceiptRemoval( endpointId: string, receipt: { threadId: string; messageId: string; emoji: string }, ) { // Publication settlement only schedules this non-critical provider I/O. // A sweep skips an action already owned by a fresh processing worker. const removals = await db .select({ id: chatActions.id, deliveryId: chatActions.deliveryId, providerActionId: chatActions.providerActionId, }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpointId), eq(chatActions.kind, "receipt_reaction"), sql`${chatActions.payload}->>'operation' = 'remove'`, sql`${chatActions.payload}->>'threadId' = ${receipt.threadId}`, sql`${chatActions.payload}->>'messageId' = ${receipt.messageId}`, sql`${chatActions.payload}->>'reaction' = ${receipt.emoji}`, ), ); expect(removals).toHaveLength(1); const removal = removals[0]!; expect(removal.deliveryId).toEqual(expect.any(String)); expect(removal.providerActionId).toBe( `receipt_reaction_remove:${removal.deliveryId}`, ); await vi.waitFor(async () => { await expect( db .select({ status: chatActions.status }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpointId), eq(chatActions.id, removal.id), ), ), ).resolves.toEqual([{ status: "processed" }]); }); return removal.id; } describe("Telegram close-owned progress retirement", () => { async function closeProgressFixture( provider: "telegram" | "slack" | "microsoft-teams" = "telegram", ) { const fixture = await seedCompany(); const context = provider === "telegram" ? await configuredTelegramEndpoint(fixture) : provider === "slack" ? await configuredSlackEndpoint(fixture) : await configuredTeamsEndpoint(fixture); const { callbacks, endpoint, runtime, service } = context; const chatId = provider === "telegram" ? "77118896" : provider === "slack" ? "D09CLOSEPROGRESS" : `teams:${Buffer.from("a:close-progress-personal").toString("base64url")}`; const userId = provider === "telegram" ? chatId : "U-CLOSE-PROGRESS"; const sourceMessageId = provider === "telegram" ? `${chatId}:930` : "1788969999.000930"; const thread = makeThread({ channelId: chatId, id: provider === "telegram" ? `telegram:${chatId}` : provider === "slack" ? `slack:${chatId}:` : chatId, isDM: true, }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider, thread: thread.thread, message: makeMessage({ id: sourceMessageId, raw: provider === "telegram" ? { message_id: 930 } : { ts: sourceMessageId }, text: "Work until I close this conversation", userId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, userId); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider, providerMessageId: sourceMessageId, }), }); const [progress] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${runId}:working:${endpoint.id}`, payload: { text: "Maya is working…", progressState: "working" }, state: "pending", }) .returning(); await service.processPendingPublications(); const providerRuntime = runtime.endpoints.get(endpoint.id)!; const closeEvent = { endpointId: endpoint.id, provider: "telegram" as const, event: { channel: { id: thread.thread.id, name: "Telegram close retirement", isDM: true, } as never, command: "/close", text: "", user: { userId: chatId, userName: "close-user", fullName: "Close User", isBot: false, isMe: false, isSystem: false, }, raw: { message_id: 931, date: Math.floor(Date.now() / 1000), chat: { id: Number(chatId), type: "private" }, from: { id: Number(chatId), is_bot: false }, text: "/close", entities: [{ offset: 0, length: 6, type: "bot_command" }], }, adapter: {} as never, openModal: async () => undefined, }, }; const stageClose = async () => { if (provider === "microsoft-teams") await deliverMessage({ callbacks, endpointId: endpoint.id, provider, thread: thread.thread, message: makeMessage({ id: "teams-close-progress-control", text: "/close", userId, }), trigger: "direct_message", }); else await callbacks.onSlashCommand!( provider === "telegram" ? closeEvent : { endpointId: endpoint.id, provider: "slack", event: { channel: { id: `slack:${chatId}`, name: "Close progress DM", isDM: true, } as never, command: endpoint.setup.command!, text: "close", triggerId: `close-progress-${randomUUID()}`, user: { ...closeEvent.event.user, userId }, raw: { command: endpoint.setup.command!, text: "close" }, adapter: {} as never, openModal: async () => undefined, }, }, ); return db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), like(chatPublications.idempotencyKey, "control:close:%"), ), ) .then((rows) => rows[0]!); }; const progressMessageId = await db .select({ id: chatPublications.providerMessageId }) .from(chatPublications) .where(eq(chatPublications.id, progress!.id)) .then((rows) => rows[0]!.id!); return { ...context, ...fixture, conversation, runId, progress: progress!, progressMessageId, providerRuntime, stageClose, closeEvent, sourceMessageId, }; } it("retires one working lane on close without cancelling the run or publishing its late final", async () => { const f = await closeProgressFixture(); let releaseRemoval!: () => void; let removalStarted = false; const removalRelease = new Promise((resolve) => { releaseRemoval = resolve; }); const originalThread = f.providerRuntime.thread.bind(f.providerRuntime); const threadSpy = vi .spyOn(f.providerRuntime, "thread") .mockImplementation((threadId) => { const thread = originalThread(threadId); const removeReaction = thread.adapter.removeReaction; thread.adapter.removeReaction = async ( reactionThreadId, messageId, emoji, ) => { if ( reactionThreadId === "telegram:77118896" && messageId === f.sourceMessageId && emoji === "eyes" ) { removalStarted = true; await removalRelease; } await removeReaction(reactionThreadId, messageId, emoji); }; return thread; }); try { const close = await f.stageClose(); await f.service.processPendingPublications(); await vi.waitFor(() => expect(removalStarted).toBe(true)); const removals = await db .select({ id: chatActions.id, status: chatActions.status }) .from(chatActions) .where( and( eq(chatActions.endpointId, f.endpoint.id), eq(chatActions.kind, "receipt_reaction"), sql`${chatActions.payload}->>'operation' = 'remove'`, sql`${chatActions.payload}->>'messageId' = ${f.sourceMessageId}`, ), ); expect(removals).toEqual([ { id: expect.any(String), status: "processing" }, ]); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.id, close.id)), ).resolves.toEqual([{ state: "published" }]); await expect( f.service.processPendingReceiptReactions(25, removals[0]!.id), ).resolves.toBe(0); expect(f.providerRuntime.removedReactions).not.toContainEqual({ threadId: "telegram:77118896", messageId: f.sourceMessageId, emoji: "eyes", }); releaseRemoval(); expect( await waitForProcessedReceiptRemoval(f.endpoint.id, { threadId: "telegram:77118896", messageId: f.sourceMessageId, emoji: "eyes", }), ).toBe(removals[0]!.id); expect(f.providerRuntime.posts.map((post) => post.text)).toEqual([ "Maya is working…", ]); expect(f.providerRuntime.edits).toEqual([ { threadId: `telegram:77118896`, messageId: f.progressMessageId, text: "This chat conversation is closed. Send another message to start a new task.", }, ]); expect(f.providerRuntime.removedReactions).toContainEqual({ threadId: "telegram:77118896", messageId: "77118896:930", emoji: "eyes", }); await expect( db .select({ status: heartbeatRuns.status }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, f.runId)), ).resolves.toEqual([{ status: "running" }]); await expect( db .select({ state: chatConversations.state }) .from(chatConversations) .where(eq(chatConversations.id, f.conversation.id)), ).resolves.toEqual([{ state: "completed" }]); await db .update(heartbeatRuns) .set({ status: "succeeded", finishedAt: new Date() }) .where(eq(heartbeatRuns.id, f.runId)); const late = await issueService(db).addComment( f.conversation.issueId, "Late output remains on the task", { agentId: f.assignedAgentId, runId: f.runId }, { authorType: "agent", authorizationReason: "paperclip_runner_protocol", }, ); await expect( db .select({ id: chatPublications.id }) .from(chatPublications) .where(eq(chatPublications.commentId, late.id)), ).resolves.toEqual([]); await f.service.processPendingPublications(); await f.callbacks.onSlashCommand!(f.closeEvent); await f.service.processPendingPublications(); expect(f.providerRuntime.posts).toHaveLength(1); expect(f.providerRuntime.edits).toHaveLength(1); await expect( db .select({ state: chatPublications.state, attempts: chatPublications.attempts, providerMessageId: chatPublications.providerMessageId, }) .from(chatPublications) .where(eq(chatPublications.id, close.id)), ).resolves.toEqual([ { state: "published", attempts: 1, providerMessageId: f.progressMessageId, }, ]); } finally { releaseRemoval(); threadSpy.mockRestore(); await retirePublicationFixture(f.service, f.endpoint.id); } }); it.each([ "missing_link", "different_run_owner", "multiple_lanes", "pending_final", "unknown_final", "consumed_link", ] as const)("does not guess a close edit for %s", async (mode) => { const f = await closeProgressFixture(); try { if (mode === "missing_link") await db .delete(chatMessageLinks) .where( and( eq(chatMessageLinks.endpointId, f.endpoint.id), eq(chatMessageLinks.direction, "outbound"), eq(chatMessageLinks.providerMessageId, f.progressMessageId), ), ); if (mode === "different_run_owner") await db .update(heartbeatRuns) .set({ agentId: f.replacementAgentId }) .where(eq(heartbeatRuns.id, f.runId)); if (mode === "multiple_lanes") { const anotherRun = randomUUID(); await db .insert(heartbeatRuns) .values({ id: anotherRun, companyId: f.companyId, agentId: f.assignedAgentId, status: "running", contextSnapshot: await chatWakeContext({ endpointId: f.endpoint.id, issueId: f.conversation.issueId, provider: "telegram", providerMessageId: "77118896:930", }), }); await db .insert(chatPublications) .values({ companyId: f.companyId, endpointId: f.endpoint.id, conversationId: f.conversation.id, issueId: f.conversation.issueId, idempotencyKey: `run:${anotherRun}:working:${f.endpoint.id}`, payload: { text: "Another owned run is working…", progressState: "working", }, state: "pending", }); await f.service.processPendingPublications(); expect(f.providerRuntime.posts).toHaveLength(2); } const close = await f.stageClose(); if ( mode === "pending_final" || mode === "unknown_final" || mode === "consumed_link" ) { const final = await addSelectedChatFinal({ agentId: f.assignedAgentId, body: "An authored answer must never be overwritten", companyId: f.companyId, issueId: f.conversation.issueId, runId: f.runId, }); const [publication] = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, final.id)); // A later final can be unconfirmed while the earlier close is the // FIFO head. The old progress link is not proof of remote absence. await db .update(chatPublications) .set({ state: mode === "pending_final" ? "pending" : mode === "unknown_final" ? "delivery_unknown" : "published", createdAt: new Date(close.createdAt.getTime() + 1000), ...(mode === "consumed_link" ? { providerMessageId: f.progressMessageId, publishedAt: new Date(), } : {}), }) .where(eq(chatPublications.id, publication.id)); if (mode === "consumed_link") await db .update(chatMessageLinks) .set({ publicationId: publication.id, commentId: final.id }) .where( and( eq(chatMessageLinks.endpointId, f.endpoint.id), eq(chatMessageLinks.direction, "outbound"), eq(chatMessageLinks.providerMessageId, f.progressMessageId), ), ); } const before = f.providerRuntime.posts.length; await f.service.processPendingPublications(); expect(f.providerRuntime.edits).toEqual([]); expect(f.providerRuntime.posts).toHaveLength(before + 1); expect(f.providerRuntime.posts.at(-1)?.text).toBe( "This chat conversation is closed. Send another message to start a new task.", ); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.id, close.id)), ).resolves.toEqual([{ state: "published" }]); } finally { await retirePublicationFixture(f.service, f.endpoint.id); } }); it.each( (["ask_user_questions", "request_confirmation"] as const).flatMap((kind) => (["pending", "delivery_unknown"] as const).map((state) => ({ kind, state, })), ), )( "does not reclaim a progress lane consumed by a $state $kind prompt", async ({ kind, state }) => { const f = await closeProgressFixture(); try { const close = await f.stageClose(); const interaction = await issueThreadInteractionService(db).create( { id: f.conversation.issueId, companyId: f.companyId }, { kind, continuationPolicy: "wake_assignee", sourceRunId: f.runId, payload: kind === "request_confirmation" ? { version: 1, prompt: "Proceed with the operation?" } : { version: 1, questions: [ { id: "choice", prompt: "Which option?", selectionMode: "single", required: true, allowOther: false, options: [ { id: "yes", label: "Yes" }, { id: "no", label: "No" }, ], }, ], }, }, { agentId: f.assignedAgentId, runId: f.runId }, ); const [prompt] = await db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `interaction:${interaction.id}:${f.endpoint.id}`, ), ); expect(prompt).toMatchObject({ commentId: null, payload: { interactionId: interaction.id }, }); // Retained ordering fixture, like the unknown-final control above: // a close with an earlier transaction timestamp does not prove a later // prompt's remote edit was absent. This is not a live HTTP/crash proof. await db .update(chatPublications) .set({ state, createdAt: new Date(close.createdAt.getTime() + 1000) }) .where(eq(chatPublications.id, prompt.id)); const beforePosts = f.providerRuntime.posts.length; const beforeRemovals = f.providerRuntime.removedReactions.length; await f.service.processPendingPublications(); await f.service.processPendingReceiptReactions(); expect(f.providerRuntime.edits).toEqual([]); expect(f.providerRuntime.posts).toHaveLength(beforePosts + 1); expect(f.providerRuntime.posts.at(-1)?.text).toBe(close.payload.text); expect(f.providerRuntime.removedReactions).toHaveLength(beforeRemovals); expect( ( await db .select() .from(chatPublications) .where(eq(chatPublications.id, close.id)) )[0], ).toMatchObject({ state: "published" }); expect( ( await db .select() .from(chatPublications) .where(eq(chatPublications.id, prompt.id)) )[0]?.state, ).toBe(state === "delivery_unknown" ? "delivery_unknown" : "cancelled"); expect( ( await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, f.runId)) )[0]?.status, ).toBe("running"); } finally { await retirePublicationFixture(f.service, f.endpoint.id); } }, ); it("does not borrow an unrelated run's interaction as progress-consumer evidence", async () => { const f = await closeProgressFixture(); try { const close = await f.stageClose(); const otherRunId = randomUUID(); await db .insert(heartbeatRuns) .values({ id: otherRunId, companyId: f.companyId, agentId: f.assignedAgentId, status: "running", contextSnapshot: { issueId: f.conversation.issueId }, }); const interaction = await issueThreadInteractionService(db).create( { id: f.conversation.issueId, companyId: f.companyId }, { kind: "request_confirmation", continuationPolicy: "wake_assignee", sourceRunId: otherRunId, payload: { version: 1, prompt: "Another run's confirmation" }, }, { agentId: f.assignedAgentId, runId: otherRunId }, ); const [prompt] = await db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `interaction:${interaction.id}:${f.endpoint.id}`, ), ); expect(prompt).toBeDefined(); await db .update(chatPublications) .set({ state: "delivery_unknown", createdAt: new Date(close.createdAt.getTime() + 1000), }) .where(eq(chatPublications.id, prompt.id)); await f.service.processPendingPublications(); expect(f.providerRuntime.edits).toEqual([ { threadId: "telegram:77118896", messageId: f.progressMessageId, text: close.payload.text, }, ]); expect( ( await db .select() .from(chatPublications) .where(eq(chatPublications.id, prompt.id)) )[0]?.state, ).toBe("delivery_unknown"); } finally { await retirePublicationFixture(f.service, f.endpoint.id); } }); it.each(["final", "confirmation"] as const)( "preserves an attempted unknown %s after public operator cancellation", async (kind) => { const f = await closeProgressFixture(); let lostReceipt = false; const originalThread = f.providerRuntime.thread.bind(f.providerRuntime); const threadSpy = vi .spyOn(f.providerRuntime, "thread") .mockImplementation((threadId) => { const thread = originalThread(threadId); const edit = thread.adapter.editMessage.bind(thread.adapter); thread.adapter.editMessage = async (...args) => { const receipt = await edit(...args); if (!lostReceipt) { lostReceipt = true; throw Object.assign(new Error("Consumer edit receipt lost"), { name: "NetworkError", code: "NETWORK_ERROR", }); } return receipt; }; return thread; }); try { let consumerId: string; if (kind === "final") { const comment = await addSelectedChatFinal({ agentId: f.assignedAgentId, body: "Preserve this possibly delivered final", companyId: f.companyId, issueId: f.conversation.issueId, runId: f.runId, }); consumerId = await db .select({ id: chatPublications.id }) .from(chatPublications) .where(eq(chatPublications.commentId, comment.id)) .then((rows) => rows[0]!.id); } else { const interaction = await issueThreadInteractionService(db).create( { id: f.conversation.issueId, companyId: f.companyId }, { kind: "request_confirmation", continuationPolicy: "wake_assignee", sourceRunId: f.runId, payload: { version: 1, prompt: "Preserve this confirmation?" }, }, { agentId: f.assignedAgentId, runId: f.runId }, ); consumerId = await db .select({ id: chatPublications.id }) .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `interaction:${interaction.id}:${f.endpoint.id}`, ), ) .then((rows) => rows[0]!.id); } await f.service.processPendingPublications(); threadSpy.mockRestore(); expect(lostReceipt).toBe(true); expect(f.providerRuntime.edits).toHaveLength(1); expect(f.providerRuntime.edits[0]?.messageId).toBe(f.progressMessageId); expect( await db .select() .from(chatPublications) .where(eq(chatPublications.id, consumerId)), ).toEqual([ expect.objectContaining({ state: "delivery_unknown", attempts: 1 }), ]); const close = await f.stageClose(); const beforePosts = f.providerRuntime.posts.length; // This public resolution stops retries, then drains the pending close. // The mocked provider edit above happened; its receipt was unconfirmed. await f.service.resolvePublication( f.endpoint.id, consumerId, "cancel", "owner-user", ); expect( await db .select() .from(chatPublications) .where(eq(chatPublications.id, consumerId)), ).toEqual([ expect.objectContaining({ state: "cancelled", attempts: 1, redactedError: "Cancelled by an operator after an unconfirmed provider delivery", }), ]); expect( await db .select() .from(activityLog) .where( and( eq(activityLog.entityId, consumerId), eq(activityLog.action, "chat.publication_cancel"), ), ), ).toEqual([ expect.objectContaining({ details: expect.objectContaining({ previousState: "delivery_unknown", }), }), ]); expect(f.providerRuntime.edits).toHaveLength(1); expect(f.providerRuntime.posts).toHaveLength(beforePosts + 1); expect(f.providerRuntime.posts.at(-1)?.text).toBe(close.payload.text); expect( await db .select() .from(chatPublications) .where(eq(chatPublications.id, close.id)), ).toEqual([expect.objectContaining({ state: "published" })]); expect( await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, f.runId)), ).toEqual([expect.objectContaining({ status: "running" })]); } finally { threadSpy.mockRestore(); await retirePublicationFixture(f.service, f.endpoint.id); } }, ); it.each([ { state: "cancelled" as const, attempts: 0, mayEdit: true }, { state: "failed" as const, attempts: 1, mayEdit: false }, ])( "treats a $state consumer with $attempts attempts conservatively", async ({ state, attempts, mayEdit }) => { const f = await closeProgressFixture(); try { const comment = await addSelectedChatFinal({ agentId: f.assignedAgentId, body: "Retained terminal consumer", companyId: f.companyId, issueId: f.conversation.issueId, runId: f.runId, }); // Retained terminal-state controls: no provider-call claim for this fixture. await db .update(chatPublications) .set({ state, attempts }) .where(eq(chatPublications.commentId, comment.id)); const close = await f.stageClose(); const beforePosts = f.providerRuntime.posts.length; await f.service.processPendingPublications(); expect(f.providerRuntime.edits).toHaveLength(mayEdit ? 1 : 0); expect(f.providerRuntime.posts).toHaveLength( beforePosts + (mayEdit ? 0 : 1), ); expect( await db .select() .from(chatPublications) .where(eq(chatPublications.id, close.id)), ).toEqual([expect.objectContaining({ state: "published" })]); } finally { await retirePublicationFixture(f.service, f.endpoint.id); } }, ); it("preserves an authored final that wins while close is admitted", async () => { const f = await closeProgressFixture(); let release!: () => void; const held = new Promise((resolve) => { release = resolve; }); let entered!: () => void; const entering = new Promise((resolve) => { entered = resolve; }); const originalThread = f.providerRuntime.thread.bind(f.providerRuntime); const threadSpy = vi .spyOn(f.providerRuntime, "thread") .mockImplementation((threadId) => { const thread = originalThread(threadId); const edit = thread.adapter.editMessage.bind(thread.adapter); thread.adapter.editMessage = async (...args) => { if ((args[2] as { markdown?: string }).markdown === "The final won") { entered(); await held; } return edit(...args); }; return thread; }); let drain: Promise | undefined; try { await addSelectedChatFinal({ agentId: f.assignedAgentId, body: "The final won", companyId: f.companyId, issueId: f.conversation.issueId, runId: f.runId, }); drain = f.service.processPendingPublications(); await entering; await f.stageClose(); expect(f.providerRuntime.edits).toEqual([]); release(); await drain; await f.service.processPendingPublications(); expect(f.providerRuntime.edits).toEqual([ { threadId: "telegram:77118896", messageId: f.progressMessageId, text: "The final won", }, ]); expect(f.providerRuntime.posts.map((post) => post.text)).toEqual([ "Maya is working…", "This chat conversation is closed. Send another message to start a new task.", ]); } finally { release(); await drain; threadSpy.mockRestore(); await retirePublicationFixture(f.service, f.endpoint.id); } }); it("retires a status-interleaved working lane", async () => { const f = await closeProgressFixture(); try { const event = f.closeEvent; await f.callbacks.onSlashCommand!({ ...event, event: { ...event.event, command: "/status", raw: { ...event.event.raw, message_id: 932, text: "/status", entities: [{ offset: 0, length: 7, type: "bot_command" }], }, }, }); await f.service.processPendingPublications(); expect(f.providerRuntime.edits).toHaveLength(1); await f.stageClose(); await f.service.processPendingPublications(); expect(f.providerRuntime.posts).toHaveLength(1); expect(f.providerRuntime.edits.at(-1)).toMatchObject({ messageId: f.progressMessageId, text: "This chat conversation is closed. Send another message to start a new task.", }); } finally { await retirePublicationFixture(f.service, f.endpoint.id); } }); it.each(["rejected", "unknown_commit"] as const)( "keeps close uncommitted after %s of its retirement edit", async (mode) => { const f = await closeProgressFixture(); let transactionSpy: ReturnType | undefined; try { const close = await f.stageClose(); if (mode === "rejected") f.providerRuntime.editError = Object.assign( new Error("Rejected close edit"), { name: "ValidationError", code: "VALIDATION_ERROR" }, ); else { const originalTransaction = db.transaction.bind(db); let injected = false; transactionSpy = vi .spyOn(db, "transaction") .mockImplementation((async ( ...args: Parameters ) => { if (!injected && f.providerRuntime.edits.length === 1) { injected = true; throw new Error( "Close edit committed remotely without durable receipt", ); } return originalTransaction(...args); }) as typeof db.transaction); } await f.service.processPendingPublications(); transactionSpy?.mockRestore(); transactionSpy = undefined; await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.id, close.id)), ).resolves.toEqual([ { state: mode === "rejected" ? "failed" : "delivery_unknown" }, ]); await expect( db .select({ state: chatConversations.state }) .from(chatConversations) .where(eq(chatConversations.id, f.conversation.id)), ).resolves.toEqual([{ state: "active" }]); await f.service.processPendingPublications(); expect(f.providerRuntime.posts).toHaveLength(1); expect(f.providerRuntime.editAttempts).toHaveLength(1); await expect( db .select({ status: heartbeatRuns.status }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, f.runId)), ).resolves.toEqual([{ status: "running" }]); } finally { transactionSpy?.mockRestore(); await retirePublicationFixture(f.service, f.endpoint.id); } }, ); it.each( (["slack", "microsoft-teams"] as const).flatMap((provider) => (["working", "final", "unknown_final", "card"] as const).map((mode) => ({ provider, mode, })), ), )( "qualifies $provider close-owned progress retirement ($mode)", async ({ provider, mode }) => { const f = await closeProgressFixture(provider); try { if (mode === "final") { await addSelectedChatFinal({ agentId: f.assignedAgentId, body: "The Slack final must remain", companyId: f.companyId, issueId: f.conversation.issueId, runId: f.runId, }); await f.service.processPendingPublications(); } if (mode === "card") await db .update(chatPublications) .set({ payload: { ...f.progress.payload, card: { title: "Not a plain progress lane" }, }, }) .where(eq(chatPublications.id, f.progress.id)); const close = await f.stageClose(); if (mode === "unknown_final") { const final = await addSelectedChatFinal({ agentId: f.assignedAgentId, body: "Possibly delivered Slack final", companyId: f.companyId, issueId: f.conversation.issueId, runId: f.runId, }); await db .update(chatPublications) .set({ state: "delivery_unknown", createdAt: new Date(close.createdAt.getTime() + 1000), }) .where(eq(chatPublications.commentId, final.id)); } const editCount = f.providerRuntime.edits.length, postCount = f.providerRuntime.posts.length; await f.service.processPendingPublications(); await f.service.processPendingReceiptReactions(); expect(f.providerRuntime.edits).toHaveLength( editCount + (mode === "working" ? 1 : 0), ); expect(f.providerRuntime.posts).toHaveLength( postCount + (mode === "working" ? 0 : 1), ); if (mode === "working") { expect(f.providerRuntime.edits.at(-1)).toMatchObject({ messageId: f.progressMessageId, text: close.payload.text, }); if (provider === "slack") { await waitForProcessedReceiptRemoval(f.endpoint.id, { threadId: f.conversation.externalThreadId, messageId: f.sourceMessageId, emoji: "eyes", }); expect(f.providerRuntime.removedReactions).toContainEqual({ threadId: f.conversation.externalThreadId, messageId: f.sourceMessageId, emoji: "eyes", }); } } await expect( db .select({ status: heartbeatRuns.status }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, f.runId)), ).resolves.toEqual([{ status: "running" }]); await f.service.processPendingPublications(); expect(f.providerRuntime.edits).toHaveLength( editCount + (mode === "working" ? 1 : 0), ); } finally { await retirePublicationFixture(f.service, f.endpoint.id); } }, ); }); it.each(["slack", "github"] as const)( "keeps intermediate %s agent comments internal and publishes only the selected final", async (provider) => { const fixture = await seedCompany(); const configured = provider === "slack" ? await configuredSlackEndpoint(fixture) : await configuredGitHubEndpoint(fixture); const { callbacks, endpoint, runtime, service } = configured; const thread = makeThread({ channelId: provider === "slack" ? "C-MULTI-FINAL" : "github:paperclipai/paperclip", id: provider === "slack" ? "slack:C-MULTI-FINAL:4045.1" : "github:paperclipai/paperclip:issue:418", name: provider === "slack" ? "multi-final" : "paperclipai/paperclip", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider, thread: thread.thread, message: makeMessage({ id: provider === "slack" ? "4045.1" : "41801", text: "@maya return one final answer after internal checkpoints", mentioned: true, }), trigger: "mention", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider, providerMessageId: provider === "slack" ? "4045.1" : "41801", }), }); await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${runId}:working:${endpoint.id}`, payload: { text: "Maya is working…", progressState: "working" }, state: "pending", }); await service.processPendingPublications(); const comments = []; for (const body of ["checkpoint-one", "checkpoint-two"]) { comments.push( await issueService(db).addComment( conversation.issueId, body, { agentId: fixture.assignedAgentId, runId }, { authorType: "agent", authorizationReason: "paperclip_runner_protocol", }, ), ); } comments.push( await addSelectedChatFinal({ agentId: fixture.assignedAgentId, body: "answer-final", companyId: fixture.companyId, issueId: conversation.issueId, runId, }), ); await Promise.all([ service.processPendingPublications(), service.processPendingPublications(), service.processPendingPublications(), ]); const providerRuntime = runtime.endpoints.get(endpoint.id); expect(providerRuntime?.posts.map((post) => post.text)).toEqual([ "Maya is working…", ]); expect(providerRuntime?.edits).toEqual([ { threadId: thread.thread.id, messageId: "outbound-1", text: "answer-final", }, ]); const commentPublications = await db .select() .from(chatPublications) .where( inArray( chatPublications.commentId, comments.map((comment) => comment.id), ), ) .orderBy(asc(chatPublications.createdAt), asc(chatPublications.id)); expect(commentPublications).toEqual([ expect.objectContaining({ commentId: comments[2].id, state: "published", providerMessageId: "outbound-1", }), ]); expect( await db .select({ body: issueComments.body }) .from(issueComments) .where( inArray( issueComments.id, comments.map((comment) => comment.id), ), ), ).toHaveLength(3); expect( await db .select() .from(chatMessageLinks) .where( and( eq(chatMessageLinks.conversationId, conversation.id), eq(chatMessageLinks.direction, "outbound"), ), ), ).toHaveLength(1); await service.shutdown(); }, ); it("does not hold another endpoint's final publication behind Slack session status transport", async () => { const firstFixture = await seedCompany(); let blockSessionStatus = false; let blockedSessionStatusCalls = 0; let releaseSessionStatus!: () => void; let sessionStatusStarted!: () => void; let anotherSessionStatusStarted!: () => void; const sessionStatusStartedPromise = new Promise((resolve) => { sessionStatusStarted = resolve; }); const anotherSessionStatusStartedPromise = new Promise((resolve) => { anotherSessionStatusStarted = resolve; }); const sessionStatusReleased = new Promise((resolve) => { releaseSessionStatus = resolve; }); const blockingFetch = ( ordinaryFetch: ReturnType, ): typeof globalThis.fetch => async (input) => { if ( String(input) === "https://slack.com/api/agents.sessions.setStatus" && blockSessionStatus ) { blockedSessionStatusCalls += 1; if (blockedSessionStatusCalls === 1) sessionStatusStarted(); else anotherSessionStatusStarted(); await sessionStatusReleased; return Response.json({ ok: true }); } return ordinaryFetch(input); }; const firstFetch = blockingFetch(fakeSlackFetch()); const first = await configuredSlackEndpoint(firstFixture, { fetch: firstFetch, }); let second: Awaited> | null = null; let secondSessionActionId: string | null = null; let blockedStatusSweep: Promise | null = null; let secondPublicationSweep: Promise | null = null; try { const firstThread = makeThread({ channelId: "CSESSIONBLOCKER", id: "slack:CSESSIONBLOCKER:1710004051.000001", name: "session-blocker", }); await deliverMessage({ callbacks: first.callbacks, endpointId: first.endpoint.id, thread: firstThread.thread, message: makeMessage({ id: "1710004051.000001", text: "@maya start the status blocker", mentioned: true, }), trigger: "mention", }); const firstConversation = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, first.endpoint.id)) .then((rows) => rows[0]!); await db.insert(chatPublications).values({ companyId: firstFixture.companyId, endpointId: first.endpoint.id, conversationId: firstConversation.id, issueId: firstConversation.issueId, idempotencyKey: `session-blocker:${randomUUID()}`, payload: { text: "Maya is working…", progressState: "working" }, state: "pending", }); await first.service.processPendingPublications(); const firstSessionAction = await db .select({ id: chatActions.id }) .from(chatActions) .where( and( eq(chatActions.endpointId, first.endpoint.id), eq(chatActions.kind, "slack_session_sync"), ), ) .then((rows) => rows[0]!); blockSessionStatus = true; blockedStatusSweep = first.service.processPendingSlackSessionSyncs( 25, firstSessionAction.id, ); await sessionStatusStartedPromise; const secondFixture = await seedCompany(); const secondFetch = blockingFetch(fakeSlackFetch()); second = await configuredSlackEndpoint(secondFixture, { fetch: secondFetch, }); const secondThread = makeThread({ channelId: "CSESSIONINDEPENDENT", id: "slack:CSESSIONINDEPENDENT:1710004051.000002", name: "session-independent", }); await deliverMessage({ callbacks: second.callbacks, endpointId: second.endpoint.id, thread: secondThread.thread, message: makeMessage({ id: "1710004051.000002", text: "@maya publish independently", mentioned: true, }), trigger: "mention", }); const secondConversation = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, second.endpoint.id)) .then((rows) => rows[0]!); await db.insert(chatPublications).values({ companyId: secondFixture.companyId, endpointId: second.endpoint.id, conversationId: secondConversation.id, issueId: secondConversation.issueId, idempotencyKey: `session-independent:${randomUUID()}`, payload: { text: "SLACK-SESSION-INDEPENDENT" }, state: "pending", }); secondPublicationSweep = second.service.processPendingPublications(); const publicationOutcome = await Promise.race([ secondPublicationSweep.then(() => "published" as const), anotherSessionStatusStartedPromise.then( () => "blocked_by_session_status" as const, ), ]); expect(publicationOutcome).toBe("published"); expect( second.runtime.endpoints.get(second.endpoint.id)?.posts, ).toContainEqual({ threadId: secondThread.thread.id, text: "SLACK-SESSION-INDEPENDENT", }); secondSessionActionId = await db .select({ id: chatActions.id }) .from(chatActions) .where( and( eq(chatActions.endpointId, second.endpoint.id), eq(chatActions.kind, "slack_session_sync"), ), ) .then((rows) => rows[0]?.id ?? null); } finally { releaseSessionStatus(); await blockedStatusSweep; await secondPublicationSweep; blockSessionStatus = false; if (second && secondSessionActionId) { await second.service.processPendingSlackSessionSyncs( 25, secondSessionActionId, ); } await first.service.shutdown(); await second?.service.shutdown(); } }); it("reconciles Slack session status without replaying messages after rate limits or restart", async () => { const fixture = await seedCompany(); const ordinaryFetch = fakeSlackFetch(); const statusCalls: string[] = []; let failStatus = true; let permanentStatusError: string | null = null; const fetch: typeof globalThis.fetch = async (input, init) => { if (String(input) !== "https://slack.com/api/agents.sessions.setStatus") return ordinaryFetch(input); statusCalls.push( (JSON.parse(String(init?.body)) as { status: string }).status, ); return failStatus ? Response.json( { ok: false, error: "ratelimited" }, { status: 429, headers: { "retry-after": "1800" } }, ) : Response.json( permanentStatusError ? { ok: false, error: permanentStatusError } : { ok: true }, ); }; const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture, { fetch }); const thread = makeThread({ channelId: "CSESSION", id: "slack:CSESSION:4052.1", name: "session-status", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: "4052.1", text: "@maya test session status", mentioned: true, }), trigger: "mention", }); const conversation = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)) .then((rows) => rows[0]!); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "slack", providerMessageId: "4052.1", }), }); const [working] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${runId}:working:${endpoint.id}`, payload: { text: "Maya is working…", progressState: "working" }, state: "pending", }) .returning(); const rateLimitedAt = Date.now(); await service.processPendingPublications(); expect(statusCalls).toEqual([]); expect( await db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.id, working.id)) .then((rows) => rows[0]?.state), ).toBe("published"); const action = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slack_session_sync"), ), ) .then((rows) => rows[0]!); expect(action.status).toBe("received"); await service.processPendingSlackSessionSyncs(25, action.id); expect(statusCalls).toEqual(["processing"]); const deferredAction = await db .select({ result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, action.id)) .then((rows) => rows[0]!); expect( Date.parse(String(deferredAction.result?.retryAt)), ).toBeGreaterThanOrEqual(rateLimitedAt + 1_800_000); await service.processPendingSlackSessionSyncs(25, action.id); expect(statusCalls).toHaveLength(1); const firstProviderRuntime = runtime.endpoints.get(endpoint.id)!; const providerPosts = firstProviderRuntime.posts.length; await service.shutdown(); // A successor reclaims a crashed status attempt. The source run is now // cancelled; its old working receipt must not revive processing. await db .update(heartbeatRuns) .set({ status: "cancelled", updatedAt: new Date() }) .where(eq(heartbeatRuns.id, runId)); await db .update(chatActions) .set({ status: "processing", result: { attempts: 1 }, updatedAt: new Date(Date.now() - 61_000), }) .where(eq(chatActions.id, action.id)); failStatus = false; const resumed = createService(runtime, fetch); await resumed.service.processPendingSlackSessionSyncs(25, action.id); expect(statusCalls).toEqual(["processing", "active"]); expect(firstProviderRuntime.posts.length).toBe(providerPosts); expect(runtime.endpoints.get(endpoint.id)?.posts.length ?? 0).toBe(0); expect( await db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, action.id)) .then((rows) => rows[0]), ).toMatchObject({ status: "processed", result: { sessionStatus: "active" }, }); // A final response produces one edit, then a fresh revision of the status // lane. Neither a retry nor restart posts another copy of that response. await addSelectedChatFinal({ agentId: fixture.assignedAgentId, body: "SLACK-SESSION-DONE", companyId: fixture.companyId, issueId: conversation.issueId, runId, }); await resumed.service.processPendingPublications(); await resumed.service.processPendingPublications(); await resumed.service.processPendingSlackSessionSyncs(25, action.id); expect(statusCalls).toEqual(["processing", "active", "active"]); expect(firstProviderRuntime.posts.length).toBe(providerPosts); expect(runtime.endpoints.get(endpoint.id)?.posts.length ?? 0).toBe(0); expect( runtime.endpoints .get(endpoint.id) ?.edits.filter((edit) => edit.text === "SLACK-SESSION-DONE"), ).toHaveLength(1); // A worker can select a due status row, then lose to another worker before // observing an endpoint pause. Its stale snapshot must not resurrect the // already completed status action or dispatch another provider request. await db .update(chatActions) .set({ status: "received", result: null, updatedAt: new Date(), }) .where(eq(chatActions.id, action.id)); let selected!: () => void; let releaseSelection!: () => void; const selectedPromise = new Promise((resolve) => { selected = resolve; }); const selectionReleased = new Promise((resolve) => { releaseSelection = resolve; }); const competing = createService(runtime, fetch, { slackSessionSyncSelectionBarrier: async () => { selected(); await selectionReleased; }, }); const staleAttempt = competing.service.processPendingSlackSessionSyncs( 25, action.id, ); try { await selectedPromise; await resumed.service.processPendingSlackSessionSyncs(25, action.id); await db .update(chatEndpoints) .set({ status: "paused", updatedAt: new Date() }) .where(eq(chatEndpoints.id, endpoint.id)); } finally { releaseSelection(); await staleAttempt; } expect(statusCalls).toEqual(["processing", "active", "active", "active"]); expect( await db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, action.id)) .then((rows) => rows[0]?.status), ).toBe("processed"); expect(await resumed.service.listActivity(endpoint.id)).toEqual( expect.arrayContaining([ expect.objectContaining({ kind: "action", actionType: "slack_session_sync", status: "processed", replayable: false, }), ]), ); await db .update(chatEndpoints) .set({ status: "active", updatedAt: new Date() }) .where(eq(chatEndpoints.id, endpoint.id)); for (const [providerError, terminalStatus] of [ ["feature_disabled", "processed"], ["missing_scope", "failed"], ] as const) { permanentStatusError = providerError; await db .update(chatActions) .set({ status: "received", result: null, updatedAt: new Date() }) .where(eq(chatActions.id, action.id)); const previousCalls = statusCalls.length; await resumed.service.processPendingSlackSessionSyncs(25, action.id); await resumed.service.processPendingSlackSessionSyncs(25, action.id); expect(statusCalls).toHaveLength(previousCalls + 1); const settled = await db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, action.id)) .then((rows) => rows[0]!); expect(settled.status).toBe(terminalStatus); expect(settled.result?.retryAt).toBeUndefined(); } await competing.service.shutdown(); await resumed.service.shutdown(); }); it("coalesces one Slack run's lifecycle and final response despite an interleaved task control", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const thread = makeThread({ channelId: "C-QUIET-RUN", id: "slack:C-QUIET-RUN:4050.1", name: "quiet-run", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: "4050.1", text: "@maya produce one quiet Slack response", mentioned: true, }), trigger: "mention", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "slack", providerMessageId: "4050.1", }), }); for (const milestone of ["queued", "working"] as const) { await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${runId}:${milestone}:${endpoint.id}`, payload: { text: milestone === "queued" ? "Maya is queued." : "Maya is working…", progressState: milestone, }, state: "pending", }); await service.processPendingPublications(); } const principal = await db .select({ id: chatExternalPrincipals.id }) .from(chatExternalPrincipals) .where(eq(chatExternalPrincipals.companyId, fixture.companyId)) .then((rows) => rows[0]); if (!principal) throw new Error("Slack principal was not created"); const stageControlPublication = async (text: string) => { const [publication] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `control:status:${randomUUID()}`, payload: { classification: "external", source: "task_control", text, }, state: "pending", }) .returning({ id: chatPublications.id }); await db.insert(chatActions).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, principalId: principal.id, kind: "task_control_authorization", providerActionId: `task-control-authorization:${publication.id}`, payload: { publicationId: publication.id }, status: "issued", }); }; await stageControlPublication("Status sampled while the run is active"); await service.processPendingPublications(); const finalText = `Final Slack result ${"with enough safe detail. ".repeat(20)}`.trim(); const finalComment = await addSelectedChatFinal({ agentId: fixture.assignedAgentId, body: finalText, companyId: fixture.companyId, issueId: conversation.issueId, runId, }); await service.processPendingPublications(); const providerRuntime = runtime.endpoints.get(endpoint.id); expect(providerRuntime?.posts).toEqual([ { threadId: thread.thread.id, text: "Maya is queued." }, ]); expect(providerRuntime?.edits).toEqual([ { threadId: thread.thread.id, messageId: "outbound-1", text: "Maya is working…", }, { threadId: thread.thread.id, messageId: "outbound-1", text: expect.stringMatching(/ — /), }, { threadId: thread.thread.id, messageId: "outbound-1", text: finalText, }, ]); const renderedSlackMessages = new Map( providerRuntime!.posts.map((post, index) => [ `outbound-${index + 1}`, post.text, ]), ); for (const edit of providerRuntime!.edits) renderedSlackMessages.set(edit.messageId, edit.text); expect([...renderedSlackMessages.values()]).toEqual([finalText]); const publications = await db .select() .from(chatPublications) .where(eq(chatPublications.conversationId, conversation.id)); expect(publications).toHaveLength(4); expect( publications .filter((publication) => publication.idempotencyKey.startsWith(`run:${runId}:`), ) .every( (publication) => publication.state === "published" && publication.providerMessageId === "outbound-1", ), ).toBe(true); expect( await db .select() .from(chatMessageLinks) .where( and( eq(chatMessageLinks.conversationId, conversation.id), eq(chatMessageLinks.direction, "outbound"), ), ), ).toEqual([ expect.objectContaining({ providerMessageId: "outbound-1", commentId: finalComment.id, }), ]); // Once the run lane is terminal, /status has no replaceable placeholder // and falls back to a distinct provider post. await stageControlPublication("Status sampled after the run finished"); await service.processPendingPublications(); expect(providerRuntime?.posts).toEqual([ { threadId: thread.thread.id, text: "Maya is queued." }, { threadId: thread.thread.id, text: expect.stringMatching(/ — /) }, ]); }); it("edits a Slack working reply into one terminal failure reply", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const thread = makeThread({ channelId: "C-FAILED-RUN", id: "slack:C-FAILED-RUN:4060.1", name: "failed-run", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: "4060.1", text: "@maya exercise a failed turn", mentioned: true, }), trigger: "mention", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", contextSnapshot: { issueId: conversation.issueId }, }); for (const milestone of ["working", "failed"] as const) { if (milestone === "failed") { await db .update(heartbeatRuns) .set({ status: "failed", updatedAt: new Date() }) .where(eq(heartbeatRuns.id, runId)); } await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${runId}:${milestone}:${endpoint.id}`, payload: { text: milestone === "working" ? "Maya is working…" : "Maya stopped before completing this turn.", progressState: milestone, }, state: "pending", }); await service.processPendingPublications(); } const providerRuntime = runtime.endpoints.get(endpoint.id); expect(providerRuntime?.posts).toEqual([ { threadId: thread.thread.id, text: "Maya is working…" }, ]); expect(providerRuntime?.edits).toEqual([ { threadId: thread.thread.id, messageId: "outbound-1", text: "Maya stopped before completing this turn.", }, ]); }); it("publishes a Telegram failure for an exact confirmation continuation that stops before commenting", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture); const chatId = "77112235"; const dm = makeThread({ channelId: `telegram:${chatId}`, id: `telegram:${chatId}`, isDM: true, name: "Telegram failed continuation", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: `${chatId}:61`, text: "Ask before the risky operation", userId: chatId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, chatId); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); if (!conversation) throw new Error("Expected Telegram continuation conversation"); const [inboundLink] = await db .select({ commentId: chatMessageLinks.commentId }) .from(chatMessageLinks) .where( and( eq(chatMessageLinks.endpointId, endpoint.id), eq(chatMessageLinks.conversationId, conversation.id), eq(chatMessageLinks.providerMessageId, `${chatId}:61`), eq(chatMessageLinks.direction, "inbound"), ), ); if (!inboundLink?.commentId) throw new Error("Expected Telegram inbound comment binding"); const [sourceRun] = await db .insert(heartbeatRuns) .values({ companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: { issueId: conversation.issueId, source: "chat:telegram", wakeCommentId: inboundLink.commentId, wakeCommentIds: [inboundLink.commentId], }, }) .returning(); if (!sourceRun) throw new Error("Expected Telegram source run"); const interaction = await issueThreadInteractionService(db).create( { id: conversation.issueId, companyId: fixture.companyId }, { kind: "request_confirmation", continuationPolicy: "wake_assignee", sourceRunId: sourceRun.id, payload: { version: 1, prompt: "Proceed with the risky operation?", }, }, { agentId: fixture.assignedAgentId, runId: sourceRun.id }, ); await service.processPendingPublications(); await db .update(issueThreadInteractions) .set({ status: "accepted", resolvedAt: new Date(), resolvedByUserId: "owner-user", result: { version: 1, outcome: "accepted" }, }) .where(eq(issueThreadInteractions.id, interaction.id)); const [wakeRequest] = await db .insert(agentWakeupRequests) .values({ companyId: fixture.companyId, agentId: fixture.assignedAgentId, source: "automation", triggerDetail: "system", reason: "issue_commented", status: "failed", idempotencyKey: `interaction:${interaction.id}:accepted`, }) .returning(); if (!wakeRequest) throw new Error("Expected Telegram continuation wake"); const [continuationRun] = await db .insert(heartbeatRuns) .values({ companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "failed", wakeupRequestId: wakeRequest.id, errorCode: "adapter_failed", contextSnapshot: { issueId: conversation.issueId, taskId: conversation.issueId, interactionId: interaction.id, interactionKind: interaction.kind, interactionStatus: "accepted", sourceRunId: sourceRun.id, source: "external_chat.interaction.resolve", }, }) .returning(); if (!continuationRun) throw new Error("Expected failed Telegram continuation run"); await db .update(agentWakeupRequests) .set({ runId: continuationRun.id }) .where(eq(agentWakeupRequests.id, wakeRequest.id)); // The milestone sweep is global; assert this continuation's exact receipt // below rather than counting work staged for other fixture companies. await enqueueChatRunMilestones(db); await service.processPendingPublications(); await expect( db .select({ conversationId: chatPublications.conversationId, endpointId: chatPublications.endpointId, payload: chatPublications.payload, state: chatPublications.state, }) .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `run:${continuationRun.id}:failed:${endpoint.id}`, ), ), ).resolves.toEqual([ { conversationId: conversation.id, endpointId: endpoint.id, payload: expect.objectContaining({ progressState: "failed", text: expect.stringContaining( "Maya stopped before completing this turn.", ), }), state: "published", }, ]); expect(runtime.endpoints.get(endpoint.id)?.posts).toEqual( expect.arrayContaining([ { threadId: dm.thread.id, text: expect.stringContaining( "Maya stopped before completing this turn.", ), }, ]), ); const forgedRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: forgedRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "failed", contextSnapshot: { issueId: conversation.issueId, interactionId: interaction.id, interactionStatus: "accepted", sourceRunId: sourceRun.id, source: "external_chat.interaction.resolve", }, updatedAt: new Date(Date.now() + 1_000), }); await db.insert(agentWakeupRequests).values({ companyId: fixture.companyId, agentId: fixture.replacementAgentId, source: "automation", triggerDetail: "system", reason: "issue_commented", status: "failed", idempotencyKey: `interaction:${interaction.id}:accepted`, runId: forgedRunId, }); await expect(enqueueChatRunMilestones(db)).resolves.toBe(0); await expect( db .select({ id: chatPublications.id }) .from(chatPublications) .where(like(chatPublications.idempotencyKey, `run:${forgedRunId}:%`)), ).resolves.toHaveLength(0); const cancelledQuestion = await issueThreadInteractionService(db).create( { id: conversation.issueId, companyId: fixture.companyId }, { kind: "ask_user_questions", continuationPolicy: "wake_assignee", sourceRunId: sourceRun.id, title: "Cancellation path", payload: { version: 1, questions: [ { id: "continue", prompt: "Should this work continue?", selectionMode: "single", required: true, allowOther: false, options: [ { id: "yes", label: "Yes" }, { id: "no", label: "No" }, ], }, ], }, }, { agentId: fixture.assignedAgentId, runId: sourceRun.id }, ); await service.processPendingPublications(); await db .update(issueThreadInteractions) .set({ status: "cancelled", resolvedAt: new Date(), resolvedByUserId: "owner-user", result: { version: 1, outcome: "cancelled", cancellationReason: "The external requester cancelled the question.", }, }) .where(eq(issueThreadInteractions.id, cancelledQuestion.id)); const [cancelledWake] = await db .insert(agentWakeupRequests) .values({ companyId: fixture.companyId, agentId: fixture.assignedAgentId, source: "automation", triggerDetail: "system", reason: "issue_commented", status: "failed", idempotencyKey: `interaction:${cancelledQuestion.id}:cancelled`, }) .returning(); if (!cancelledWake) throw new Error("Expected cancelled-question continuation wake"); const [cancelledRun] = await db .insert(heartbeatRuns) .values({ companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "failed", wakeupRequestId: cancelledWake.id, errorCode: "adapter_failed", contextSnapshot: { issueId: conversation.issueId, taskId: conversation.issueId, interactionId: cancelledQuestion.id, interactionKind: cancelledQuestion.kind, interactionStatus: "cancelled", sourceRunId: sourceRun.id, source: "issue.interaction.cancel", }, }) .returning(); if (!cancelledRun) throw new Error("Expected cancelled-question continuation run"); await db .update(agentWakeupRequests) .set({ runId: cancelledRun.id }) .where(eq(agentWakeupRequests.id, cancelledWake.id)); await expect(enqueueChatRunMilestones(db)).resolves.toBe(1); await service.processPendingPublications(); await expect( db .select({ conversationId: chatPublications.conversationId, endpointId: chatPublications.endpointId, payload: chatPublications.payload, state: chatPublications.state, }) .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `run:${cancelledRun.id}:failed:${endpoint.id}`, ), ), ).resolves.toEqual([ { conversationId: conversation.id, endpointId: endpoint.id, payload: expect.objectContaining({ progressState: "failed", text: expect.stringContaining( "Maya stopped before completing this turn.", ), }), state: "published", }, ]); }); it("drains bounded Slack chat-origin milestones without admitting an internal issue run", async () => { const fixture = await seedCompany(); const { callbacks, endpoint } = await configuredSlackEndpoint(fixture); const thread = makeThread({ channelId: "C-MILESTONE-DRAIN", id: "slack:C-MILESTONE-DRAIN:4065.1", name: "milestone-drain", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: "4065.1", text: "@maya exercise bounded milestone draining", mentioned: true, }), trigger: "mention", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const [inboundLink] = await db .select({ commentId: chatMessageLinks.commentId }) .from(chatMessageLinks) .where( and( eq(chatMessageLinks.endpointId, endpoint.id), eq(chatMessageLinks.providerMessageId, "4065.1"), eq(chatMessageLinks.direction, "inbound"), ), ); if (!inboundLink?.commentId) { throw new Error("Expected the inbound Slack comment link"); } const runCases = [ { id: randomUUID(), status: "queued", milestone: "queued" }, { id: randomUUID(), status: "running", milestone: "working" }, { id: randomUUID(), status: "failed", milestone: "failed", errorCode: "low_trust_isolation_unavailable", }, ] as const; const resolverRejectedRuns = Array.from({ length: 30 }, (_, index) => ({ id: randomUUID(), companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running" as const, contextSnapshot: { issueId: conversation.issueId, source: "chat:slack", // The broad SQL candidate fence sees the verified inbound comment, // while the authoritative lineage resolver correctly rejects this // mismatched endpoint. More than one page proves rejected rows cannot // permanently starve the valid milestone that follows them. endpointId: randomUUID(), wakeCommentId: inboundLink.commentId, wakeCommentIds: [inboundLink.commentId], }, updatedAt: new Date( new Date("2026-09-05T14:59:00.000Z").getTime() + index, ), })); const internalRunId = randomUUID(); await db.insert(heartbeatRuns).values([ ...resolverRejectedRuns, { id: internalRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", contextSnapshot: { issueId: conversation.issueId, source: "issue.comment", wakeCommentId: inboundLink.commentId, wakeCommentIds: [inboundLink.commentId], }, updatedAt: new Date("2026-09-05T14:59:59.000Z"), }, ...runCases.map(({ id, status, ...runCase }, index) => ({ id, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status, errorCode: "errorCode" in runCase ? runCase.errorCode : null, contextSnapshot: { issueId: conversation.issueId, source: "chat:slack", wakeCommentId: inboundLink.commentId, wakeCommentIds: [inboundLink.commentId], }, updatedAt: new Date(`2026-09-05T15:00:0${index}.000Z`), })), ]); const inserted: number[] = []; for (let index = 0; index < 4; index += 1) { inserted.push( await enqueueChatRunMilestones(db, { since: new Date("2026-09-05T14:00:00.000Z"), limit: 1, }), ); } expect(inserted).toEqual([1, 1, 1, 0]); const publications = await db .select() .from(chatPublications) .where(eq(chatPublications.conversationId, conversation.id)); expect(publications).toHaveLength(3); expect( new Set(publications.map((publication) => publication.idempotencyKey)), ).toEqual( new Set( runCases.map( ({ id, milestone }) => `run:${id}:${milestone}:${endpoint.id}`, ), ), ); expect( publications.find((publication) => publication.idempotencyKey.startsWith(`run:${internalRunId}:`), ), ).toBeUndefined(); expect( publications.find((publication) => publication.idempotencyKey.includes(":failed:"), )?.payload, ).toMatchObject({ progressState: "failed", text: expect.stringContaining( "Ask a Paperclip admin to create a private identity link for this account or enable isolated guest execution, then start a new task.", ), }); }); async function publicationLaneFixture(count: number) { const fixture = await seedCompany(); const storage = createStorageService(); const runtime = new FakeChatSdkRuntime(); const providerFetch = (input: string | URL | Request, init?: RequestInit) => fakeSlackFetch( `U-LANE-${new Headers(init?.headers).get("authorization")}`, )(input, init); const { service } = createService(runtime, providerFetch as typeof fetch, { storage: storage.storage, }); const lanes = []; for (let index = 0; index < count; index += 1) { const endpoint = await service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId, }, "owner-user", ); await service.configure( endpoint.id, { action: "configure", credentials: { botToken: `xoxb-lane-${endpoint.id}`, signingSecret: `signing-${endpoint.id}`, }, }, "owner-user", ); await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, endpoint.id)); const channelId = `C-LANE-${endpoint.id}`; await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: channelId, label: "publication-lane", availability: "available", enabled: true, }); const callbacks = runtime.configurations.get(endpoint.id)?.callbacks; if (!callbacks) throw new Error("Expected publication lane runtime"); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: makeThread({ channelId, id: `slack:${channelId}:5000.1`, name: "publication-lane", }).thread, message: makeMessage({ id: "5000.1", text: "@maya publication lane", mentioned: true, }), trigger: "mention", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!conversation || !providerRuntime) throw new Error("Expected publication lane conversation"); lanes.push({ endpoint, conversation, providerRuntime }); } const enqueue = async ( index: number, text: string, offset = index, payload?: typeof chatPublications.$inferInsert.payload, ) => { const { endpoint, conversation } = lanes[index]!; const [publication] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `lane:${randomUUID()}`, payload: payload ?? { text }, state: "pending", createdAt: new Date(Date.UTC(2000, 0, 1) + offset), }) .returning(); return publication!; }; const cleanup = async () => { await service.shutdown(); await db .update(chatEndpoints) .set({ status: "paused" }) .where( inArray( chatEndpoints.id, lanes.map(({ endpoint }) => endpoint.id), ), ); }; return { ...fixture, storage, runtime, service, lanes, enqueue, cleanup }; } it("refills four tracked publication endpoint lanes after a scheduled budget and joins shutdown", async () => { const fixture = await publicationLaneFixture(6); const { service, lanes, enqueue, storage } = fixture; const releases: Array<() => void> = []; const entered = new Set(); let active = 0; let maximumActive = 0; const gates = lanes.map((lane, index) => { const gate = new Promise((resolve) => { releases[index] = resolve; }); lane.providerRuntime.postHook = async () => { entered.add(index); active += 1; maximumActive = Math.max(maximumActive, active); try { await gate; } finally { active -= 1; } }; return gate; }); let shutdown: Promise | undefined; try { const comment = await issueService(db).addComment( lanes[0]!.conversation.issueId, "Slow upload", { userId: "owner-user" }, ); const stored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${lanes[0]!.conversation.issueId}`, originalFilename: "slow.txt", contentType: "text/plain", body: Buffer.from("held file"), }); const attachment = await issueService(db).createAttachment({ issueId: lanes[0]!.conversation.issueId, issueCommentId: comment.id, ...stored, createdByUserId: "owner-user", }); const first = await enqueue(0, "", 0, { text: "", attachmentIds: [attachment.id], }); await db .update(chatPublications) .set({ commentId: comment.id }) .where(eq(chatPublications.id, first.id)); const tail = await enqueue(0, "Same conversation must wait", 100); const initial = [first]; for (let index = 1; index < lanes.length; index += 1) initial.push(await enqueue(index, `Final ${index}`)); let scheduledReturned = false; const scheduled = service.schedulePendingPublications(4).then(() => { scheduledReturned = true; }); await vi.waitFor(() => { expect(scheduledReturned).toBe(true); expect(entered).toEqual(new Set([0, 1, 2, 3])); }); await scheduled; expect(maximumActive).toBe(4); await service.schedulePendingPublications(4); expect(entered.size).toBe(4); expect( await db .select({ state: chatPublications.state, attempts: chatPublications.attempts, }) .from(chatPublications) .where(eq(chatPublications.id, tail.id)), ).toEqual([{ state: "pending", attempts: 0 }]); releases[1]!(); await vi.waitFor(async () => { expect( await db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.id, initial[1]!.id)), ).toEqual([{ state: "published" }]); }); const sourceRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: sourceRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: { issueId: lanes[1]!.conversation.issueId, source: "automation", }, }); const interaction = await issueThreadInteractionService(db).create( { id: lanes[1]!.conversation.issueId, companyId: fixture.companyId }, { kind: "ask_user_questions", continuationPolicy: "wake_assignee", sourceRunId, title: "Late question", payload: { version: 1, title: "Late question", questions: [ { id: "priority", prompt: "Which priority?", selectionMode: "single", required: true, options: [ { id: "high", label: "High" }, { id: "normal", label: "Normal" }, ], }, ], }, }, { agentId: fixture.assignedAgentId, runId: sourceRunId }, ); releases[4]!(); releases[5]!(); await vi.waitFor( async () => { await service.schedulePendingPublications(4); expect( await db .select({ state: chatPublications.state }) .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `interaction:${interaction.id}:${lanes[1]!.endpoint.id}`, ), ), ).toEqual([{ state: "published" }]); }, { timeout: 5_000 }, ); expect(entered).toEqual(new Set([0, 1, 2, 3, 4, 5])); expect(maximumActive).toBe(4); expect(lanes[0]!.providerRuntime.posts).toHaveLength(0); expect(lanes[1]!.providerRuntime.posts).toHaveLength(2); expect(lanes[1]!.providerRuntime.posts[1]!.text).toContain( "Late question", ); let shutdownDone = false; shutdown = service.shutdown().then(() => { shutdownDone = true; }); await new Promise((resolve) => setImmediate(resolve)); expect(shutdownDone).toBe(false); expect(lanes[0]!.providerRuntime.shutdown).not.toHaveBeenCalled(); expect(await service.schedulePendingPublications()).toBe(0); releases.forEach((release) => release()); await shutdown; expect(lanes[0]!.providerRuntime.posts).toHaveLength(1); expect(lanes[0]!.providerRuntime.posts[0]!.files).toHaveLength(1); expect( await db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.id, tail.id)), ).toEqual([{ state: "pending" }]); expect( await db .select() .from(chatEndpointLeases) .where( and( eq(chatEndpointLeases.companyId, fixture.companyId), like(chatEndpointLeases.leaseKey, "publication:%"), ), ), ).toHaveLength(0); } finally { releases.forEach((release) => release()); await Promise.all(gates); await shutdown; await fixture.cleanup(); } }); it("does not hide another endpoint behind a page of busy-bot conversation heads", async () => { const fixture = await publicationLaneFixture(2); const { service, lanes, enqueue } = fixture; let release!: () => void; const held = new Promise((resolve) => { release = resolve; }); let entered = false; lanes[0]!.providerRuntime.postHook = async () => { entered = true; await held; }; let drain: Promise | undefined; try { const first = await enqueue(0, "First in conversation"); const tail = await enqueue(0, "Second in conversation", 100); for (let index = 0; index < 26; index += 1) { const original = lanes[0]!.conversation; const [conversation] = await db .insert(chatConversations) .values({ companyId: fixture.companyId, endpointId: original.endpointId, resourceId: original.resourceId, issueId: original.issueId, externalConversationId: original.externalConversationId, externalThreadId: `slack:${original.externalConversationId}:${6000 + index}.1`, externalLabel: `Backlog ${index}`, }) .returning(); await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: original.endpointId, conversationId: conversation!.id, issueId: original.issueId, idempotencyKey: `lane-backlog:${randomUUID()}`, payload: { text: `Backlog ${index}` }, createdAt: new Date(Date.UTC(2000, 0, 1) + index + 1), }); } const ready = await enqueue(1, "Unrelated final", 200); await service.schedulePendingPublications(25); await vi.waitFor(async () => { expect(entered).toBe(true); expect( await db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.id, ready.id)), ).toEqual([{ state: "published" }]); }); expect( await db .select({ state: chatPublications.state, attempts: chatPublications.attempts, }) .from(chatPublications) .where(eq(chatPublications.id, tail.id)), ).toEqual([{ state: "pending", attempts: 0 }]); release(); drain = service.processPendingPublications(50); await drain; const sameConversationPosts = lanes[0]!.providerRuntime.posts.filter( (post) => post.threadId === lanes[0]!.conversation.externalThreadId, ); expect(sameConversationPosts.map((post) => post.text)).toEqual([ "First in conversation", "Second in conversation", ]); expect( await db .select({ state: chatPublications.state, attempts: chatPublications.attempts, }) .from(chatPublications) .where(inArray(chatPublications.id, [first.id, tail.id])), ).toEqual([ { state: "published", attempts: 1 }, { state: "published", attempts: 1 }, ]); } finally { release(); await drain; await fixture.cleanup(); } }); it("keeps an aged live publication owned across standby sweeps but quarantines only exact-lease orphans", async () => { const fixture = await publicationLaneFixture(2); const { service, lanes, enqueue } = fixture; const standby = createService(); let release!: () => void; const held = new Promise((resolve) => { release = resolve; }); let entered = false; lanes[0]!.providerRuntime.postHook = async () => { entered = true; await held; }; vi.useFakeTimers({ toFake: ["Date", "setInterval", "clearInterval"] }); try { const live = await enqueue(0, "Long running provider send"); await service.schedulePendingPublications(1); await vi.waitFor(() => expect(entered).toBe(true)); const staleAt = new Date(Date.now() - 61_000); await db .update(chatPublications) .set({ updatedAt: staleAt }) .where(eq(chatPublications.id, live.id)); const [lease] = await db .select() .from(chatEndpointLeases) .where(eq(chatEndpointLeases.leaseKey, `publication:${live.id}:1`)); expect(lease).toMatchObject({ companyId: fixture.companyId, endpointId: live.endpointId, }); expect(lease!.expiresAt.getTime()).toBeGreaterThan(Date.now()); // Advance the renewal clock, not wall time. Both the first claim and // its stale updatedAt now exceed 60s, while exact ownership renews. await vi.advanceTimersByTimeAsync(31_000); await vi.waitFor(async () => { const [renewed] = await db .select() .from(chatEndpointLeases) .where(eq(chatEndpointLeases.id, lease!.id)); expect(renewed!.expiresAt.getTime()).toBeGreaterThan( lease!.expiresAt.getTime(), ); }); await vi.advanceTimersByTimeAsync(31_000); await vi.waitFor(async () => { const [renewed] = await db .select() .from(chatEndpointLeases) .where(eq(chatEndpointLeases.id, lease!.id)); expect(renewed!.expiresAt.getTime()).toBeGreaterThan( Date.now() + 60_000, ); }); const orphans = []; for (const kind of [ "expired", "wrong-attempt", "wrong-endpoint", ] as const) { const orphan = await enqueue(1, kind); await db .update(chatPublications) .set({ state: "streaming", attempts: 7, updatedAt: staleAt }) .where(eq(chatPublications.id, orphan.id)); await db.insert(chatEndpointLeases).values({ companyId: fixture.companyId, endpointId: kind === "wrong-endpoint" ? lanes[0]!.endpoint.id : orphan.endpointId, leaseKey: `publication:${orphan.id}:${kind === "wrong-attempt" ? 6 : 7}`, token: randomUUID(), expiresAt: new Date( Date.now() + (kind === "expired" ? -1_000 : 90_000), ), }); orphans.push(orphan); } await standby.service.schedulePendingPublications(0); expect( await db .select({ state: chatPublications.state, attempts: chatPublications.attempts, }) .from(chatPublications) .where(eq(chatPublications.id, live.id)), ).toEqual([{ state: "streaming", attempts: 1 }]); expect( await db .select({ state: chatPublications.state, attempts: chatPublications.attempts, }) .from(chatPublications) .where( inArray( chatPublications.id, orphans.map((row) => row.id), ), ), ).toEqual( orphans.map(() => ({ state: "delivery_unknown", attempts: 7 })), ); release(); await service.shutdown(); expect( await db .select({ state: chatPublications.state, attempts: chatPublications.attempts, }) .from(chatPublications) .where(eq(chatPublications.id, live.id)), ).toEqual([{ state: "published", attempts: 1 }]); expect(lanes[0]!.providerRuntime.posts).toHaveLength(1); expect(vi.getTimerCount()).toBe(0); } finally { release(); await standby.service.shutdown(); await fixture.cleanup(); vi.useRealTimers(); } }); it("does not retry a provider-accepted publication after its exact attempt lease is lost", async () => { const fixture = await publicationLaneFixture(1); const { service, lanes, enqueue } = fixture; let release!: () => void; const held = new Promise((resolve) => { release = resolve; }); let entered = false; lanes[0]!.providerRuntime.postHook = async () => { entered = true; await held; }; try { const publication = await enqueue(0, "Accepted without lease authority"); await service.schedulePendingPublications(1); await vi.waitFor(() => expect(entered).toBe(true)); await db .update(chatEndpointLeases) .set({ expiresAt: new Date(Date.now() - 1_000) }) .where( eq(chatEndpointLeases.leaseKey, `publication:${publication.id}:1`), ); release(); await service.processPendingPublications(); expect(lanes[0]!.providerRuntime.posts).toHaveLength(1); expect( await db .select({ state: chatPublications.state, attempts: chatPublications.attempts, providerMessageId: chatPublications.providerMessageId, }) .from(chatPublications) .where(eq(chatPublications.id, publication.id)), ).toEqual([{ state: "streaming", attempts: 1, providerMessageId: null }]); await db .update(chatPublications) .set({ updatedAt: new Date(Date.now() - 61_000) }) .where(eq(chatPublications.id, publication.id)); await service.processPendingPublications(); await service.processPendingPublications(); expect(lanes[0]!.providerRuntime.posts).toHaveLength(1); expect( await db .select({ state: chatPublications.state, attempts: chatPublications.attempts, }) .from(chatPublications) .where(eq(chatPublications.id, publication.id)), ).toEqual([{ state: "delivery_unknown", attempts: 1 }]); expect( await db .select() .from(chatMessageLinks) .where(eq(chatMessageLinks.publicationId, publication.id)), ).toHaveLength(0); } finally { release(); await fixture.cleanup(); } }); it("does not resurrect publication ownership after waiting beyond expiry on a real lease row lock", async () => { const fixture = await publicationLaneFixture(1); const { service, lanes, enqueue } = fixture; let release!: () => void; const held = new Promise((resolve) => { release = resolve; }); let entered = false; lanes[0]!.providerRuntime.postHook = async () => { entered = true; await held; }; let drain: Promise | undefined; vi.useFakeTimers({ toFake: ["Date"] }); try { const baseTime = Date.now(); const publication = await enqueue( 0, "Accepted while receipt ownership waits", ); await service.schedulePendingPublications(1); await vi.waitFor(() => expect(entered).toBe(true)); const leaseKey = `publication:${publication.id}:1`; await db .update(chatEndpointLeases) .set({ expiresAt: new Date(baseTime + 10_000) }) .where(eq(chatEndpointLeases.leaseKey, leaseKey)); await db.transaction(async (tx) => { await tx .select() .from(chatEndpointLeases) .where(eq(chatEndpointLeases.leaseKey, leaseKey)) .for("update"); const [backend] = (await tx.execute( sql`select pg_backend_pid() as pid`, )) as unknown as Array<{ pid: number }>; release(); drain = service.processPendingPublications(); // Observe the real blocker; advancing a clock before the ownership // query starts would not distinguish a stale pre-lock decision clock. await vi.waitFor(async () => { const [state] = (await db.execute(sql`select exists ( select 1 from pg_stat_activity where ${backend!.pid} = any(pg_blocking_pids(pid)) ) as waiting`)) as unknown as Array<{ waiting: boolean }>; expect(state!.waiting).toBe(true); }); expect(lanes[0]!.providerRuntime.posts).toHaveLength(1); vi.setSystemTime(new Date(baseTime + 20_000)); }); await drain; expect( await db .select({ state: chatPublications.state, providerMessageId: chatPublications.providerMessageId, }) .from(chatPublications) .where(eq(chatPublications.id, publication.id)), ).toEqual([{ state: "streaming", providerMessageId: null }]); expect( await db .select() .from(chatMessageLinks) .where(eq(chatMessageLinks.publicationId, publication.id)), ).toHaveLength(0); await db .update(chatPublications) .set({ updatedAt: new Date(Date.now() - 61_000) }) .where(eq(chatPublications.id, publication.id)); await service.processPendingPublications(); expect( await db .select({ state: chatPublications.state, attempts: chatPublications.attempts, }) .from(chatPublications) .where(eq(chatPublications.id, publication.id)), ).toEqual([{ state: "delivery_unknown", attempts: 1 }]); expect(lanes[0]!.providerRuntime.posts).toHaveLength(1); } finally { release(); await drain; await fixture.cleanup(); vi.useRealTimers(); } }); it("publishes equal-time Slack outbox rows once in stable order across concurrent drains", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const thread = makeThread({ channelId: "C-STABLE-OUTBOX", id: "slack:C-STABLE-OUTBOX:4075.1", name: "stable-outbox", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: "4075.1", text: "@maya preserve publication order", mentioned: true, }), trigger: "mention", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const createdAt = new Date("2026-09-05T15:00:00.000Z"); const firstId = "00000000-0000-4000-8000-000000000001"; const secondId = "00000000-0000-4000-8000-000000000002"; await db.insert(chatPublications).values([ { id: secondId, companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `ordering:second:${endpoint.id}`, payload: { text: "Second publication" }, state: "pending", createdAt, }, { id: firstId, companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `ordering:first:${endpoint.id}`, payload: { text: "First publication" }, state: "pending", createdAt, }, ]); await Promise.all([ service.processPendingPublications(), service.processPendingPublications(), ]); expect(runtime.endpoints.get(endpoint.id)?.posts).toEqual([ { threadId: thread.thread.id, text: "First publication" }, { threadId: thread.thread.id, text: "Second publication" }, ]); expect( await db .select({ id: chatPublications.id, state: chatPublications.state }) .from(chatPublications) .where(inArray(chatPublications.id, [firstId, secondId])) .orderBy(asc(chatPublications.id)), ).toEqual([ { id: firstId, state: "published" }, { id: secondId, state: "published" }, ]); }); it("does not let one blocked Slack conversation starve another outbox head", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const blockedThread = makeThread({ channelId: "C-OUTBOX-FAIRNESS", id: "slack:C-OUTBOX-FAIRNESS:4080.1", name: "outbox-fairness", }); const readyThread = makeThread({ channelId: "C-OUTBOX-FAIRNESS", id: "slack:C-OUTBOX-FAIRNESS:4080.2", name: "outbox-fairness", }); for (const [thread, id, text] of [ [blockedThread, "4080.1", "@maya create the blocked task"], [readyThread, "4080.2", "@maya create the ready task"], ] as const) { await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id, text, mentioned: true }), trigger: "mention", }); } const conversations = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const blockedConversation = conversations.find( (conversation) => conversation.externalThreadId === blockedThread.thread.id, ); const readyConversation = conversations.find( (conversation) => conversation.externalThreadId === readyThread.thread.id, ); if (!blockedConversation || !readyConversation) { throw new Error("Expected both Slack task conversations"); } const baseTime = new Date("2026-09-05T15:10:00.000Z"); await db.insert(chatPublications).values([ { companyId: fixture.companyId, endpointId: endpoint.id, conversationId: blockedConversation.id, issueId: blockedConversation.issueId, idempotencyKey: `fairness:unknown:${endpoint.id}`, payload: { text: "Ambiguous predecessor" }, state: "delivery_unknown", createdAt: baseTime, }, ...Array.from({ length: 30 }, (_, index) => ({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: blockedConversation.id, issueId: blockedConversation.issueId, idempotencyKey: `fairness:blocked:${index}:${endpoint.id}`, payload: { text: `Blocked publication ${index}` }, state: "pending", createdAt: new Date(baseTime.getTime() + index + 1), })), { companyId: fixture.companyId, endpointId: endpoint.id, conversationId: readyConversation.id, issueId: readyConversation.issueId, idempotencyKey: `fairness:ready:${endpoint.id}`, payload: { text: "Ready publication" }, state: "pending", createdAt: new Date(baseTime.getTime() + 60_000), }, ]); await service.processPendingPublications(25); expect(runtime.endpoints.get(endpoint.id)?.posts).toEqual([ { threadId: readyThread.thread.id, text: "Ready publication" }, ]); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where( eq(chatPublications.idempotencyKey, `fairness:ready:${endpoint.id}`), ), ).resolves.toEqual([{ state: "published" }]); expect( await db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.conversationId, blockedConversation.id)), ).toEqual( expect.arrayContaining([ { state: "delivery_unknown" }, ...Array.from({ length: 30 }, () => ({ state: "pending" })), ]), ); }); it("skips paused outbox heads and resumes promptly without bypassing provider backoff", async () => { const fixture = await seedCompany(); const blockedFetch = fakeSlackFetch("U-BLOCKED-OUTBOX"); const readyFetch = fakeSlackFetch("U-READY-OUTBOX"); let includeResumeChannel = false; const providerFetch = ( input: string | URL | Request, init?: RequestInit, ) => { const authorization = new Headers(init?.headers).get("authorization"); if ( includeResumeChannel && String(input).startsWith("https://slack.com/api/conversations.list") && authorization === "Bearer xoxb-blocked-outbox" ) { return Promise.resolve( Response.json({ ok: true, channels: [ { id: "C-PAUSED-ENDPOINT", name: "paused-endpoint", is_member: true, is_archived: false, }, ], response_metadata: { next_cursor: "" }, }), ); } return authorization === "Bearer xoxb-blocked-outbox" ? blockedFetch(input, init) : readyFetch(input, init); }; const runtime = new FakeChatSdkRuntime(); const { service } = createService( runtime, providerFetch as typeof globalThis.fetch, ); const blockedEndpoint = await service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); const readyEndpoint = await service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.replacementAgentId }, "owner-user", ); for (const [endpoint, botToken] of [ [blockedEndpoint, "xoxb-blocked-outbox"], [readyEndpoint, "xoxb-ready-outbox"], ] as const) { await service.configure( endpoint.id, { action: "configure", credentials: { botToken, signingSecret: `signing-${botToken}`, }, }, "owner-user", ); await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, endpoint.id)); } const blockedCallbacks = runtime.configurations.get( blockedEndpoint.id, )?.callbacks; const readyCallbacks = runtime.configurations.get( readyEndpoint.id, )?.callbacks; if (!blockedCallbacks || !readyCallbacks) { throw new Error("Expected both Slack runtimes"); } await db.insert(chatEndpointResources).values([ { companyId: fixture.companyId, endpointId: blockedEndpoint.id, type: "channel", providerResourceId: "C-PAUSED-ENDPOINT", label: "paused-endpoint", availability: "available", enabled: true, }, { companyId: fixture.companyId, endpointId: readyEndpoint.id, type: "channel", providerResourceId: "C-READY-ENDPOINT", label: "ready-endpoint", availability: "available", enabled: true, }, ]); const blockedThread = makeThread({ channelId: "C-PAUSED-ENDPOINT", id: "slack:C-PAUSED-ENDPOINT:4081.1", name: "paused-endpoint", }); const readyThread = makeThread({ channelId: "C-READY-ENDPOINT", id: "slack:C-READY-ENDPOINT:4081.2", name: "ready-endpoint", }); await deliverMessage({ callbacks: blockedCallbacks, endpointId: blockedEndpoint.id, message: makeMessage({ id: "4081.1", mentioned: true, text: "@maya blocked endpoint", }), thread: blockedThread.thread, trigger: "mention", }); await deliverMessage({ callbacks: readyCallbacks, endpointId: readyEndpoint.id, message: makeMessage({ id: "4081.2", mentioned: true, text: "@linus healthy endpoint", }), thread: readyThread.thread, trigger: "mention", }); const conversations = await db .select() .from(chatConversations) .where( inArray(chatConversations.endpointId, [ blockedEndpoint.id, readyEndpoint.id, ]), ); const blockedConversation = conversations.find( (conversation) => conversation.endpointId === blockedEndpoint.id, ); const readyConversation = conversations.find( (conversation) => conversation.endpointId === readyEndpoint.id, ); if (!blockedConversation || !readyConversation) { throw new Error("Expected both endpoint conversations"); } const [blockedSetup] = await db .select({ setup: chatEndpoints.setup }) .from(chatEndpoints) .where(eq(chatEndpoints.id, blockedEndpoint.id)); await db .update(chatEndpoints) .set({ setup: { ...blockedSetup.setup, step: "complete" } }) .where(eq(chatEndpoints.id, blockedEndpoint.id)); await service.configure( blockedEndpoint.id, { action: "pause" }, "owner-user", ); const createdAt = new Date("2026-09-05T15:11:00.000Z"); const providerRetryAt = new Date(Date.now() + 60_000); await db.insert(chatPublications).values([ { companyId: fixture.companyId, endpointId: blockedEndpoint.id, conversationId: blockedConversation.id, issueId: blockedConversation.issueId, idempotencyKey: `paused-endpoint:${blockedEndpoint.id}`, payload: { text: "Blocked by paused endpoint" }, state: "pending", createdAt, }, { companyId: fixture.companyId, endpointId: readyEndpoint.id, conversationId: readyConversation.id, issueId: readyConversation.issueId, idempotencyKey: `ready-endpoint:${readyEndpoint.id}`, payload: { text: "Healthy endpoint publication" }, state: "pending", createdAt: new Date(createdAt.getTime() + 1), }, { companyId: fixture.companyId, endpointId: blockedEndpoint.id, conversationId: blockedConversation.id, issueId: blockedConversation.issueId, idempotencyKey: `provider-backoff:${blockedEndpoint.id}`, payload: { text: "Provider rate limit still applies" }, state: "retry", attempts: 2, nextAttemptAt: providerRetryAt, createdAt: new Date(createdAt.getTime() + 2), }, ]); // Even the smallest global page must skip paused work without consuming // its attempt budget or imposing an artificial resume deadline. try { await service.processPendingPublications(1); await expect( db .select({ idempotencyKey: chatPublications.idempotencyKey, nextAttemptAt: chatPublications.nextAttemptAt, state: chatPublications.state, }) .from(chatPublications) .where( inArray(chatPublications.idempotencyKey, [ `paused-endpoint:${blockedEndpoint.id}`, `ready-endpoint:${readyEndpoint.id}`, ]), ) .orderBy(asc(chatPublications.createdAt)), ).resolves.toEqual([ { idempotencyKey: `paused-endpoint:${blockedEndpoint.id}`, nextAttemptAt: null, state: "pending", }, { idempotencyKey: `ready-endpoint:${readyEndpoint.id}`, nextAttemptAt: null, state: "published", }, ]); expect(runtime.endpoints.get(readyEndpoint.id)?.posts).toEqual([ { text: "Healthy endpoint publication", threadId: readyThread.thread.id, }, ]); includeResumeChannel = true; await service.configure( blockedEndpoint.id, { action: "resume" }, "owner-user", ); await service.processPendingPublications(1); expect(runtime.endpoints.get(blockedEndpoint.id)?.posts).toEqual([ { text: "Blocked by paused endpoint", threadId: blockedThread.thread.id, }, ]); await expect( db .select({ state: chatPublications.state, attempts: chatPublications.attempts, nextAttemptAt: chatPublications.nextAttemptAt, }) .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `provider-backoff:${blockedEndpoint.id}`, ), ), ).resolves.toEqual([ { state: "retry", attempts: 2, nextAttemptAt: providerRetryAt }, ]); } finally { // Deliberate pending fixtures must not leak into later shared-DB tests, // even when a fairness or resume assertion fails. await db .update(chatPublications) .set({ state: "cancelled", nextAttemptAt: null }) .where( and( inArray(chatPublications.idempotencyKey, [ `paused-endpoint:${blockedEndpoint.id}`, `ready-endpoint:${readyEndpoint.id}`, `provider-backoff:${blockedEndpoint.id}`, ]), inArray(chatPublications.state, ["pending", "retry"]), ), ); await service.shutdown(); } }); it("does not let pause return while a Slack publication is still in provider transport", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const thread = makeThread({ channelId: "C-PAUSE-OUTBOX", id: "slack:C-PAUSE-OUTBOX:4090.1", name: "pause-outbox", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: "4090.1", text: "@maya create a pause-fenced task", mentioned: true, }), trigger: "mention", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const [storedEndpoint] = await db .select({ setup: chatEndpoints.setup }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)); await db .update(chatEndpoints) .set({ status: "active", setup: { ...storedEndpoint.setup, step: "complete" }, activatedAt: new Date(), }) .where(eq(chatEndpoints.id, endpoint.id)); const [publication] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `pause-fence:${endpoint.id}`, payload: { text: "Publication already entering Slack transport" }, state: "pending", }) .returning(); const endpointRuntime = runtime.endpoints.get(endpoint.id); if (!endpointRuntime) throw new Error("Expected Slack runtime"); let markPostStarted!: () => void; const postStarted = new Promise((resolve) => { markPostStarted = resolve; }); let releasePost!: () => void; const postBlocked = new Promise((resolve) => { releasePost = resolve; }); endpointRuntime.postHook = async () => { markPostStarted(); await postBlocked; }; const processing = service.processPendingPublications(); await postStarted; const pausing = service.configure( endpoint.id, { action: "pause" }, "owner-user", ); await expect( Promise.race([ pausing.then(() => "paused"), new Promise((resolve) => setTimeout(() => resolve("transport-in-flight"), 100), ), ]), ).resolves.toBe("transport-in-flight"); releasePost(); await Promise.all([processing, pausing]); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.id, publication.id)), ).resolves.toEqual([{ state: "published" }]); await expect( db .select({ status: chatEndpoints.status }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)), ).resolves.toEqual([{ status: "paused" }]); expect(endpointRuntime.posts).toContainEqual({ threadId: thread.thread.id, text: "Publication already entering Slack transport", }); await service.shutdown(); }); it("quarantines Telegram maintenance success when credential-lease ownership is reclaimed", async () => { const fixture = await seedCompany(); const botToken = "123456:telegram-lease-reclaim"; let reclaimLease = false; let markLeaseReclaimed!: () => void; const leaseReclaimed = new Promise((resolve) => { markLeaseReclaimed = resolve; }); let blockCommands = false; let acceptedCommandMutations = 0; const providerFetch = vi.fn(async (input: string | URL | Request) => { const url = String(input); if (url.endsWith("/getMe")) { return new Response( JSON.stringify({ ok: true, result: { id: 884422, username: "paperclip_lease_reclaim_bot", first_name: "Paperclip Lease Reclaim", }, }), { status: 200, headers: { "content-type": "application/json" } }, ); } if (url.endsWith("/getWebhookInfo")) { return new Response(JSON.stringify({ ok: true, result: { url: "" } }), { status: 200, headers: { "content-type": "application/json" }, }); } if (url.endsWith("/setWebhook")) { return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } if (url.endsWith("/setMyCommands")) { if (blockCommands) { reclaimLease = true; await leaseReclaimed; } acceptedCommandMutations += 1; return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } throw new Error(`Unexpected provider request: ${url}`); }) as typeof globalThis.fetch; const { service } = createService(new FakeChatSdkRuntime(), providerFetch, { credentialMutationLeaseRenewalIntervalMs: 5, renewCredentialMutationLease: async (input) => { if (!reclaimLease) return true; const reclaimed = await db .update(chatEndpointLeases) .set({ token: `reclaimed-${randomUUID()}`, expiresAt: new Date(Date.now() + 90_000), updatedAt: new Date(), }) .where( and( eq(chatEndpointLeases.endpointId, input.endpointId), eq(chatEndpointLeases.leaseKey, input.leaseKey), eq(chatEndpointLeases.token, input.token), ), ) .returning({ id: chatEndpointLeases.id }); if (reclaimed.length > 0) markLeaseReclaimed(); return false; }, }); const endpoint = await service.create( fixture.companyId, { provider: "telegram", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await service.configure( endpoint.id, { action: "configure", credentials: { botToken } }, "owner-user", ); const maintenance = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "telegram_maintenance"), ), ) .then((rows) => rows[0]!); await db .update(chatActions) .set({ status: "failed", result: { attempts: 1, providerConfirmed: false, retryable: true, retryAt: new Date(0).toISOString(), }, updatedAt: new Date(), }) .where(eq(chatActions.id, maintenance.id)); acceptedCommandMutations = 0; blockCommands = true; await service.processPendingDeliveries(); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, maintenance.id)), ).resolves.toEqual([ { status: "failed", result: expect.objectContaining({ code: "telegram_maintenance_delivery_unknown", providerConfirmed: true, retryable: true, }), }, ]); const [reclaimedLease] = await db .select({ token: chatEndpointLeases.token }) .from(chatEndpointLeases) .where( and( eq(chatEndpointLeases.endpointId, endpoint.id), eq(chatEndpointLeases.leaseKey, "credentials"), ), ); expect(reclaimedLease?.token).toMatch(/^reclaimed-/); expect(acceptedCommandMutations).toBe(1); // The reclaimed owner is synthetic and will never release its 90-second // lease. Remove only this fixture's lease and retryable maintenance row so // later global recovery sweeps do not wait for a nonexistent process. await db.transaction(async (tx) => { await tx .update(chatActions) .set({ status: "cancelled", result: { code: "test_fixture_complete" }, updatedAt: new Date(), }) .where(eq(chatActions.id, maintenance.id)); await tx .delete(chatEndpointLeases) .where( and( eq(chatEndpointLeases.endpointId, endpoint.id), eq(chatEndpointLeases.leaseKey, "credentials"), ), ); }); await service.shutdown(); }); it("does not let stale Telegram removal clear successor credentials after lease reclamation", async () => { const fixture = await seedCompany(); const botToken = "123456:telegram-removal-lease-reclaim"; let connectionId: string | null = null; let successorRefs: typeof toolConnections.$inferSelect.credentialSecretRefs = []; let reclaimLease = false; let markLeaseReclaimed!: () => void; const leaseReclaimed = new Promise((resolve) => { markLeaseReclaimed = resolve; }); let blockRemoval = false; let acceptedRemovalMutations = 0; const providerFetch = vi.fn(async (input: string | URL | Request) => { const url = String(input); if (url.endsWith("/getMe")) { return new Response( JSON.stringify({ ok: true, result: { id: 884423, username: "paperclip_removal_reclaim_bot", first_name: "Paperclip Removal Reclaim", }, }), { status: 200, headers: { "content-type": "application/json" } }, ); } if (url.endsWith("/getWebhookInfo")) { return new Response(JSON.stringify({ ok: true, result: { url: "" } }), { status: 200, headers: { "content-type": "application/json" }, }); } if (url.endsWith("/setWebhook") || url.endsWith("/setMyCommands")) { return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } if (url.endsWith("/deleteWebhook")) { acceptedRemovalMutations += 1; return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } if (url.endsWith("/deleteMyCommands")) { if (blockRemoval) { reclaimLease = true; await leaseReclaimed; } acceptedRemovalMutations += 1; return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } throw new Error(`Unexpected provider request: ${url}`); }) as typeof globalThis.fetch; const { service } = createService(new FakeChatSdkRuntime(), providerFetch, { credentialMutationLeaseRenewalIntervalMs: 5, renewCredentialMutationLease: async (input) => { if (!reclaimLease) return true; const reclaimed = await db .update(chatEndpointLeases) .set({ token: `reclaimed-${randomUUID()}`, expiresAt: new Date(Date.now() + 90_000), updatedAt: new Date(), }) .where( and( eq(chatEndpointLeases.endpointId, input.endpointId), eq(chatEndpointLeases.leaseKey, input.leaseKey), eq(chatEndpointLeases.token, input.token), ), ) .returning({ id: chatEndpointLeases.id }); if (reclaimed.length > 0) { if (!connectionId) throw new Error("Expected Telegram connection"); await db .update(toolConnections) .set({ credentialSecretRefs: successorRefs, updatedAt: new Date() }) .where(eq(toolConnections.id, connectionId)); markLeaseReclaimed(); } return false; }, }); const endpoint = await service.create( fixture.companyId, { provider: "telegram", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await service.configure( endpoint.id, { action: "configure", credentials: { botToken } }, "owner-user", ); connectionId = endpoint.connectionId; const currentRefs = await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, connectionId)) .then((rows) => rows[0]!.refs); successorRefs = currentRefs.map((ref) => ({ ...ref, secretId: randomUUID(), })); blockRemoval = true; await expect( service.configure(endpoint.id, { action: "remove" }, "owner-user"), ).rejects.toMatchObject({ code: "CHAT_CREDENTIAL_LEASE_LOST" }); await expect( db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, connectionId)), ).resolves.toEqual([{ refs: successorRefs }]); const removalAction = await db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "telegram_maintenance"), sql`${chatActions.payload}->>'operation' = 'remove_endpoint'`, ), ) .then((rows) => rows[0]); expect(removalAction).toEqual({ status: "failed", result: expect.objectContaining({ code: "telegram_maintenance_delivery_unknown", providerConfirmed: true, }), }); expect(acceptedRemovalMutations).toBe(2); // The simulated successor has no live worker. Retire only its synthetic // lease and removal retry after proving stale settlement could not clear // the successor refs, keeping later global recovery tests independent. await db.transaction(async (tx) => { await tx .update(chatActions) .set({ status: "cancelled", result: { code: "test_fixture_complete" }, updatedAt: new Date(), }) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "telegram_maintenance"), sql`${chatActions.payload}->>'operation' = 'remove_endpoint'`, ), ); await tx .delete(chatEndpointLeases) .where( and( eq(chatEndpointLeases.endpointId, endpoint.id), eq(chatEndpointLeases.leaseKey, "credentials"), ), ); }); await service.shutdown(); }); it("does not let stale GitHub setup-secret rotation overwrite successor credentials", async () => { const fixture = await seedCompany(); let connectionId: string | null = null; let successorRefs: typeof toolConnections.$inferSelect.credentialSecretRefs = []; let blockPersistence = false; let reclaimLease = false; let markLeaseReclaimed!: () => void; const leaseReclaimed = new Promise((resolve) => { markLeaseReclaimed = resolve; }); const { service } = createService( new FakeChatSdkRuntime(), fakeSlackFetch() as typeof globalThis.fetch, { credentialMutationLeaseRenewalIntervalMs: 5, setupSecretCredentialPersistBarrier: async () => { if (!blockPersistence) return; reclaimLease = true; await leaseReclaimed; }, renewCredentialMutationLease: async (input) => { if (!reclaimLease) return true; const reclaimed = await db .update(chatEndpointLeases) .set({ token: `reclaimed-${randomUUID()}`, expiresAt: new Date(Date.now() + 90_000), updatedAt: new Date(), }) .where( and( eq(chatEndpointLeases.endpointId, input.endpointId), eq(chatEndpointLeases.leaseKey, input.leaseKey), eq(chatEndpointLeases.token, input.token), ), ) .returning({ id: chatEndpointLeases.id }); if (reclaimed.length > 0) { if (!connectionId) throw new Error("Expected GitHub connection"); await db .update(toolConnections) .set({ credentialSecretRefs: successorRefs, updatedAt: new Date(), }) .where(eq(toolConnections.id, connectionId)); markLeaseReclaimed(); } return false; }, }, ); const endpoint = await service.create( fixture.companyId, { provider: "github", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); connectionId = endpoint.connectionId; await service.generateSetupSecret(endpoint.id, "owner-user"); const currentRefs = await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, connectionId)) .then((rows) => rows[0]!.refs); successorRefs = currentRefs.map((ref) => ({ ...ref, secretId: randomUUID(), })); blockPersistence = true; await expect( service.generateSetupSecret(endpoint.id, "owner-user"), ).rejects.toMatchObject({ code: "CHAT_CREDENTIAL_LEASE_LOST" }); await expect( db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, connectionId)), ).resolves.toEqual([{ refs: successorRefs }]); await service.shutdown(); }); it("returns a committed one-time GitHub setup secret after final lease loss when its exact ref remains current", async () => { const fixture = await seedCompany(); let loseFinalOwnership = false; const { service } = createService( new FakeChatSdkRuntime(), fakeSlackFetch() as typeof globalThis.fetch, { credentialMutationLeaseRenewalIntervalMs: 60_000, setupSecretFinalOwnershipBarrier: async () => { loseFinalOwnership = true; }, renewCredentialMutationLease: async (input) => { if (!loseFinalOwnership) return true; await db .update(chatEndpointLeases) .set({ token: `reclaimed-${randomUUID()}`, expiresAt: new Date(Date.now() + 90_000), updatedAt: new Date(), }) .where( and( eq(chatEndpointLeases.endpointId, input.endpointId), eq(chatEndpointLeases.leaseKey, input.leaseKey), eq(chatEndpointLeases.token, input.token), ), ); return false; }, }, ); const endpoint = await service.create( fixture.companyId, { provider: "github", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); const generated = await service.generateSetupSecret( endpoint.id, "owner-user", ); expect(generated.webhookSecret).toMatch(/^[a-f0-9]{64}$/); await expect( db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)), ).resolves.toEqual([ { refs: [ expect.objectContaining({ configPath: "credentials.webhookSecret", secretId: expect.any(String), }), ], }, ]); await expect( db .select({ token: chatEndpointLeases.token }) .from(chatEndpointLeases) .where( and( eq(chatEndpointLeases.endpointId, endpoint.id), eq(chatEndpointLeases.leaseKey, "credentials"), ), ), ).resolves.toEqual([{ token: expect.stringMatching(/^reclaimed-/) }]); await service.shutdown(); }); it("withholds a committed one-time GitHub setup secret after final lease loss when its ref was superseded", async () => { const fixture = await seedCompany(); let connectionId: string | null = null; let finalOwnershipLost = false; const { service } = createService( new FakeChatSdkRuntime(), fakeSlackFetch() as typeof globalThis.fetch, { credentialMutationLeaseRenewalIntervalMs: 60_000, setupSecretFinalOwnershipBarrier: async () => { if (!connectionId) throw new Error("Expected GitHub connection"); await db.transaction(async (tx) => { const [reclaimed] = await tx .update(chatEndpointLeases) .set({ token: `reclaimed-${randomUUID()}`, expiresAt: new Date(Date.now() + 90_000), updatedAt: new Date(), }) .where( and( eq(chatEndpointLeases.endpointId, endpoint.id), eq(chatEndpointLeases.leaseKey, "credentials"), ), ) .returning({ id: chatEndpointLeases.id }); if (!reclaimed) throw new Error("Expected to reclaim the setup-secret lease"); await tx .delete(companySecretBindings) .where( and( eq(companySecretBindings.companyId, fixture.companyId), eq(companySecretBindings.targetType, "tool_connection"), eq(companySecretBindings.targetId, connectionId), ), ); await tx .update(toolConnections) .set({ credentialSecretRefs: [], updatedAt: new Date() }) .where(eq(toolConnections.id, connectionId)); }); finalOwnershipLost = true; }, renewCredentialMutationLease: async () => !finalOwnershipLost, }, ); const endpoint = await service.create( fixture.companyId, { provider: "github", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); connectionId = endpoint.connectionId; await expect( service.generateSetupSecret(endpoint.id, "owner-user"), ).rejects.toMatchObject({ code: "CHAT_CREDENTIAL_LEASE_LOST" }); await expect( db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)), ).resolves.toEqual([{ refs: [] }]); await service.shutdown(); }); it("does not commit endpoint reach changes after credential-lease ownership is lost", async () => { const fixture = await seedCompany(); const { service } = createService( new FakeChatSdkRuntime(), fakeSlackFetch() as typeof globalThis.fetch, { credentialMutationLeaseRenewalIntervalMs: 60_000, renewCredentialMutationLease: async () => false, }, ); const endpoint = await service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await expect( service.update(endpoint.id, { allowDirectMessages: false }, "owner-user"), ).rejects.toMatchObject({ code: "CHAT_CREDENTIAL_LEASE_LOST" }); await expect( db .select({ allowDirectMessages: chatEndpoints.allowDirectMessages }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)), ).resolves.toEqual([{ allowDirectMessages: true }]); await expect( db .select({ id: activityLog.id }) .from(activityLog) .where( and( eq(activityLog.entityId, endpoint.connectionId), eq(activityLog.action, "chat_endpoint.updated"), ), ), ).resolves.toEqual([]); await service.shutdown(); }); it("does not commit stale Slack task-start recovery after credential-lease ownership is lost", async () => { const fixture = await seedCompany(); const { service } = createService( new FakeChatSdkRuntime(), fakeSlackFetch() as typeof globalThis.fetch, { credentialMutationLeaseRenewalIntervalMs: 60_000, renewCredentialMutationLease: async () => false, }, ); const endpoint = await service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await db.insert(chatActions).values([ { companyId: fixture.companyId, endpointId: endpoint.id, kind: "slash_task_start", providerActionId: `slash_task:lease-recovery-resolving-${randomUUID()}`, payload: {}, result: { attemptCount: 1 }, status: "resolving", updatedAt: new Date(0), }, { companyId: fixture.companyId, endpointId: endpoint.id, kind: "slash_task_start", providerActionId: `slash_task:lease-recovery-validating-${randomUUID()}`, payload: {}, result: { attemptCount: 1 }, status: "validating", updatedAt: new Date(0), }, ]); await service.processPendingDeliveries(); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.endpointId, endpoint.id)) .orderBy(asc(chatActions.providerActionId)), ).resolves.toEqual([{ status: "resolving" }, { status: "validating" }]); await service.shutdown(); }); it("does not let an obsolete Slack task-start failure quarantine successor credentials or reach", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture, { scheduleDeferredWork: () => undefined, }); if (!callbacks.onSlashCommand || !endpoint.setup.command) { throw new Error("Slack slash command setup was incomplete"); } const [resource] = await db .insert(chatEndpointResources) .values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: "C-STALE-FAILURE-FENCE", label: "stale-failure-fence", availability: "available", enabled: true, }) .returning(); const current = await db .select({ refs: toolConnections.credentialSecretRefs, setup: chatEndpoints.setup, }) .from(chatEndpoints) .innerJoin( toolConnections, eq(toolConnections.id, chatEndpoints.connectionId), ) .where(eq(chatEndpoints.id, endpoint.id)) .then((rows) => rows[0]!); const successorRefs = current.refs.map((ref) => ({ ...ref, secretId: randomUUID(), })); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack provider runtime"); providerRuntime.postHook = async () => { await db.transaction(async (tx) => { await tx .update(toolConnections) .set({ credentialSecretRefs: successorRefs, updatedAt: new Date() }) .where(eq(toolConnections.id, endpoint.connectionId)); await tx .update(chatEndpoints) .set({ setup: { ...current.setup, runtimeGeneration: Number( (current.setup as { runtimeGeneration?: unknown }) .runtimeGeneration ?? 0, ) + 1, }, updatedAt: new Date(), }) .where(eq(chatEndpoints.id, endpoint.id)); // Model a successor that reclaimed, rotated, and released the lease // while the obsolete provider request was in flight. await tx .delete(chatEndpointLeases) .where( and( eq(chatEndpointLeases.endpointId, endpoint.id), eq(chatEndpointLeases.leaseKey, "credentials"), ), ); }); }; providerRuntime.postError = Object.assign(new Error("Unauthorized"), { adapter: "slack", code: "AUTH_FAILED", name: "AuthenticationError", status: 401, }); await callbacks.onSlashCommand({ endpointId: endpoint.id, provider: "slack", event: { channel: { id: "C-STALE-FAILURE-FENCE", name: "stale-failure-fence", isDM: false, post: vi.fn(), postEphemeral: vi.fn(), } as never, command: endpoint.setup.command, text: "do not let stale auth quarantine successor credentials", triggerId: `trigger-stale-failure-${randomUUID()}`, user: { userId: "U-STALE-FAILURE-FENCE", userName: "stale-failure-fence", fullName: "Stale Failure Fence", isBot: false, isMe: false, isSystem: false, }, raw: { command: endpoint.setup.command }, adapter: {} as never, openModal: async () => undefined, }, }); await service.processPendingDeliveries(); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slash_task_start"), ), ), ).resolves.toEqual([ { status: "queued", result: expect.objectContaining({ code: "slash_task_runtime_superseded", retryable: true, }), }, ]); await expect( db .select({ lastError: chatEndpoints.lastError, status: chatEndpoints.status, }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)), ).resolves.toEqual([{ lastError: null, status: "verifying" }]); await expect( db .select({ enabled: toolConnections.enabled, healthStatus: toolConnections.healthStatus, lastError: toolConnections.lastError, refs: toolConnections.credentialSecretRefs, status: toolConnections.status, }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)), ).resolves.toEqual([ { enabled: true, healthStatus: "healthy", lastError: null, refs: successorRefs, status: "active", }, ]); await expect( db .select({ availability: chatEndpointResources.availability }) .from(chatEndpointResources) .where(eq(chatEndpointResources.id, resource!.id)), ).resolves.toEqual([{ availability: "available" }]); expect(providerRuntime.posts).toEqual([]); // This fault fixture deliberately points the successor at nonexistent // secret ids. Keep that synthetic queued retry from entering later tests' // global outbox scans and obscuring their own timing/ownership assertions. await db .update(chatActions) .set({ status: "cancelled", result: { code: "test_fixture_complete" }, updatedAt: new Date(), }) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slash_task_start"), ), ); await service.shutdown(); }); it("does not cancel an ambiguous Slack task start after credential-lease ownership is lost", async () => { const fixture = await seedCompany(); const { service } = createService( new FakeChatSdkRuntime(), fakeSlackFetch() as typeof globalThis.fetch, { credentialMutationLeaseRenewalIntervalMs: 60_000, renewCredentialMutationLease: async () => false, }, ); const endpoint = await service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); const [action] = await db .insert(chatActions) .values({ companyId: fixture.companyId, endpointId: endpoint.id, kind: "slash_task_start", providerActionId: `slash_task:lease-cancel-${randomUUID()}`, payload: {}, status: "received", updatedAt: new Date(0), }) .returning(); await expect( service.resolveAction(endpoint.id, action!.id, "cancel", "owner-user"), ).rejects.toMatchObject({ code: "CHAT_CREDENTIAL_LEASE_LOST" }); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, action!.id)), ).resolves.toEqual([{ status: "received" }]); await expect( db .select({ id: activityLog.id }) .from(activityLog) .where( and( eq(activityLog.entityId, action!.id), eq(activityLog.action, "chat.slack_command_cancel"), ), ), ).resolves.toEqual([]); await service.shutdown(); }); it("does not normalize a stale Slack retry-anyway action after credential-lease ownership is lost", async () => { const fixture = await seedCompany(); let loseLease = false; const { callbacks, endpoint, service } = await configuredSlackEndpoint( fixture, { allowUnlinkedPeople: true, credentialMutationLeaseRenewalIntervalMs: 60_000, renewCredentialMutationLease: async () => !loseLease, scheduleDeferredWork: () => undefined, }, ); if (!callbacks.onSlashCommand || !endpoint.setup.command) { throw new Error("Slack slash command setup was incomplete"); } await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: "C-LEASE-RETRY", label: "lease-retry", availability: "available", enabled: true, }); await callbacks.onSlashCommand({ endpointId: endpoint.id, provider: "slack", event: { channel: { id: "C-LEASE-RETRY", name: "lease-retry", isDM: false, post: vi.fn(), postEphemeral: vi.fn(), } as never, command: endpoint.setup.command, text: "retry this task only with a live credential lease", triggerId: `trigger-lease-retry-${randomUUID()}`, user: { userId: "U-LEASE-RETRY", userName: "lease-retry", fullName: "Lease Retry User", isBot: false, isMe: false, isSystem: false, }, raw: { command: endpoint.setup.command }, adapter: {} as never, openModal: async () => undefined, }, }); const action = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slash_task_start"), ), ) .then((rows) => rows[0]!); await db .update(chatActions) .set({ status: "received", result: { attemptCount: 4 }, updatedAt: new Date(0), }) .where(eq(chatActions.id, action.id)); loseLease = true; await expect( service.resolveAction( endpoint.id, action.id, "retry_anyway", "owner-user", ), ).rejects.toMatchObject({ code: "CHAT_CREDENTIAL_LEASE_LOST" }); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, action.id)), ).resolves.toEqual([{ status: "received", result: { attemptCount: 4 } }]); await expect( db .select({ id: activityLog.id }) .from(activityLog) .where( and( eq(activityLog.entityId, action.id), eq(activityLog.action, "chat.slack_command_retry_anyway"), ), ), ).resolves.toEqual([]); await service.shutdown(); }); it("preserves delivery quarantine when a slow Slack post finishes after stale recovery", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const thread = makeThread({ channelId: "C-SLOW-OUTBOX", id: "slack:C-SLOW-OUTBOX:4092.1", name: "slow-outbox", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: "4092.1", text: "@maya create a slow-provider task", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const [publication] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `slow-provider:${endpoint.id}`, payload: { text: "A slow provider accepted this exactly once" }, state: "pending", }) .returning(); const endpointRuntime = runtime.endpoints.get(endpoint.id); if (!endpointRuntime) throw new Error("Expected Slack runtime"); endpointRuntime.posts.length = 0; let markPostStarted!: () => void; const postStarted = new Promise((resolve) => { markPostStarted = resolve; }); let releasePost!: () => void; const postBlocked = new Promise((resolve) => { releasePost = resolve; }); endpointRuntime.postHook = async () => { markPostStarted(); await postBlocked; }; const originalWorker = service.processPendingPublications(); try { await postStarted; await db .update(chatPublications) .set({ updatedAt: new Date(Date.now() - 120_000) }) .where(eq(chatPublications.id, publication.id)); // An aged but renewed live worker is not orphaned. Explicitly expire its // exact durable attempt before simulating the periodic recovery scan. await db .update(chatEndpointLeases) .set({ expiresAt: new Date(0) }) .where( and( eq(chatEndpointLeases.endpointId, endpoint.id), eq(chatEndpointLeases.leaseKey, `publication:${publication.id}:1`), ), ); await service.schedulePendingPublications(0); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.id, publication.id)), ).resolves.toEqual([{ state: "delivery_unknown" }]); releasePost(); await originalWorker; await service.processPendingPublications(); expect(endpointRuntime.posts).toEqual([ { threadId: thread.thread.id, text: "A slow provider accepted this exactly once", }, ]); await expect( db .select({ attempts: chatPublications.attempts, providerMessageId: chatPublications.providerMessageId, state: chatPublications.state, }) .from(chatPublications) .where(eq(chatPublications.id, publication.id)), ).resolves.toEqual([ { attempts: 1, providerMessageId: null, state: "delivery_unknown", }, ]); } finally { releasePost(); try { await originalWorker; } finally { await service.shutdown(); } } }); it("releases task-control authorization locks before provider I/O and quarantines stale claims", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const thread = makeThread({ channelId: "C-CONTROL-CLAIM", id: "slack:C-CONTROL-CLAIM:4093.1", name: "control-claim", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: "4093.1", text: "@maya create a task-control claim fixture", mentioned: true, userId: "U-CONTROL-CLAIM", }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id, "U-CONTROL-CLAIM"); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const [principal] = await db .select() .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, fixture.companyId), eq(chatExternalPrincipals.externalId, "U-CONTROL-CLAIM"), ), ); if (!conversation || !principal) { throw new Error("Expected the Slack control authorization fixture"); } const [publication] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `control:status:${randomUUID()}`, payload: { classification: "external", source: "task_control", text: "Status", }, state: "pending", }) .returning(); const [authorization] = await db .insert(chatActions) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, principalId: principal.id, kind: "task_control_authorization", providerActionId: `task-control-authorization:${publication.id}`, payload: { publicationId: publication.id }, status: "issued", }) .returning(); const endpointRuntime = runtime.endpoints.get(endpoint.id); if (!endpointRuntime) throw new Error("Expected Slack runtime"); endpointRuntime.posts.length = 0; let markPostStarted!: () => void; const postStarted = new Promise((resolve) => { markPostStarted = resolve; }); let releasePost!: () => void; const postBlocked = new Promise((resolve) => { releasePost = resolve; }); endpointRuntime.postHook = async () => { markPostStarted(); await postBlocked; }; const originalWorker = service.processPendingPublications(); try { await postStarted; await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, authorization.id)), ).resolves.toEqual([ { status: "processing", result: expect.objectContaining({ attempts: 1, credentialFingerprint: expect.any(String), runtimeGeneration: expect.any(Number), }), }, ]); const authorizationLockProbe = db.transaction(async (tx) => { await tx.execute( sql`select pg_advisory_xact_lock(hashtextextended(${`chat-identity:${fixture.companyId}:${principal.id}`}, 0))`, ); return "authorization-lock-released" as const; }); const lockOutcome = await Promise.race([ authorizationLockProbe, new Promise<"authorization-lock-held">((resolve) => setTimeout(() => resolve("authorization-lock-held"), 500), ), ]); if (lockOutcome !== "authorization-lock-released") { releasePost(); await originalWorker; await authorizationLockProbe; } expect(lockOutcome).toBe("authorization-lock-released"); await db .update(chatPublications) .set({ updatedAt: new Date(Date.now() - 120_000) }) .where(eq(chatPublications.id, publication.id)); await db .update(chatEndpointLeases) .set({ expiresAt: new Date(0) }) .where( and( eq(chatEndpointLeases.endpointId, endpoint.id), eq(chatEndpointLeases.leaseKey, `publication:${publication.id}:1`), ), ); await service.schedulePendingPublications(0); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, authorization.id)), ).resolves.toEqual([{ status: "delivery_unknown" }]); releasePost(); await originalWorker; await service.processPendingPublications(); expect(endpointRuntime.posts).toHaveLength(1); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.id, publication.id)), ).resolves.toEqual([{ state: "delivery_unknown" }]); endpointRuntime.postHook = undefined; await service.resolvePublication( endpoint.id, publication.id, "retry_anyway", "owner-user", ); await service.processPendingPublications(); expect(endpointRuntime.posts).toHaveLength(2); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, authorization.id)), ).resolves.toEqual([{ status: "processed" }]); } finally { releasePost(); try { await originalWorker; } finally { await service.shutdown(); } } }); it("does not let Slack reach disables return ahead of in-flight provider sends", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const thread = makeThread({ channelId: "C-RESOURCE-OUTBOX", id: "slack:C-RESOURCE-OUTBOX:4095.1", name: "resource-outbox", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, message: makeMessage({ id: "4095.1", text: "@maya create a resource-fenced task", mentioned: true, }), trigger: "mention", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const resource = (await service.listResources(endpoint.id)).find( (candidate) => candidate.providerResourceId === "C-RESOURCE-OUTBOX", ); if (!resource) throw new Error("Expected the admitted Slack resource"); const [publication] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `resource-fence:${endpoint.id}`, payload: { text: "Publication already entering Slack transport" }, state: "pending", }) .returning(); const endpointRuntime = runtime.endpoints.get(endpoint.id); if (!endpointRuntime) throw new Error("Expected Slack runtime"); let markPostStarted!: () => void; const postStarted = new Promise((resolve) => { markPostStarted = resolve; }); let releasePost!: () => void; const postBlocked = new Promise((resolve) => { releasePost = resolve; }); endpointRuntime.postHook = async () => { markPostStarted(); await postBlocked; }; const processing = service.processPendingPublications(); await postStarted; const disabling = service.replaceResources(endpoint.id, [ { id: resource.id, enabled: false }, ]); await expect( Promise.race([ disabling.then(() => "disabled"), new Promise((resolve) => setTimeout(() => resolve("transport-in-flight"), 100), ), ]), ).resolves.toBe("transport-in-flight"); releasePost(); await Promise.all([processing, disabling]); expect(endpointRuntime.posts).toEqual([ { threadId: thread.thread.id, text: "Publication already entering Slack transport", }, ]); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.id, publication.id)), ).resolves.toEqual([{ state: "published" }]); endpointRuntime.postHook = undefined; const [revokedPublication] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `resource-fence-revoked:${endpoint.id}`, payload: { text: "This must not reach Slack after disable" }, state: "pending", }) .returning(); await service.processPendingPublications(); expect(endpointRuntime.posts).toHaveLength(1); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.id, revokedPublication.id)), ).resolves.toEqual([{ state: "cancelled" }]); const dm = makeThread({ channelId: "D-REACH-OUTBOX", id: "slack:D-REACH-OUTBOX:", isDM: true, name: "direct message", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: dm.thread, message: makeMessage({ id: "4096.1", text: "create a direct-message-fenced task", }), trigger: "direct_message", }); const dmConversation = (await service.listConversations(endpoint.id)).find( (candidate) => candidate.externalThreadId === dm.thread.id, ); if (!dmConversation) throw new Error("Expected the admitted Slack DM"); const [dmPublication] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: dmConversation.id, issueId: dmConversation.issueId, idempotencyKey: `dm-reach-fence:${endpoint.id}`, payload: { text: "DM publication already entering Slack transport" }, state: "pending", }) .returning(); let markDmPostStarted!: () => void; const dmPostStarted = new Promise((resolve) => { markDmPostStarted = resolve; }); let releaseDmPost!: () => void; const dmPostBlocked = new Promise((resolve) => { releaseDmPost = resolve; }); endpointRuntime.postHook = async () => { markDmPostStarted(); await dmPostBlocked; }; const processingDm = service.processPendingPublications(); await dmPostStarted; const disablingDm = service.update( endpoint.id, { allowDirectMessages: false }, "owner-user", ); await expect( Promise.race([ disablingDm.then(() => "disabled"), new Promise((resolve) => setTimeout(() => resolve("transport-in-flight"), 100), ), ]), ).resolves.toBe("transport-in-flight"); releaseDmPost(); await Promise.all([processingDm, disablingDm]); expect(endpointRuntime.posts.at(-1)).toEqual({ threadId: dm.thread.id, text: "DM publication already entering Slack transport", }); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.id, dmPublication.id)), ).resolves.toEqual([{ state: "published" }]); endpointRuntime.postHook = undefined; const [revokedDmPublication] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: dmConversation.id, issueId: dmConversation.issueId, idempotencyKey: `dm-reach-fence-revoked:${endpoint.id}`, payload: { text: "This must not reach Slack after DM disable" }, state: "pending", }) .returning(); await service.processPendingPublications(); expect(endpointRuntime.posts).toHaveLength(2); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.id, revokedDmPublication.id)), ).resolves.toEqual([{ state: "cancelled" }]); await service.shutdown(); }); it("streams long output in bounded chunks after applying the safe external projection", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const channel = makeThread({ channelId: "C-LONG-SAFE", id: "slack:C-LONG-SAFE:4100.1", name: "long-safe-output", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "4100.1", text: "@maya send the long public summary", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const publicParagraph = "External-safe result. ".repeat(220).trim(); const comment = await issueService(db).addComment( conversation.issueId, `${publicParagraph}\n\nprivate chain of thought must never stream`, { userId: "owner-user" }, { authorType: "user" }, ); await service.publishComment(endpoint.id, conversation.id, comment.id); const [publication] = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, comment.id)); expect(publication).toMatchObject({ state: "published", payload: { text: publicParagraph }, }); const providerRuntime = runtime.endpoints.get(endpoint.id); const streamed = providerRuntime?.posts.at(-1); expect(streamed?.chunks?.length).toBeGreaterThan(1); expect( streamed?.chunks?.every((chunk) => Array.from(chunk).length <= 2_000), ).toBe(true); expect(streamed?.chunks?.join("")).toBe(publication.payload.text); expect(streamed?.text).toBe(publicParagraph); expect(JSON.stringify(streamed)).not.toContain("private chain of thought"); }); it("imports every Discord upload when text files include MIME parameters", async () => { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, service, wakeup } = await configuredDiscordEndpoint(fixture, { storage: storage.storage }); const textBody = Buffer.from("multi-file Discord text", "utf8"); const imageBody = Buffer.from("multi-file Discord image", "utf8"); const textFetch = vi.fn(async () => textBody); const imageFetch = vi.fn(async () => imageBody); const rejectedFetch = vi.fn(async () => Buffer.from("not imported")); const rootMessageId = "555555555555555596"; const channel = makeThread({ channelId: "333333333333333329", id: `discord:1457808928258658549:333333333333333329:${rootMessageId}`, name: "discord-multi-upload", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "discord", thread: channel.thread, message: makeMessage({ attachments: [ { type: "file", name: "native-inbound.txt", mimeType: "text/plain; charset=utf-8", size: textBody.length, fetchData: textFetch, fetchMetadata: { testRecoveryKey: "discord-multi-text" }, } as Attachment, { type: "image", name: "native-inbound.png", mimeType: "image/png", size: imageBody.length, fetchData: imageFetch, fetchMetadata: { testRecoveryKey: "discord-multi-image" }, } as Attachment, { type: "file", name: "unsupported.exe", mimeType: "application/x-msdownload", size: 12, fetchData: rejectedFetch, } as Attachment, ], id: rootMessageId, mentioned: true, text: "@maya inspect both files", }), trigger: "mention", }); expect(textFetch).toHaveBeenCalledTimes(1); expect(imageFetch).toHaveBeenCalledTimes(1); expect(rejectedFetch).not.toHaveBeenCalled(); expect(storage.putFile).toHaveBeenCalledTimes(2); const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(delivery).toMatchObject({ state: "processed", redactedError: "1 external attachment was omitted (unsupported type: 1)", }); expect(wakeup).toHaveBeenCalledTimes(1); expect(wakeup.mock.calls[0]?.[1]?.contextSnapshot).toMatchObject({ externalAttachmentOmissions: [ { commentId: expect.any(String), reasons: { unsupported_type: 1 }, }, ], }); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Discord upload conversation"); await expect( db .select({ contentType: assets.contentType, originalFilename: assets.originalFilename, }) .from(issueAttachments) .innerJoin(assets, eq(assets.id, issueAttachments.assetId)) .where(eq(issueAttachments.issueId, conversation.issueId)) .orderBy(asc(assets.originalFilename)), ).resolves.toEqual([ { contentType: "image/png", originalFilename: "native-inbound.png", }, { contentType: "text/plain", originalFilename: "native-inbound.txt", }, ]); }); it("audits repeated Discord reaction cycles while deduplicating an exact Gateway replay", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service, wakeup } = await configuredDiscordEndpoint(fixture); const rootMessageId = "555555555555555597"; const channel = makeThread({ channelId: "333333333333333330", id: `discord:1457808928258658549:333333333333333330:${rootMessageId}`, name: "discord-reaction-cycles", }); const original = makeMessage({ id: rootMessageId, mentioned: true, text: "@maya observe repeated reactions", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "discord", thread: channel.thread, message: original, trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); if (!callbacks.onReaction) { throw new Error("Discord reaction callback was not registered"); } const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); if (!conversation) throw new Error("Expected Discord conversation"); const commentCountBefore = await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)) .then((rows) => rows.length); const wakeupCountBefore = wakeup.mock.calls.length; const emoji = { name: "thumbsup", toJSON: () => "👍", toString: () => "👍", }; const reaction = ( added: boolean, sessionFingerprint: string, sequence: number, ) => ({ endpointId: endpoint.id, provider: "discord" as const, event: { adapter: {} as never, added, emoji, message: original, messageId: original.id, raw: { channel_id: "333333333333333330", emoji: { id: null, name: "👍" }, gateway_dispatch: { eventType: added ? "MESSAGE_REACTION_ADD" : "MESSAGE_REACTION_REMOVE", sequence, sessionFingerprint, shardId: 0, }, guild_id: "1457808928258658549", message_id: original.id, user_id: original.author.userId, }, rawEmoji: "👍", thread: channel.thread, threadId: channel.thread.id, user: original.author, }, }); const firstSession = "a".repeat(24); const nextSession = "b".repeat(24); await callbacks.onReaction(reaction(true, firstSession, 42)); await callbacks.onReaction(reaction(true, firstSession, 42)); await callbacks.onReaction(reaction(false, firstSession, 43)); await callbacks.onReaction(reaction(true, firstSession, 44)); await callbacks.onReaction(reaction(false, firstSession, 45)); await callbacks.onReaction(reaction(true, nextSession, 42)); const reactions = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.conversationId, conversation.id)) .then((rows) => rows.filter((row) => row.eventKind.startsWith("reaction_")), ); expect(reactions).toHaveLength(5); expect(new Set(reactions.map((row) => row.providerEventId)).size).toBe(5); expect(reactions.map((row) => row.eventKind).sort()).toEqual([ "reaction_added", "reaction_added", "reaction_added", "reaction_removed", "reaction_removed", ]); expect(reactions.every((row) => row.state === "processed")).toBe(true); await expect( db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)), ).resolves.toHaveLength(commentCountBefore); expect(wakeup.mock.calls).toHaveLength(wakeupCountBefore); await service.shutdown(); }); it("durably audits a rejected Discord Gateway action before surfacing transport rejection", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredDiscordEndpoint(fixture); let pinned: ReturnType | undefined; try { await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, endpoint.id)); if (!callbacks.onAction) throw new Error("Discord action callback was not registered"); const configuration = runtime.configurations.get(endpoint.id)!; const observedErrors: string[] = []; const onAction = vi.fn( async ( event: Parameters>[0], ) => { try { await callbacks.onAction!(event); } catch (error) { observedErrors.push(String((error as { code?: unknown }).code)); throw error; } }, ); pinned = createChatSdkEndpointRuntime({ ...configuration, callbacks: { onMessage() {}, onAction }, enableDiscordGateway: false, logger: "silent", }); await pinned.initialize(); const applicationId = configuration.providerConfig.provider === "discord" ? configuration.providerConfig.credentials.applicationId : undefined; const gatewayInteraction = { applicationId, channel: { id: "555555555555555598", parentId: "333333333333333333", type: 11, }, channelId: "555555555555555598", componentType: 2, customId: "pcq:forged-discord-action\nforged-value", deferUpdate: vi.fn().mockResolvedValue(undefined), guildId: "1457808928258658549", id: "777777777777777710", isChatInputCommand: () => false, isMessageComponent: () => true, message: { id: "555555555555555597" }, reply: vi.fn(async () => { const rows = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "action"), ), ); expect(rows).toHaveLength(1); expect(rows[0]?.state).toBe("filtered"); }), token: "synthetic-interaction-token", type: 3, user: { id: "444444444444444444", username: "discord-user", globalName: "Discord User", bot: false, }, version: 1, }; const adapter = pinned.getProviderAdapter() as unknown as { handleGatewayInteraction( event: typeof gatewayInteraction, ): Promise; }; const priorWakeups = wakeup.mock.calls.length; // Repeat one synthetic Gateway delivery. These are response attempts, // not proof that Discord accepts two replies to one interaction token. await adapter.handleGatewayInteraction(gatewayInteraction); await adapter.handleGatewayInteraction(gatewayInteraction); expect(onAction).toHaveBeenCalledTimes(2); expect(onAction.mock.calls[0]![0].event.raw).not.toHaveProperty( "deferUpdate", ); expect(wakeup.mock.calls).toHaveLength(priorWakeups); const denials = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "action"), ), ); expect(denials).toEqual([ expect.objectContaining({ state: "filtered", attempts: 1, redactedError: "External action denied by Paperclip authorization", normalizedEvent: { providerEventId: expect.stringMatching( /^action-denied:[a-f0-9]{64}$/, ), kind: "action", authorization: { outcome: "denied" }, }, }), ]); expect(JSON.stringify(denials)).not.toContain( "pcq:forged-discord-action", ); expect({ errors: observedErrors, successAcknowledgements: gatewayInteraction.deferUpdate.mock.calls.length, rejectionReplies: gatewayInteraction.reply.mock.calls.length, }).toEqual({ errors: [ "chat_discord_gateway_action_rejected", "chat_discord_gateway_action_rejected", ], successAcknowledgements: 0, rejectionReplies: 2, }); expect(gatewayInteraction.reply).toHaveBeenLastCalledWith({ content: "This action is no longer available. Open the linked Paperclip task or ask an operator to link this account.", flags: 64, }); } finally { await pinned?.shutdown(); await service.shutdown(); } }); it("retires a Discord receipt after terminal failure without letting an add retry resurrect it", async () => { const fixture = await seedCompany(); const deferred: Array<() => void> = []; const { callbacks, endpoint, runtime, service, wakeup } = await configuredDiscordEndpoint(fixture, { scheduleDeferredWork: (task) => deferred.push(task), }); try { const guildId = "1457808928258658549"; const channelId = "333333333333333333"; const rootMessageId = "555555555555555609"; const externalUserId = "444444444444444409"; const threadId = `discord:${guildId}:${channelId}:${rootMessageId}`; const thread = makeThread({ channelId, id: threadId, name: "discord-terminal-receipt", }); const admitRootMention = callbacks.onDiscordRootMentionAdmission; if (!admitRootMention) { throw new Error("Expected Discord root-mention admission callback"); } const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Discord runtime"); providerRuntime.reactionErrors.push( Object.assign(new Error("Discord receipt rate limited"), { status: 429, retryAfterMs: 60_000, }), ); await expect( admitRootMention({ endpointId: endpoint.id, guildId, channelId, messageId: rootMessageId, message: { ...makeMessage({ id: rootMessageId, text: "@maya demonstrate terminal receipt cleanup", mentioned: true, userId: externalUserId, }), threadId, } as Message, threadId, userId: externalUserId, }), ).resolves.toBe(false); const [delivery] = await db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.providerEventId, `${threadId}:${rootMessageId}`), ), ); if (!delivery) throw new Error("Expected Discord root delivery"); await service.processPendingDeliveries(25, delivery.id); expect(providerRuntime.reactions).toHaveLength(0); const [addAction] = await db .select() .from(chatActions) .where( eq(chatActions.providerActionId, `receipt_reaction:${delivery.id}`), ); expect(addAction).toMatchObject({ status: "failed", result: { attempts: 1, code: "receipt_reaction_retry", retryable: true, }, }); deferred.length = 0; providerRuntime.removeReactionErrors.push( Object.assign(new Error("connection reset before response"), { code: "ECONNRESET", name: "NetworkError", }), ); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Discord conversation"); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "failed", errorCode: "low_trust_isolation_unavailable", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "discord", providerMessageId: rootMessageId, }), }); await expect( enqueueChatRunMilestones(db, { publicBaseUrl: "https://paperclip.example", }), ).resolves.toBe(1); await service.processPendingPublications(1_000); const [failedPublication] = await db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `run:${runId}:failed:${endpoint.id}`, ), ); expect(failedPublication).toMatchObject({ state: "published", payload: { progressState: "failed", text: expect.stringContaining("couldn't safely start this turn"), }, }); expect( providerRuntime.posts.filter((post) => post.text.includes("couldn't safely start this turn"), ), ).toHaveLength(1); expect(wakeup).toHaveBeenCalledTimes(1); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, addAction.id)), ).resolves.toEqual([ { status: "cancelled", result: { attempts: 1, code: "receipt_reaction_superseded_by_terminal_publication", }, }, ]); expect(deferred).toHaveLength(1); for (const task of deferred.splice(0)) task(); let removal: typeof chatActions.$inferSelect | undefined; await vi.waitFor(async () => { removal = await db .select() .from(chatActions) .where( eq( chatActions.providerActionId, `receipt_reaction_remove:${delivery.id}`, ), ) .then((rows) => rows[0]); expect(removal?.status).toBe("failed"); }); if (!removal) throw new Error("Expected Discord receipt removal"); expect(removal).toMatchObject({ payload: { operation: "remove", threadId, messageId: rootMessageId, reaction: "eyes", }, result: { attempts: 1, code: "receipt_reaction_removal_delivery_unknown", retryable: true, }, }); // Even if a stale scheduler presents the old add as retryable after the // terminal marker commits, it must be cancelled before provider I/O. await db .update(chatActions) .set({ status: "failed", result: { attempts: 1, retryable: true, retryAt: new Date(0).toISOString(), }, updatedAt: new Date(), }) .where(eq(chatActions.id, addAction.id)); await service.processPendingReceiptReactions(1, addAction.id); expect(providerRuntime.reactions).toHaveLength(0); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, addAction.id)), ).resolves.toEqual([ { status: "cancelled", result: { attempts: 1, code: "receipt_reaction_superseded_by_terminal_publication", }, }, ]); await db .update(chatActions) .set({ result: { ...removal.result, retryAt: new Date(0).toISOString(), }, updatedAt: new Date(), }) .where(eq(chatActions.id, removal.id)); await service.processPendingReceiptReactions(1, removal.id); await service.processPendingReceiptReactions(1, removal.id); expect(providerRuntime.removedReactions).toEqual([ { threadId, messageId: rootMessageId, emoji: "eyes" }, ]); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, removal.id)), ).resolves.toEqual([ { status: "processed", result: { attempts: 2 }, }, ]); } finally { await service.shutdown(); } }); it("composes a parsed Discord question answer with one same-thread continuation publication", async () => { const fixture = await seedCompany(); const continuationRunId = randomUUID(); let continuationWakeCount = 0; const { callbacks, endpoint, runtime, service } = await configuredDiscordEndpoint(fixture, { wakeup: async (agentId, options) => { if (options.contextSnapshot?.source !== "issue.interaction.respond") { return { accepted: true }; } continuationWakeCount += 1; // The runner is outside this provider/control-plane composition. // Seed only its synthetic result, never claim a real model turn. const [created] = await db .insert(heartbeatRuns) .values({ id: continuationRunId, companyId: fixture.companyId, agentId, status: "succeeded", contextSnapshot: options.contextSnapshot, }) .onConflictDoNothing() .returning(); return ( created ?? db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, continuationRunId)) .then((rows) => rows[0]) ); }, }); let pinned: ReturnType | undefined; let threadSpy: ReturnType | undefined; try { const guildId = "1457808928258658549"; const channelId = "333333333333333333"; const rootMessageId = "555555555555555610"; const externalUserId = "444444444444444410"; const threadId = `discord:${guildId}:${channelId}:${rootMessageId}`; const channel = makeThread({ channelId, id: threadId, name: "discord-native-question", }); const admitRootMention = callbacks.onDiscordRootMentionAdmission; if (!admitRootMention || !callbacks.onAction) { throw new Error("Expected Discord root and action callbacks"); } await expect( admitRootMention({ endpointId: endpoint.id, guildId, channelId, messageId: rootMessageId, message: { ...makeMessage({ id: rootMessageId, text: "@maya help me choose a priority", mentioned: true, userId: externalUserId, }), threadId, } as Message, threadId, userId: externalUserId, }), ).resolves.toBe(false); const [rootDelivery] = await db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.providerEventId, `${threadId}:${rootMessageId}`), ), ); if (!rootDelivery) throw new Error("Expected Discord root delivery"); await service.processPendingDeliveries(25, rootDelivery.id); await qualifySetupRoundTrip(service, endpoint.id, externalUserId); await service.test(endpoint.id, "owner-user"); const activeEndpoint = await service.get(endpoint.id); if (!activeEndpoint.providerAccountId) { throw new Error("Expected Discord provider account identity"); } const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Discord conversation"); const linkedUserId = `discord-question-user-${randomUUID()}`; const now = new Date(); await db.insert(authUsers).values({ id: linkedUserId, name: "Discord Question User", email: `${linkedUserId}@example.com`, emailVerified: true, createdAt: now, updatedAt: now, }); await db.insert(companyMemberships).values({ companyId: fixture.companyId, principalType: "user", principalId: linkedUserId, status: "active", membershipRole: "operator", }); const principal = await db .select() .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, fixture.companyId), eq(chatExternalPrincipals.provider, "discord"), eq( chatExternalPrincipals.providerAccountId, activeEndpoint.providerAccountId, ), eq(chatExternalPrincipals.externalId, externalUserId), ), ) .then((rows) => rows[0]); if (!principal) throw new Error("Expected Discord external principal"); const intent = await service.createLinkIntent( endpoint.id, principal.id, 1_800, ); const linkToken = new URL(intent.confirmationUrl).searchParams.get( "token", ); if (!linkToken) throw new Error("Discord identity token was absent"); await service.confirmIdentityLink(linkToken, linkedUserId); const configuration = runtime.configurations.get(endpoint.id)!; const onAction = vi.fn(callbacks.onAction); pinned = createChatSdkEndpointRuntime({ ...configuration, callbacks: { onMessage() {}, onAction }, enableDiscordGateway: false, logger: "silent", }); await pinned.initialize(); const parsedAdapter = pinned.getProviderAdapter() as unknown as { buildMessagePayload(message: unknown): { payload: Record; }; handleGatewayInteraction(event: unknown): Promise; }; const providerRuntime = runtime.endpoints.get(endpoint.id)!; const renderedPayloads: Record[] = []; const originalThread = providerRuntime.thread.bind(providerRuntime); threadSpy = vi .spyOn(providerRuntime, "thread") .mockImplementation((id) => { const original = originalThread(id); return { ...original, post: async (message: Parameters[0]) => { renderedPayloads.push( parsedAdapter.buildMessagePayload(message).payload, ); return original.post(message); }, }; }); const sourceRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: sourceRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: { issueId: conversation.issueId, taskId: conversation.issueId, source: "automation", }, }); const interaction = await issueThreadInteractionService(db).create( { id: conversation.issueId, companyId: fixture.companyId }, { kind: "ask_user_questions", continuationPolicy: "wake_assignee", sourceRunId, title: "Choose the priority", payload: { version: 1, title: "Choose the priority", questions: [ { id: "priority", prompt: "Which priority should we use?", selectionMode: "single", required: true, allowOther: false, options: [ { id: "high", label: "High" }, { id: "normal", label: "Normal" }, ], }, ], }, }, { agentId: fixture.assignedAgentId, runId: sourceRunId }, ); await service.processPendingPublications(1_000); const questionPublication = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), eq(chatPublications.issueId, conversation.issueId), ), ) .then((rows) => rows.find((row) => row.payload.interactionId === interaction.id), ); if (!questionPublication?.providerMessageId) { throw new Error("Discord question publication was not delivered"); } const highAction = questionPublication.payload.card?.actions?.find( (candidate) => candidate.type === "callback" && candidate.label === "High", ); if (!highAction || highAction.type !== "callback") { throw new Error("Discord question callback was not projected"); } const findHighButton = (value: unknown): string[] => { if (Array.isArray(value)) return value.flatMap(findHighButton); if (!value || typeof value !== "object") return []; const object = value as Record; return [ ...(object.label === "High" && typeof object.custom_id === "string" ? [object.custom_id] : []), ...Object.values(object).flatMap(findHighButton), ]; }; const customIds = renderedPayloads.flatMap(findHighButton); expect(customIds).toEqual([`${highAction.actionId}\n${interaction.id}`]); const applicationId = configuration.providerConfig.provider === "discord" ? configuration.providerConfig.credentials.applicationId : undefined; const click = (id: string) => ({ applicationId, channel: { id: rootMessageId, parentId: channelId, type: 11 }, channelId: rootMessageId, componentType: 2, customId: customIds[0], deferUpdate: vi.fn().mockResolvedValue(undefined), guildId, id, isChatInputCommand: () => false, isMessageComponent: () => true, message: { id: questionPublication.providerMessageId }, reply: vi.fn().mockResolvedValue(undefined), token: "synthetic-interaction-token", type: 3, user: { id: externalUserId, username: "discord-user", globalName: "Discord User", bot: false, }, version: 1, }); const first = click("777777777777777711"); const concurrent = click("777777777777777712"); await Promise.all([ parsedAdapter.handleGatewayInteraction(first), parsedAdapter.handleGatewayInteraction(concurrent), ]); expect(onAction).toHaveBeenCalledTimes(2); expect(onAction.mock.calls[0]![0]).toMatchObject({ endpointId: endpoint.id, provider: "discord", transport: "discord_gateway", event: { actionId: highAction.actionId, value: interaction.id, threadId, }, }); expect(onAction.mock.calls[0]![0].event.raw).not.toHaveProperty( "deferUpdate", ); expect(first.deferUpdate).toHaveBeenCalledOnce(); expect(concurrent.deferUpdate).toHaveBeenCalledOnce(); expect(first.reply).not.toHaveBeenCalled(); expect(concurrent.reply).not.toHaveBeenCalled(); await service.processPendingPublications(1_000); const [storedInteraction] = await db .select() .from(issueThreadInteractions) .where(eq(issueThreadInteractions.id, interaction.id)); expect(storedInteraction).toMatchObject({ status: "answered", resolvedByUserId: linkedUserId, result: { version: 1, answers: [{ questionId: "priority", optionIds: ["high"] }], }, }); let resolutionPublication: typeof chatPublications.$inferSelect | undefined; await vi.waitFor(async () => { resolutionPublication = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), eq( chatPublications.idempotencyKey, `interaction-resolution:${interaction.id}:${endpoint.id}`, ), ), ) .then((rows) => rows[0]); expect(resolutionPublication?.state).toBe("published"); }); expect(resolutionPublication).toMatchObject({ state: "published", providerMessageId: questionPublication.providerMessageId, payload: { interactionId: interaction.id, text: "Answered: High.", card: { kind: "question", title: "Which priority should we use?", body: "Answered: High.", }, }, }); expect(resolutionPublication?.payload.card?.actions).toBeUndefined(); expect(providerRuntime?.edits).toEqual([ expect.objectContaining({ threadId, messageId: questionPublication.providerMessageId, }), ]); expect(providerRuntime?.edits[0]?.text).toContain("Answered: High."); await vi.waitFor(async () => { await expect( db .select({ id: heartbeatRuns.id }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, continuationRunId)), ).resolves.toHaveLength(1); }); expect(continuationWakeCount).toBe(1); const presentationAuthorization = await resolveChatRunPresentationAuthorizationReason(db, { companyId: fixture.companyId, issueId: conversation.issueId, runId: continuationRunId, }); expect(presentationAuthorization).toBe("allow_chat_run_presentation"); const exactMarker = "DISCORD-PRIORITY-HIGH"; const continuationComment = await issueService(db).addComment( conversation.issueId, exactMarker, { agentId: fixture.assignedAgentId, runId: continuationRunId }, { authorType: "agent", authorizationReason: presentationAuthorization, }, ); await service.processPendingPublications(1_000); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.commentId, continuationComment.id)), ).resolves.toEqual([ expect.objectContaining({ endpointId: endpoint.id, conversationId: conversation.id, state: "published", payload: expect.objectContaining({ text: exactMarker }), }), ]); expect(providerRuntime?.posts).toEqual( expect.arrayContaining([ expect.objectContaining({ threadId, text: exactMarker }), ]), ); // A late delivery of this resolved action is idempotent at the service // boundary; it must not create a second continuation or final reply. const late = click("777777777777777713"); await parsedAdapter.handleGatewayInteraction(late); await service.processPendingPublications(1_000); expect(late.deferUpdate).toHaveBeenCalledOnce(); expect(late.reply).not.toHaveBeenCalled(); expect(continuationWakeCount).toBe(1); expect( providerRuntime.posts.filter((post) => post.text === exactMarker), ).toHaveLength(1); } finally { threadSpy?.mockRestore(); await pinned?.shutdown(); await service.shutdown(); } }); it.each(["identical", "different"])( "opens and corrects an actual Discord modal through current service authority exactly once (%s concurrent answers)", async (concurrentMode) => { const fixture = await seedCompany(); let holdConcurrentAnswers = false; let answerWaiters = 0; let releaseAnswers!: () => void; const answersReady = new Promise((resolve) => { releaseAnswers = resolve; }); const { callbacks, endpoint, runtime, service, wakeup } = await configuredDiscordEndpoint(fixture, { questionResolutionPersistBarrier: async () => { if (!holdConcurrentAnswers) return; if (++answerWaiters === 2) releaseAnswers(); await answersReady; }, }); let pinned: ReturnType | undefined; let cleanupSpy: ReturnType | undefined; try { const guildId = "1457808928258658549"; const channelId = "333333333333333333"; const rootMessageId = "555555555555555615"; const externalUserId = "444444444444444415"; const threadId = `discord:${guildId}:${channelId}:${rootMessageId}`; if ( !callbacks.onDiscordRootMentionAdmission || !callbacks.onAction || !callbacks.onModalSubmit ) throw new Error("Discord callbacks unavailable"); await callbacks.onDiscordRootMentionAdmission({ endpointId: endpoint.id, guildId, channelId, messageId: rootMessageId, message: { ...makeMessage({ id: rootMessageId, text: "@maya collect deployment details", mentioned: true, userId: externalUserId, }), threadId, } as Message, threadId, userId: externalUserId, }); const delivery = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, `${threadId}:${rootMessageId}`, ), ), ) .then((rows) => rows[0]); if (!delivery) throw new Error("Discord delivery absent"); await service.processPendingDeliveries(25, delivery.id); await qualifySetupRoundTrip(service, endpoint.id, externalUserId); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); const active = await service.get(endpoint.id); if (!conversation || !active.providerAccountId) throw new Error("Discord setup incomplete"); expect(active.capabilities.modals).toBe(true); const linkedUserId = `discord-modal-${randomUUID()}`; const now = new Date(); await db.insert(authUsers).values({ id: linkedUserId, name: "Discord Modal Operator", email: `${linkedUserId}@example.com`, emailVerified: true, createdAt: now, updatedAt: now, }); await db.insert(companyMemberships).values({ companyId: fixture.companyId, principalType: "user", principalId: linkedUserId, status: "active", membershipRole: "operator", }); const principal = await db .select() .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, fixture.companyId), eq(chatExternalPrincipals.provider, "discord"), eq( chatExternalPrincipals.providerAccountId, active.providerAccountId, ), eq(chatExternalPrincipals.externalId, externalUserId), ), ) .then((rows) => rows[0]); if (!principal) throw new Error("Discord principal absent"); const intent = await service.createLinkIntent( endpoint.id, principal.id, 1800, ); const linkToken = new URL(intent.confirmationUrl).searchParams.get( "token", )!; await service.confirmIdentityLink(linkToken, linkedUserId); const onAction = vi.fn(callbacks.onAction); const onModalSubmit = vi.fn(callbacks.onModalSubmit); const configuration = runtime.configurations.get(endpoint.id)!; pinned = createChatSdkEndpointRuntime({ ...configuration, callbacks: { onMessage() {}, onAction, onModalSubmit }, enableDiscordGateway: false, logger: "silent", }); await pinned.initialize(); const adapter = pinned.getProviderAdapter() as unknown as { handleGatewayInteraction(input: unknown): Promise; fetchMessage(...args: unknown[]): Promise; buildMessagePayload(input: unknown): { payload: Record; }; }; // Provider HTTP/socket are the only transport doubles; modal source, // typed callback, token authorization, answers and wake ledger are real. adapter.fetchMessage = vi.fn().mockResolvedValue(null); const sourceRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: sourceRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: { issueId: conversation.issueId, taskId: conversation.issueId, source: "automation", }, }); const interaction = await issueThreadInteractionService(db).create( { id: conversation.issueId, companyId: fixture.companyId }, { kind: "ask_user_questions", continuationPolicy: "wake_assignee", sourceRunId, title: "Deployment details", payload: { version: 1, title: "Deployment details", questions: [ { id: "environment", prompt: "Environment", selectionMode: "single", required: true, allowOther: false, options: [ { id: "staging", label: "Staging" }, { id: "production", label: "Production" }, ], }, { id: "note", prompt: "Release note", selectionMode: "single", required: true, allowOther: true, options: [ { id: "__paperclip_text__", label: "Type an answer", freeText: true, }, ], }, ], questionSet: { schema: "paperclip.question_set.v1", title: "Deployment details", questions: [ { id: "environment", prompt: "Environment", required: true, answerMode: "single_select", options: [ { id: "staging", label: "Staging" }, { id: "production", label: "Production" }, ], }, { id: "note", prompt: "Release note", required: true, answerMode: "text", textValidation: { minLength: 3, maxLength: 4000 }, }, ], }, }, }, { agentId: fixture.assignedAgentId, runId: sourceRunId }, ); await service.processPendingPublications(1000); const publication = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), eq(chatPublications.issueId, conversation.issueId), ), ) .then((rows) => rows.find((row) => row.payload.interactionId === interaction.id), ); if (!publication?.providerMessageId) throw new Error("Discord question not published"); const open = publication.payload.card?.actions?.find( (action) => action.type === "callback" && action.actionId.startsWith("pcf:"), ); if (!open || open.type !== "callback") throw new Error("Native Discord form was not projected"); type NativeModal = { custom_id: string; title: string; components: Array<{ type: number; label: string; component: { type: number; custom_id: string; max_length?: number; value?: string; options?: Array<{ label: string; value: string; default?: boolean; }>; }; }>; }; const click = ( customId: string, id: string, message = publication.providerMessageId!, ) => ({ applicationId: "123456789012345678", channel: { id: rootMessageId, parentId: channelId, type: 11 }, channelId: rootMessageId, componentType: 2, customId, guildId, id, type: 3, version: 1, isChatInputCommand: () => false, isMessageComponent: () => true, isModalSubmit: () => false, message: { id: message }, token: "synthetic-modal-token-never-persist", user: { id: externalUserId, username: "operator", globalName: "Operator", bot: false, }, showModal: vi .fn<(modal: NativeModal) => Promise>() .mockResolvedValue(undefined), deferUpdate: vi.fn().mockResolvedValue(undefined), reply: vi.fn().mockResolvedValue(undefined), }); const first = click( `${open.actionId}\n${interaction.id}`, "777777777777777720", ); wakeup.mockClear(); await adapter.handleGatewayInteraction(first); expect(first.showModal).toHaveBeenCalledOnce(); expect(first.deferUpdate).not.toHaveBeenCalled(); const duplicateOpen = click( `${open.actionId}\n${interaction.id}`, first.id, ); await adapter.handleGatewayInteraction(duplicateOpen); expect(duplicateOpen.showModal).not.toHaveBeenCalled(); expect(duplicateOpen.deferUpdate).not.toHaveBeenCalled(); expect(duplicateOpen.reply).not.toHaveBeenCalled(); const uncertainOpen = click( `${open.actionId}\n${interaction.id}`, "777777777777777729", ); uncertainOpen.showModal.mockRejectedValue( new Error("synthetic lost modal response"), ); await adapter.handleGatewayInteraction(uncertainOpen); expect(uncertainOpen.showModal).toHaveBeenCalledOnce(); expect(uncertainOpen.deferUpdate).not.toHaveBeenCalled(); const uncertainReplay = click( `${open.actionId}\n${interaction.id}`, uncertainOpen.id, ); await adapter.handleGatewayInteraction(uncertainReplay); expect(uncertainReplay.showModal).not.toHaveBeenCalled(); expect(uncertainReplay.deferUpdate).not.toHaveBeenCalled(); expect(uncertainReplay.reply).not.toHaveBeenCalled(); const modal = first.showModal.mock.calls[0]![0]; expect(modal.components).toHaveLength(2); const select = modal.components.find( (field) => field.component.type === 3, )!.component; const text = modal.components.find( (field) => field.component.type === 4, )!.component; expect(text.max_length).toBe(4000); const stagingValue = select.options!.find( (option) => option.label === "Staging", )!.value; const submission = ( id: string, note: string, customId = modal.custom_id, ) => ({ ...click(customId, id), type: 5, isMessageComponent: () => false, isModalSubmit: () => true, components: [ { type: 18, component: { type: 3, customId: select.custom_id, values: [stagingValue], }, }, { type: 18, component: { type: 4, customId: text.custom_id, value: note }, }, ], }); const invalid = submission("777777777777777721", ""); await adapter.handleGatewayInteraction(invalid); expect(onModalSubmit).toHaveBeenCalledOnce(); expect(onModalSubmit.mock.calls[0]![0].event.relatedThread?.id).toBe( threadId, ); expect(invalid.showModal).not.toHaveBeenCalled(); expect(invalid.reply).toHaveBeenCalledOnce(); const correction = invalid.reply.mock.calls[0]![0] as { content: string; flags: number; components: Array<{ components: Array<{ custom_id: string }> }>; }; expect(correction.content).toContain("Release note"); expect(correction.content).not.toContain(text.custom_id); expect(correction.flags).toBe(64); const reopenId = correction.components[0]!.components[0]!.custom_id; expect(reopenId).toMatch(/^pcfr:[\w-]{43}$/); const readState = () => db .select() .from(chatSdkState) .where(eq(chatSdkState.endpointId, endpoint.id)); const invalidState = await readState(); expect(JSON.stringify(invalidState)).not.toContain(first.token); expect( invalidState.filter((row) => row.stateKey.startsWith("discord-question-correction:"), ), ).toHaveLength(1); // The correction button belongs to a different ephemeral message. It // must reauthorize the original published source and current actor. await db .update(companyMemberships) .set({ membershipRole: "viewer" }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalId, linkedUserId), ), ); const denied = click( reopenId, "777777777777777722", "888888888888888888", ); await adapter.handleGatewayInteraction(denied); expect(denied.showModal).not.toHaveBeenCalled(); expect(denied.deferUpdate).not.toHaveBeenCalled(); expect(denied.reply).toHaveBeenCalledWith( expect.objectContaining({ flags: 64, content: expect.stringContaining("no longer available"), }), ); await db .update(companyMemberships) .set({ membershipRole: "operator" }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalId, linkedUserId), ), ); const wrongThread = click( reopenId, "777777777777777730", "888888888888888888", ); wrongThread.channelId = "555555555555555999"; wrongThread.channel.id = wrongThread.channelId; await adapter.handleGatewayInteraction(wrongThread); expect(wrongThread.showModal).not.toHaveBeenCalled(); expect(wrongThread.deferUpdate).not.toHaveBeenCalled(); const wrongActor = click( reopenId, "777777777777777731", "888888888888888888", ); wrongActor.user.id = "444444444444444999"; await adapter.handleGatewayInteraction(wrongActor); expect(wrongActor.showModal).not.toHaveBeenCalled(); await db .update(chatPublications) .set({ providerMessageId: "999999999999999999" }) .where(eq(chatPublications.id, publication.id)); const changedSource = click( reopenId, "777777777777777732", "888888888888888888", ); await adapter.handleGatewayInteraction(changedSource); expect(changedSource.showModal).not.toHaveBeenCalled(); expect(changedSource.deferUpdate).not.toHaveBeenCalled(); await db .update(chatPublications) .set({ providerMessageId: publication.providerMessageId }) .where(eq(chatPublications.id, publication.id)); const reopened = click( reopenId, "777777777777777723", "888888888888888888", ); await adapter.handleGatewayInteraction(reopened); expect(reopened.showModal).toHaveBeenCalledOnce(); expect(reopened.deferUpdate).not.toHaveBeenCalled(); const edited = reopened.showModal.mock.calls[0]![0]; expect( edited.components[0]!.component.options?.find( (option) => option.default, )?.value, ).toBe(stagingValue); expect(edited.components[1]!.component.value).toBe(""); expect(edited.custom_id.split(":").slice(0, 2).join(":")).toBe( modal.custom_id.split(":").slice(0, 2).join(":"), ); const corrected = submission( "777777777777777724", "Ship safely", edited.custom_id, ); corrected.message.id = "888888888888888888"; const competingOpen = click( `${open.actionId}\n${interaction.id}`, "777777777777777734", ); await adapter.handleGatewayInteraction(competingOpen); expect(competingOpen.showModal).toHaveBeenCalledOnce(); const competingText = concurrentMode === "identical" ? "Ship safely" : "Ship Friday"; const competing = submission( "777777777777777735", competingText, competingOpen.showModal.mock.calls[0]![0].custom_id, ); cleanupSpy = vi .spyOn(discordQuestionForms, "deleteDiscordQuestionFormCorrection") .mockRejectedValueOnce(new Error("synthetic draft cleanup failure")); holdConcurrentAnswers = true; await Promise.all([ adapter.handleGatewayInteraction(corrected), adapter.handleGatewayInteraction(competing), ]); expect(answerWaiters).toBe(2); expect(cleanupSpy).toHaveBeenCalledTimes( concurrentMode === "identical" ? 2 : 1, ); const [answered] = await db .select() .from(issueThreadInteractions) .where(eq(issueThreadInteractions.id, interaction.id)); const winnerText = ( answered!.result as { answers: Array<{ questionId: string; otherText?: string }>; } ).answers.find((answer) => answer.questionId === "note")!.otherText!; expect(["Ship safely", competingText]).toContain(winnerText); for (const [callback, submittedText] of [ [corrected, "Ship safely"], [competing, competingText], ] as const) { expect(callback.reply).toHaveBeenCalledOnce(); expect(callback.reply.mock.calls[0]![0].content).toBe( submittedText === winnerText ? "Your response was received." : "This response was not accepted. Open the linked Paperclip task or reopen the question to try again.", ); } expect(answered).toMatchObject({ status: "answered", resolvedByUserId: linkedUserId, result: { answers: [ { questionId: "environment", optionIds: ["staging"] }, { questionId: "note", optionIds: [], otherText: winnerText }, ], }, }); await vi.waitFor(async () => { const [response] = await db .select() .from(issueQuestionResponseDeliveries) .where( eq(issueQuestionResponseDeliveries.interactionId, interaction.id), ); expect(response?.status).toBe("fallback_queued"); }); const continuationWakeCount = wakeup.mock.calls.filter( ([, options]) => options.contextSnapshot?.source === "issue.interaction.respond", ).length; expect(continuationWakeCount).toBe(1); expect( (await readState()).filter((row) => row.stateKey.startsWith("discord-question-correction:"), ), ).toHaveLength(concurrentMode === "identical" ? 0 : 1); const duplicate = submission( "777777777777777725", winnerText, edited.custom_id, ); await adapter.handleGatewayInteraction(duplicate); expect( onModalSubmit.mock.calls.at(-1)![0].event.relatedThread, ).toBeUndefined(); expect(duplicate.reply).toHaveBeenCalledWith( expect.objectContaining({ content: "Your response was received." }), ); expect( (await readState()).filter((row) => row.stateKey.startsWith("discord-question-correction:"), ), ).toHaveLength(0); expect( wakeup.mock.calls.filter( ([, options]) => options.contextSnapshot?.source === "issue.interaction.respond", ), ).toHaveLength(1); const differentAnswer = submission( "777777777777777733", "Ship something else", edited.custom_id, ); await adapter.handleGatewayInteraction(differentAnswer); expect(differentAnswer.reply).toHaveBeenCalledWith( expect.objectContaining({ content: expect.stringContaining("not accepted"), flags: 64, }), ); expect(differentAnswer.reply.mock.calls[0]![0]).not.toHaveProperty( "components", ); expect( wakeup.mock.calls.filter( ([, options]) => options.contextSnapshot?.source === "issue.interaction.respond", ), ).toHaveLength(1); const stale = click( reopenId, "777777777777777726", "888888888888888888", ); await adapter.handleGatewayInteraction(stale); expect(stale.showModal).not.toHaveBeenCalled(); expect(stale.deferUpdate).not.toHaveBeenCalled(); const unknown = submission( "777777777777777727", "Ship safely", `pcfs:${"Z".repeat(22)}:${randomUUID()}`, ); await adapter.handleGatewayInteraction(unknown); expect(unknown.reply).toHaveBeenCalledWith( expect.objectContaining({ content: expect.stringContaining("not accepted"), flags: 64, }), ); expect(unknown.reply.mock.calls[0]![0]).not.toHaveProperty( "components", ); // Scheduler is simulated: this proves wake_fallback exactly once, not a // model turn or a live Discord login/modal interaction. } finally { releaseAnswers(); cleanupSpy?.mockRestore(); try { await pinned?.shutdown(); } finally { await retirePublicationFixture(service, endpoint.id); } } }, ); it.each([ "warm", "cold", "failed initialization", "stale generation", "disabled connection", ])( "automatically enables native modals for an existing Discord endpoint after %s runtime qualification", async (mode) => { const fixture = await seedCompany(); const configured = await configuredDiscordEndpoint(fixture); const { endpoint } = configured; let { runtime, service } = configured; try { const [current] = await db .select() .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)); await db .update(chatEndpoints) .set({ capabilities: { ...current!.capabilities, modals: false } }) .where(eq(chatEndpoints.id, endpoint.id)); const [before] = await db .select() .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)); const [connectionBefore] = await db .select() .from(toolConnections) .where(eq(toolConnections.id, before!.connectionId)); const resourcesBefore = await db .select() .from(chatEndpointResources) .where(eq(chatEndpointResources.endpointId, endpoint.id)); const instance = runtime.endpoints.get(endpoint.id); if (mode !== "warm") { const config = runtime.configurations.get( endpoint.id, )!.providerConfig; if (config.provider !== "discord") throw new Error("Discord configuration absent"); await service.shutdown(); ({ runtime, service } = createService( new FakeChatSdkRuntime(), fakeDiscordFetch( config.credentials.applicationId, ) as typeof globalThis.fetch, )); } let targetInitializations = 0; if (mode === "failed initialization") runtime.initializeHook = async (initializingEndpointId) => { if (initializingEndpointId !== endpoint.id) return; targetInitializations += 1; throw new Error("synthetic initialization failed"); }; if (mode === "stale generation") runtime.initializeHook = async (initializingEndpointId) => { if (initializingEndpointId !== endpoint.id) return; targetInitializations += 1; await db .update(chatEndpoints) .set({ setup: { ...before!.setup, runtimeGeneration: Number(before!.setup.runtimeGeneration ?? 0) + 1, }, }) .where(eq(chatEndpoints.id, endpoint.id)); }; if (mode === "disabled connection") runtime.initializeHook = async (initializingEndpointId) => { if (initializingEndpointId !== endpoint.id) return; targetInitializations += 1; await db .update(toolConnections) .set({ enabled: false }) .where(eq(toolConnections.id, before!.connectionId)); }; await service.reconcileProviderRuntimes(); if (mode !== "warm" && mode !== "cold") expect(targetInitializations).toBe(1); const [after] = await db .select() .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)); expect(after!.capabilities.modals).toBe( mode === "warm" || mode === "cold", ); expect({ ...after, updatedAt: before!.updatedAt, capabilities: before!.capabilities, ...(mode === "stale generation" ? { setup: before!.setup } : {}), }).toEqual(before); const [connectionAfter] = await db .select() .from(toolConnections) .where(eq(toolConnections.id, before!.connectionId)); expect({ ...connectionAfter, updatedAt: connectionBefore!.updatedAt, ...(mode === "disabled connection" ? { enabled: connectionBefore!.enabled } : {}), }).toEqual(connectionBefore); expect( await db .select() .from(chatEndpointResources) .where(eq(chatEndpointResources.endpointId, endpoint.id)), ).toEqual(resourcesBefore); if (mode === "warm") expect(runtime.endpoints.get(endpoint.id)).toBe(instance); if (mode === "warm" || mode === "cold") expect( runtime.configurations.get(endpoint.id)!.callbacks.onModalSubmit, ).toBeTypeOf("function"); } finally { try { await service.shutdown(); } finally { await db .update(chatEndpoints) .set({ status: "paused" }) .where(eq(chatEndpoints.id, endpoint.id)); } } }, ); it.for(["retired runtime", "replaced runtime", "changed credentials"])( "does not enable Discord modals after a connection-lock wait with %s", async (mode, { signal }) => { const fixture = await seedCompany(); const { endpoint, runtime, service } = await configuredDiscordEndpoint(fixture); let reconciliation: Promise | undefined; let transactionSpy: ReturnType | undefined; let releaseModalQuery!: () => void; const modalQueryGate = new Promise((resolve) => { releaseModalQuery = resolve; }); let resolveModalQuery!: (value: { pid: number; query: string }) => void; let rejectModalQuery!: (error: unknown) => void; const modalQueryReady = new Promise<{ pid: number; query: string }>( (resolve, reject) => { resolveModalQuery = resolve; rejectModalQuery = reject; }, ); // Readiness shares the existing test deadline; a timeout must also // release the intercepted statement so reconciliation can settle. const abort = () => { rejectModalQuery(signal.reason); releaseModalQuery(); }; signal.addEventListener("abort", abort, { once: true }); try { const [before] = await db .select() .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)); await db .update(chatEndpoints) .set({ capabilities: { ...before!.capabilities, modals: false } }) .where(eq(chatEndpoints.id, endpoint.id)); const original = runtime.get(endpoint.id); expect(original).not.toBeNull(); const originalTransaction = db.transaction.bind(db); type ObservedSession = { prepareQuery(...args: unknown[]): { execute(...args: unknown[]): Promise; }; }; let capturedModalQuery = false; transactionSpy = vi.spyOn(db, "transaction").mockImplementation((async ( ...args: Parameters ) => { const [callback, config] = args; return originalTransaction(async (tx) => { const session = (tx as unknown as { session: ObservedSession }) .session; const prepareQuery = session.prepareQuery; session.prepareQuery = (...queryArgs) => { const query = queryArgs[0] as { sql: string; params: unknown[]; }; const prepared = prepareQuery.apply(session, queryArgs); const execute = prepared.execute.bind(prepared); prepared.execute = async (...executeArgs) => { if ( !capturedModalQuery && query.sql.startsWith( 'select "enabled", "status", "credential_secret_refs" from "tool_connections"', ) && query.sql.endsWith("for no key update") && query.params.length === 2 && query.params[0] === fixture.companyId && query.params[1] === before!.connectionId ) { capturedModalQuery = true; const [backend] = (await tx.execute( sql`select pg_backend_pid() as pid`, )) as unknown as Array<{ pid: number }>; resolveModalQuery({ pid: backend!.pid, query: query.sql }); await modalQueryGate; signal.throwIfAborted(); } return execute(...executeArgs); }; return prepared; }; try { return await callback(tx); } finally { session.prepareQuery = prepareQuery; } }, config); }) as typeof db.transaction); // Do not hold the connection during the preceding database-wide // runtime/command sweep: command authorization locks it too. Stop // only at the exact modal-upgrade query after runtime qualification. reconciliation = service.reconcileProviderRuntimes(); void reconciliation.then( () => rejectModalQuery(new Error("Modal upgrade query was not reached")), rejectModalQuery, ); const modalQuery = await modalQueryReady; await db.transaction(async (tx) => { await tx .select() .from(toolConnections) .where(eq(toolConnections.id, before!.connectionId)) .for("no key update"); const [backend] = (await tx.execute( sql`select pg_backend_pid() as pid`, )) as unknown as Array<{ pid: number }>; expect(modalQuery.pid).not.toBe(backend!.pid); releaseModalQuery(); await vi.waitFor(async () => { const [state] = (await db.execute(sql`select exists ( select 1 from pg_stat_activity where pid = ${modalQuery.pid} and datname = current_database() and ${backend!.pid} = any(pg_blocking_pids(pid)) and query = ${modalQuery.query} ) as waiting`)) as unknown as Array<{ waiting: boolean }>; expect(state!.waiting).toBe(true); }); // The actual upgrade has already qualified this runtime and now // waits for this exact connection lock. No provider call is mocked. expect(runtime.get(endpoint.id)).toBe(original); if (mode === "changed credentials") { await tx .update(toolConnections) .set({ credentialSecretRefs: [] }) .where(eq(toolConnections.id, before!.connectionId)); } else { await runtime.removeEndpoint(endpoint.id); if (mode === "replaced runtime") { await runtime.replaceEndpoint( runtime.configurations.get(endpoint.id)!, ); expect(runtime.get(endpoint.id)).not.toBe(original); } } }); await reconciliation; const [after] = await db .select() .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)); expect(after!.capabilities.modals).toBe(false); expect({ ...after, updatedAt: before!.updatedAt, capabilities: before!.capabilities, }).toEqual(before); } finally { releaseModalQuery(); await reconciliation?.catch(() => undefined); transactionSpy?.mockRestore(); signal.removeEventListener("abort", abort); await retirePublicationFixture(service, endpoint.id); } }, ); it("orders rapid Discord replies by provider time before waking one task", async () => { const fixture = await seedCompany(); const deferred: Array<() => void> = []; const { callbacks, endpoint, service, wakeup } = await configuredDiscordEndpoint(fixture, { deferWebhookProcessing: true, scheduleDeferredWork: (task) => deferred.push(task), }); try { const guildId = "1457808928258658549"; const channelId = "333333333333333333"; const rootMessageId = "555555555555555620"; const threadId = `discord:${guildId}:${channelId}:${rootMessageId}`; const channel = makeThread({ channelId, id: threadId, name: "discord-rapid-replies", }); const admitRootMention = callbacks.onDiscordRootMentionAdmission; if (!admitRootMention) { throw new Error("Expected Discord root-mention admission callback"); } await expect( admitRootMention({ endpointId: endpoint.id, guildId, channelId, messageId: rootMessageId, message: { ...makeMessage({ id: rootMessageId, text: "@maya create a queued reply test", mentioned: true, userId: "444444444444444420", }), threadId, } as Message, threadId, userId: "444444444444444420", }), ).resolves.toBe(false); const [rootDelivery] = await db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.providerEventId, `${threadId}:${rootMessageId}`), ), ); if (!rootDelivery) throw new Error("Expected Discord root delivery"); await service.processPendingDeliveries(25, rootDelivery.id); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Discord conversation"); await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, endpoint.id)); wakeup.mockClear(); deferred.length = 0; const providerSentAt = new Date("2026-09-06T22:30:00.000Z"); const laterReply = makeMessage({ id: "555555555555555622", text: "second rapid Discord reply", userId: "444444444444444420", }); laterReply.metadata.dateSent = providerSentAt; const earlierReply = makeMessage({ id: "555555555555555621", text: "first rapid Discord reply", userId: "444444444444444420", }); earlierReply.metadata.dateSent = providerSentAt; await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "discord", thread: channel.thread, message: laterReply, trigger: "subscribed_message", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "discord", thread: channel.thread, message: earlierReply, trigger: "subscribed_message", }); const rapidIds = [ `${threadId}:${earlierReply.id}`, `${threadId}:${laterReply.id}`, ]; const rapidDeliveries = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), inArray(chatDeliveries.providerEventId, rapidIds), ), ); expect(rapidDeliveries).toHaveLength(2); expect(rapidDeliveries.every((row) => row.state === "received")).toBe( true, ); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(inArray(chatDeliveries.providerEventId, rapidIds)); await service.processPendingDeliveries(1_000); const comments = await db .select({ id: issueComments.id, body: issueComments.body }) .from(issueComments) .where( and( eq(issueComments.issueId, conversation.issueId), inArray(issueComments.body, [earlierReply.text, laterReply.text]), ), ) .orderBy(asc(issueComments.createdAt), asc(issueComments.id)); expect(comments.map((comment) => comment.body)).toEqual([ earlierReply.text, laterReply.text, ]); expect(wakeup).toHaveBeenCalledTimes(2); expect( wakeup.mock.calls.map((call) => call[1]?.payload?.wakeCommentId), ).toEqual(comments.map((comment) => comment.id)); } finally { await service.shutdown(); } }); it("delivers oversized Discord Markdown losslessly as one retryable attachment", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredDiscordEndpoint(fixture); const guildId = "1457808928258658549"; const channelId = "333333333333333333"; const messageId = "555555555555555599"; const threadId = `discord:${guildId}:${channelId}:${messageId}`; const message = { ...makeMessage({ id: messageId, text: "@maya send the complete long result", mentioned: true, userId: "444444444444444444", }), threadId, } as Message; const admitRootMention = callbacks.onDiscordRootMentionAdmission; if (!admitRootMention) throw new Error("Expected Discord root-mention admission callback"); await expect( admitRootMention({ endpointId: endpoint.id, guildId, channelId, messageId, message, threadId, userId: "444444444444444444", }), ).resolves.toBe(false); const [delivery] = await db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.providerEventId, `${threadId}:${messageId}`), ), ); if (!delivery) throw new Error("Expected Discord root delivery"); await service.processPendingDeliveries(25, delivery.id); await qualifySetupRoundTrip(service, endpoint.id, "444444444444444444"); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Discord conversation"); const source = [ "## Complete result 🙂", "[Open the evidence](https://example.test/evidence?case=discord)", `\`\`\`ts\n${"const value = 1;\n".repeat(80)}\`\`\``, "@alice ".repeat(180).trim(), ].join("\n\n"); const providerSafeSource = projectSafeChatPublicationText(source); const comment = await issueService(db).addComment( conversation.issueId, source, { userId: "owner-user" }, { authorType: "user" }, ); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Discord provider runtime"); providerRuntime.posts.length = 0; let transportAttempt = 0; providerRuntime.postHook = async () => { transportAttempt += 1; if (transportAttempt === 1) { throw Object.assign(new Error("Discord attachment rate limited"), { adapter: "discord", status: 429, retryAfterMs: 1_000, }); } }; const blocked = await service.publishComment( endpoint.id, conversation.id, comment.id, ); expect(blocked).toMatchObject({ state: "retry" }); const [afterFailure] = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, comment.id)); expect(afterFailure).toMatchObject({ state: "retry", attempts: 1, payload: { text: providerSafeSource, transportPart: { count: 1, index: 0, mode: "discord_markdown_attachment", }, }, }); expect(providerRuntime.posts).toHaveLength(0); providerRuntime.postHook = undefined; await db .update(chatPublications) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatPublications.id, afterFailure.id)); await service.processPendingPublications(); const [completed] = await db .select() .from(chatPublications) .where(eq(chatPublications.id, afterFailure.id)); expect(completed).toMatchObject({ state: "published", attempts: 2 }); expect(providerRuntime.posts).toHaveLength(1); const delivered = providerRuntime.posts[0]!; expect(delivered.text).toBe( "Paperclip attached the complete response because it exceeds Discord’s message limit.", ); expect(delivered.text).not.toContain("..."); expect(delivered.files).toHaveLength(1); const uploaded = delivered.files?.[0] as { data: Buffer; filename: string; mimeType: string; }; expect(uploaded.filename).toBe("paperclip-response.md"); expect(uploaded.mimeType).toBe("text/markdown; charset=utf-8"); expect(Buffer.isBuffer(uploaded.data)).toBe(true); // The attachment is lossless after Paperclip's mandatory provider-safety // projection (which strips URL query strings before any transport work). expect(uploaded.data.toString("utf8")).toBe(providerSafeSource); expect(uploaded.data.toString("utf8")).not.toContain("?case=discord"); providerRuntime.posts.length = 0; providerRuntime.edits.length = 0; providerRuntime.editAttempts.length = 0; const discordLongRunId = randomUUID(); const replacementCreatedAt = new Date(); await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${discordLongRunId}:working:${endpoint.id}`, payload: { text: "Working…", progressState: "working" }, state: "published", providerMessageId: "working-message-1", createdAt: replacementCreatedAt, updatedAt: replacementCreatedAt, }); const [replacementPublication] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${discordLongRunId}:completed:${endpoint.id}`, payload: { text: providerSafeSource, progressState: "completed" }, state: "pending", createdAt: new Date(replacementCreatedAt.getTime() + 1), updatedAt: new Date(replacementCreatedAt.getTime() + 1), }) .returning(); await service.processPendingPublications(); const replacementBatch = await db .select() .from(chatPublications) .where( sql`${chatPublications.payload}->'transportPart'->>'batchId' = ${replacementPublication.id}`, ) .orderBy( sql`(${chatPublications.payload}->'transportPart'->>'index')::int`, ); expect(replacementBatch).toHaveLength(2); expect(replacementBatch.map((row) => row.state)).toEqual([ "published", "published", ]); expect( replacementBatch.map((row) => row.payload.transportPart?.mode), ).toEqual(["inline", "discord_markdown_attachment"]); expect(providerRuntime.editAttempts).toEqual([ { threadId, messageId: "working-message-1", }, ]); expect(providerRuntime.edits[0]?.text).toBe( "This response needs a separate attachment because it exceeds the message limit.", ); expect(providerRuntime.posts).toHaveLength(1); expect(providerRuntime.posts[0]?.text).toBe("Complete response attached."); const replacementUpload = providerRuntime.posts[0]?.files?.[0] as { data: Buffer; }; expect(replacementUpload.data.toString("utf8")).toBe(providerSafeSource); providerRuntime.posts.length = 0; providerRuntime.edits.length = 0; providerRuntime.editAttempts.length = 0; const rejectedRunId = randomUUID(); const rejectedCreatedAt = new Date(); await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${rejectedRunId}:working:${endpoint.id}`, payload: { text: "Working on the attachment…", progressState: "working" }, state: "published", providerMessageId: "working-message-rejected-attachment", createdAt: rejectedCreatedAt, updatedAt: rejectedCreatedAt, }); const [rejectedPublication] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${rejectedRunId}:completed:${endpoint.id}`, payload: { text: providerSafeSource, progressState: "completed" }, state: "pending", createdAt: new Date(rejectedCreatedAt.getTime() + 1), updatedAt: new Date(rejectedCreatedAt.getTime() + 1), }) .returning(); let rejectAttachmentOnce = true; providerRuntime.postHook = async () => { if (!rejectAttachmentOnce) return; rejectAttachmentOnce = false; throw Object.assign(new Error("Discord rejected the attachment"), { adapter: "discord", response: { status: 400 }, status: 400, }); }; await service.processPendingPublications(); providerRuntime.postHook = undefined; const rejectedBatch = await db .select() .from(chatPublications) .where( sql`${chatPublications.payload}->'transportPart'->>'batchId' = ${rejectedPublication.id}`, ) .orderBy( sql`(${chatPublications.payload}->'transportPart'->>'index')::int`, ); expect(rejectedBatch.map((row) => row.state)).toEqual([ "published", "failed", ]); const definiteFailureNotices = await db .select() .from(chatPublications) .where( and( eq(chatPublications.companyId, fixture.companyId), like( chatPublications.idempotencyKey, `attachment-failure-notice:${rejectedBatch[1]!.id}:%`, ), ), ); expect(definiteFailureNotices).toHaveLength(1); const rejectedAttachmentNotice = "Paperclip could not send the response attachment. The complete response remains on its Paperclip task for an operator to retry." + ` Open task: https://paperclip.example/issues/${conversation.issueId}`; expect(definiteFailureNotices[0]).toMatchObject({ commentId: null, state: "published", payload: { text: rejectedAttachmentNotice, }, }); expect(providerRuntime.editAttempts).toEqual([ { threadId, messageId: "working-message-rejected-attachment", }, ]); expect(providerRuntime.edits[0]?.text).toBe( "This response needs a separate attachment because it exceeds the message limit.", ); expect(providerRuntime.posts).toEqual([ { threadId, text: rejectedAttachmentNotice, }, ]); await service.processPendingPublications(); expect(providerRuntime.posts).toHaveLength(1); const staleNoticeId = randomUUID(); await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `attachment-failure-notice:${staleNoticeId}:999:${"a".repeat(64)}`, payload: { text: "stale attachment failure notice" }, state: "pending", }); await service.processPendingPublications(); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `attachment-failure-notice:${staleNoticeId}:999:${"a".repeat(64)}`, ), ), ).resolves.toEqual([{ state: "cancelled" }]); expect(providerRuntime.posts).toHaveLength(1); const cardKey = `discord-card-not-split:${endpoint.id}`; await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: cardKey, payload: { text: source, card: { schema: "paperclip.chat.card.v1", kind: "status", title: "Long structured card", }, }, state: "pending", }); await service.processPendingPublications(); await expect( db .select({ payload: chatPublications.payload }) .from(chatPublications) .where(like(chatPublications.idempotencyKey, `${cardKey}%`)), ).resolves.toEqual([ { payload: expect.not.objectContaining({ transportPart: expect.anything(), }), }, ]); expect(providerRuntime.posts.at(-1)?.files).toBeUndefined(); const ambiguousRunId = randomUUID(); const ambiguousCreatedAt = new Date(); await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${ambiguousRunId}:working:${endpoint.id}`, payload: { text: "Working on another attachment…", progressState: "working", }, state: "published", providerMessageId: "working-message-ambiguous-attachment", createdAt: ambiguousCreatedAt, updatedAt: ambiguousCreatedAt, }); const [ambiguousPublication] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${ambiguousRunId}:completed:${endpoint.id}`, payload: { text: providerSafeSource, progressState: "completed" }, state: "pending", createdAt: new Date(ambiguousCreatedAt.getTime() + 1), updatedAt: new Date(ambiguousCreatedAt.getTime() + 1), }) .returning(); providerRuntime.postHook = async () => { throw Object.assign(new Error("Discord attachment response was lost"), { adapter: "discord", name: "NetworkError", }); }; await service.processPendingPublications(); providerRuntime.postHook = undefined; await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where( sql`${chatPublications.payload}->'transportPart'->>'batchId' = ${ambiguousPublication.id}`, ) .orderBy( sql`(${chatPublications.payload}->'transportPart'->>'index')::int`, ), ).resolves.toEqual([{ state: "published" }, { state: "delivery_unknown" }]); await expect( db .select({ id: chatPublications.id }) .from(chatPublications) .where( and( eq(chatPublications.companyId, fixture.companyId), eq(chatPublications.state, "published"), like( chatPublications.idempotencyKey, "attachment-failure-notice:%", ), ), ), ).resolves.toHaveLength(1); await service.shutdown(); }); it("isolates malformed publication preparation so later messages still deliver", async () => { const fixture = await seedCompany(); let transientPublicationId: string | null = null; let failTransientPreparation = true; const { callbacks, endpoint, runtime, service } = await configuredDiscordEndpoint(fixture, { publicationTransportPreparationBarrier: async ({ publicationId }) => { if ( publicationId === transientPublicationId && failTransientPreparation ) { failTransientPreparation = false; throw Object.assign(new Error("temporary preparation outage"), { name: "NetworkError", }); } }, }); const guildId = "1457808928258658549"; // Match the text channel returned by fakeDiscordFetch; an unknown channel // is correctly filtered before setup can activate its first destination. const channelId = "333333333333333333"; const messageId = "555555555555555600"; const threadId = `discord:${guildId}:${channelId}:${messageId}`; const admitRootMention = callbacks.onDiscordRootMentionAdmission; if (!admitRootMention) throw new Error("Expected Discord root-mention admission callback"); await expect( admitRootMention({ endpointId: endpoint.id, guildId, channelId, messageId, message: { ...makeMessage({ id: messageId, text: "@maya verify publication queue isolation", mentioned: true, userId: "444444444444444444", }), threadId, } as Message, threadId, userId: "444444444444444444", }), ).resolves.toBe(false); const [delivery] = await db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.providerEventId, `${threadId}:${messageId}`), ), ); if (!delivery) throw new Error("Expected Discord root delivery"); await service.processPendingDeliveries(25, delivery.id); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Discord conversation"); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Discord provider runtime"); providerRuntime.posts.length = 0; const createdAt = new Date(); const [malformed, healthy] = await db .insert(chatPublications) .values([ { companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `malformed-publication:${randomUUID()}`, payload: {} as never, state: "pending" as const, createdAt, updatedAt: createdAt, }, { companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `healthy-after-malformed:${randomUUID()}`, payload: { text: "Healthy publication after malformed durable row" }, state: "pending" as const, createdAt: new Date(createdAt.getTime() + 1), updatedAt: new Date(createdAt.getTime() + 1), }, ]) .returning(); await expect(service.processPendingPublications()).resolves.toBe(2); await expect( db .select({ id: chatPublications.id, redactedError: chatPublications.redactedError, state: chatPublications.state, }) .from(chatPublications) .where(inArray(chatPublications.id, [malformed.id, healthy.id])) .orderBy(asc(chatPublications.createdAt)), ).resolves.toEqual([ { id: malformed.id, redactedError: "Publication payload could not be prepared for provider delivery", state: "failed", }, { id: healthy.id, redactedError: null, state: "published" }, ]); expect(providerRuntime.posts).toHaveLength(1); expect(providerRuntime.posts[0]?.text).toBe( "Healthy publication after malformed durable row", ); const [transient] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `transient-publication:${randomUUID()}`, payload: { text: "Healthy publication after a transient outage" }, state: "pending", }) .returning(); transientPublicationId = transient.id; await expect(service.processPendingPublications()).resolves.toBe(1); await expect( db .select({ attempts: chatPublications.attempts, state: chatPublications.state, }) .from(chatPublications) .where(eq(chatPublications.id, transient.id)), ).resolves.toEqual([{ attempts: 1, state: "retry" }]); expect(providerRuntime.posts).toHaveLength(1); await db .update(chatPublications) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatPublications.id, transient.id)); await expect(service.processPendingPublications()).resolves.toBe(1); await expect( db .select({ attempts: chatPublications.attempts, state: chatPublications.state, }) .from(chatPublications) .where(eq(chatPublications.id, transient.id)), ).resolves.toEqual([{ attempts: 2, state: "published" }]); expect(providerRuntime.posts.at(-1)?.text).toBe( "Healthy publication after a transient outage", ); await service.shutdown(); }); it.each([ { label: "prose", source: `${"The cat rests beside the window in warm light. ".repeat(55)}Done.`, }, { label: "formatted code", source: `\`\`\`ts\n${"const ready = true;\n".repeat(110)}\`\`\``, }, ])( "keeps medium Telegram $label in one native publication", async ({ source }) => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture); const chatId = "77118846"; const dm = makeThread({ channelId: chatId, id: `telegram:${chatId}`, isDM: true, name: "Telegram intact output", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: `${chatId}:1`, text: "Send the complete result", userId: chatId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, chatId); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Telegram intact conversation"); const comment = await issueService(db).addComment( conversation.issueId, source, { userId: "owner-user" }, { authorType: "user" }, ); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Telegram provider runtime"); providerRuntime.posts.length = 0; const publication = await service.publishComment( endpoint.id, conversation.id, comment.id, ); expect(publication).toMatchObject({ state: "published", attempts: 1 }); const rows = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, comment.id)); expect(rows).toHaveLength(1); expect(rows[0]?.payload.transportPart).toBeUndefined(); expect(providerRuntime.posts).toHaveLength(1); expect(providerRuntime.posts[0]?.text).toBe(source); expect(providerRuntime.posts[0]?.attachments ?? []).toHaveLength(0); await service.processPendingPublications(); expect(providerRuntime.posts).toHaveLength(1); }, ); it("segments long Telegram output into durable FIFO publications and resumes at the failed part", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture); const chatId = "77118844"; const dm = makeThread({ channelId: chatId, id: `telegram:${chatId}`, isDM: true, name: "Telegram segmented output", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: `${chatId}:1`, text: "Send the complete long result", userId: chatId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, chatId); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Telegram segmented conversation"); const source = Array.from({ length: 5_003 }, (_value, index) => String(index % 10), ).join(""); const comment = await issueService(db).addComment( conversation.issueId, source, { userId: "owner-user" }, { authorType: "user" }, ); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Telegram provider runtime"); providerRuntime.posts.length = 0; let transportAttempt = 0; providerRuntime.postHook = async () => { transportAttempt += 1; if (transportAttempt === 2) { throw Object.assign(new Error("Telegram part rate limited"), { status: 429, retryAfterMs: 1_000, }); } }; const blocked = await service.publishComment( endpoint.id, conversation.id, comment.id, ); expect(blocked).toMatchObject({ state: "retry" }); const afterFailure = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, comment.id)) .orderBy( sql`(${chatPublications.payload}->'transportPart'->>'index')::int`, ); expect(afterFailure.map((row) => row.payload.transportPart?.index)).toEqual( [0, 1, 2, 3], ); expect(afterFailure.map((row) => row.state)).toEqual([ "published", "retry", "pending", "pending", ]); expect(providerRuntime.posts.map((post) => post.text).join("")).toBe( source.slice(0, 1_600), ); providerRuntime.postHook = undefined; await db .update(chatPublications) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatPublications.id, blocked.id)); await service.processPendingPublications(); const completed = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, comment.id)) .orderBy( sql`(${chatPublications.payload}->'transportPart'->>'index')::int`, ); expect(completed.map((row) => row.state)).toEqual([ "published", "published", "published", "published", ]); expect(completed.map((row) => row.attempts)).toEqual([1, 2, 1, 1]); expect(providerRuntime.posts.map((post) => post.text).join("")).toBe( source, ); expect( providerRuntime.posts.every( (post) => Array.from(post.text).length <= 1_600, ), ).toBe(true); }); it("publishes long structured Telegram Markdown as one durable lossless document", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture); const chatId = "77118845"; const dm = makeThread({ channelId: chatId, id: `telegram:${chatId}`, isDM: true, name: "Telegram structured output", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: `${chatId}:1`, text: "Send the complete formatted result", userId: chatId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, chatId); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Telegram structured conversation"); const source = [ "## Complete result", "[Open the evidence](https://example.test/evidence?case=telegram)", `\`\`\`ts\n${"const value = 1;\n".repeat(250)}\`\`\``, Array.from({ length: 100 }, (_value, index) => `- Finding ${index}`).join( "\n", ), ].join("\n\n"); const providerSafeSource = projectSafeChatPublicationText(source); const comment = await issueService(db).addComment( conversation.issueId, source, { userId: "owner-user" }, { authorType: "user" }, ); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Telegram provider runtime"); providerRuntime.posts.length = 0; let transportAttempt = 0; providerRuntime.postHook = async () => { transportAttempt += 1; if (transportAttempt === 1) { throw Object.assign(new Error("Telegram document rate limited"), { adapter: "telegram", status: 429, retryAfterMs: 1_000, }); } }; const blocked = await service.publishComment( endpoint.id, conversation.id, comment.id, ); expect(blocked).toMatchObject({ state: "retry" }); const [afterFailure] = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, comment.id)); expect(afterFailure).toMatchObject({ state: "retry", attempts: 1, payload: { text: providerSafeSource, transportPart: { count: 1, index: 0, mode: "telegram_markdown_attachment", }, }, }); expect(providerRuntime.posts).toHaveLength(0); providerRuntime.postHook = undefined; await db .update(chatPublications) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatPublications.id, afterFailure.id)); await service.processPendingPublications(); const [completed] = await db .select() .from(chatPublications) .where(eq(chatPublications.id, afterFailure.id)); expect(completed).toMatchObject({ state: "published", attempts: 2 }); expect(providerRuntime.posts).toHaveLength(1); expect(providerRuntime.posts[0]?.text).toBe( "Paperclip attached the complete response to preserve its Markdown formatting.", ); const attachment = providerRuntime.posts[0]?.attachments?.[0] as { data: Buffer; mimeType: string; name: string; size: number; type: string; }; expect(attachment).toMatchObject({ mimeType: "text/markdown; charset=utf-8", name: "paperclip-response.md", size: Buffer.byteLength(providerSafeSource), type: "file", }); expect(Buffer.isBuffer(attachment.data)).toBe(true); expect(attachment.data.toString("utf8")).toBe(providerSafeSource); expect(attachment.data.toString("utf8")).not.toContain("?case=telegram"); providerRuntime.posts.length = 0; providerRuntime.edits.length = 0; providerRuntime.editAttempts.length = 0; const runId = randomUUID(); const workingCreatedAt = new Date(); await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${runId}:working:${endpoint.id}`, payload: { text: "Maya is working…", progressState: "working" }, state: "published", providerMessageId: "telegram-working-message-1", createdAt: workingCreatedAt, updatedAt: workingCreatedAt, }); const [finalPublication] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${runId}:completed:${endpoint.id}`, payload: { text: providerSafeSource, progressState: "completed" }, state: "pending", createdAt: new Date(workingCreatedAt.getTime() + 1), updatedAt: new Date(workingCreatedAt.getTime() + 1), }) .returning(); await service.processPendingPublications(); const replacementBatch = await db .select() .from(chatPublications) .where( sql`${chatPublications.payload}->'transportPart'->>'batchId' = ${finalPublication.id}`, ) .orderBy( sql`(${chatPublications.payload}->'transportPart'->>'index')::int`, ); expect(replacementBatch).toHaveLength(2); expect(replacementBatch.map((row) => row.state)).toEqual([ "published", "published", ]); expect( replacementBatch.map((row) => row.payload.transportPart?.mode), ).toEqual(["inline", "telegram_markdown_attachment"]); expect(providerRuntime.editAttempts).toEqual([ { threadId: dm.thread.id, messageId: "telegram-working-message-1", }, ]); expect(providerRuntime.edits[0]?.text).toBe( "This response needs a separate attachment because it exceeds the message limit.", ); expect(providerRuntime.posts).toHaveLength(1); expect(providerRuntime.posts[0]?.text).toBe("Complete response attached."); const replacementAttachment = providerRuntime.posts[0] ?.attachments?.[0] as { data: Buffer }; expect(replacementAttachment.data.toString("utf8")).toBe( providerSafeSource, ); }); it.each([ { label: "rate limits as an automatic retry", error: Object.assign(new Error("provider rate limit"), { status: 429, retryAfterMs: 5_000, }), expectedPublicationState: "retry", expectedEndpointStatus: "active", expectedConversationState: "active", expectedResourceAvailability: "available", }, { label: "authentication failures as endpoint attention", error: Object.assign(new Error("provider token expired"), { status: 401, }), expectedPublicationState: "failed", expectedEndpointStatus: "attention", expectedConversationState: "active", expectedResourceAvailability: "available", }, { label: "missing destinations as resource unavailable", error: Object.assign(new Error("provider destination missing"), { status: 404, }), expectedPublicationState: "cancelled", expectedEndpointStatus: "active", expectedConversationState: "unavailable", expectedResourceAvailability: "unavailable", }, { label: "Discord missing channel permissions as resource unavailable", error: Object.assign(new Error("Discord API error: 403"), { name: "NetworkError", adapter: "discord", code: "NETWORK_ERROR", status: 403, response: { status: 403, headers: {} }, originalError: { name: "DiscordApiError", code: 50013, status: 403, }, }), expectedPublicationState: "cancelled", expectedEndpointStatus: "active", expectedConversationState: "unavailable", expectedResourceAvailability: "unavailable", }, { label: "pre-transport validation as a definite failure", error: Object.assign(new Error("untrusted Teams service URL"), { name: "TeamsServiceUrlValidationError", code: "CHAT_PROVIDER_PRETRANSPORT_REJECTED", }), expectedPublicationState: "failed", expectedEndpointStatus: "active", expectedConversationState: "active", expectedResourceAvailability: "available", }, { label: "wrapped Slack platform rejection as a definite failure", error: Object.assign(new Error("Slack block fallback failed"), { cause: Object.assign(new Error("Slack invalid_blocks"), { code: "slack_webapi_platform_error", data: { error: "invalid_blocks" }, }), }), expectedPublicationState: "failed", expectedEndpointStatus: "active", expectedConversationState: "active", expectedResourceAvailability: "available", }, ])( "classifies publication $label", async ({ error, expectedConversationState, expectedEndpointStatus, expectedPublicationState, expectedResourceAvailability, }) => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); try { const channelId = `C-PUBLICATION-ERROR-${randomUUID().slice(0, 8)}`; const channel = makeThread({ channelId, id: `slack:${channelId}:4200.1`, name: "publication-errors", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "4200.1", text: "@maya exercise provider failure handling", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const comment = await issueService(db).addComment( conversation.issueId, "Safe provider response", { userId: "owner-user" }, { authorType: "user" }, ); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected provider runtime"); providerRuntime.postError = error; const beforeAttempt = Date.now(); await service.publishComment(endpoint.id, conversation.id, comment.id); const [publication] = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, comment.id)); expect(publication).toMatchObject({ state: expectedPublicationState, attempts: 1, redactedError: error.message, }); if (expectedPublicationState === "retry") { expect(publication.nextAttemptAt?.getTime()).toBeGreaterThanOrEqual( beforeAttempt + 4_500, ); } else { expect(publication.nextAttemptAt).toBeNull(); } await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: expectedEndpointStatus, }); const [storedConversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.id, conversation.id)); expect(storedConversation.state).toBe(expectedConversationState); const [resource] = await db .select() .from(chatEndpointResources) .where(eq(chatEndpointResources.id, conversation.resourceId!)); expect(resource.availability).toBe(expectedResourceAvailability); if (expectedEndpointStatus === "attention") { expect(runtime.endpoints.has(endpoint.id)).toBe(false); const [connection] = await db .select() .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(connection).toMatchObject({ status: "disabled", enabled: false, healthStatus: "degraded", }); } } finally { await retirePublicationFixture(service, endpoint.id); } }, ); it.each([ { label: "a blocked destination", error: Object.assign( new Error("Permission denied: cannot sendMessage in telegram"), { name: "PermissionError", adapter: "telegram", code: "PERMISSION_DENIED", action: "sendMessage", }, ), expectedPublicationState: "cancelled", expectedEndpointStatus: "active", expectedConversationState: "unavailable", expectedConnection: { status: "active", enabled: true, healthStatus: "healthy", }, expectedResourceAvailability: "unavailable", runtimeRetained: true, }, { label: "an invalid bot token", error: Object.assign(new Error("Unauthorized"), { name: "AuthenticationError", adapter: "telegram", code: "AUTH_FAILED", }), expectedPublicationState: "failed", expectedEndpointStatus: "attention", expectedConversationState: "active", expectedConnection: { status: "disabled", enabled: false, healthStatus: "degraded", }, expectedResourceAvailability: "available", runtimeRetained: false, }, ])( "scopes Telegram publication failure from $label correctly", async ({ error, expectedConnection, expectedConversationState, expectedEndpointStatus, expectedPublicationState, expectedResourceAvailability, runtimeRetained, }) => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture); const chatId = `-10077${randomUUID().replaceAll("-", "").slice(0, 8)}`; const [resource] = await db .insert(chatEndpointResources) .values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "chat", providerResourceId: chatId, label: "Telegram publication errors", availability: "available", enabled: true, }) .returning(); const groupThread = makeThread({ channelId: chatId, id: `telegram:${chatId}`, name: "Telegram publication errors", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: groupThread.thread, message: makeMessage({ id: `${chatId}:1`, raw: { message_id: 1 }, text: "@paperclip create a Telegram publication failure fixture", userId: chatId, mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id, chatId); await service.test(endpoint.id, "owner-user"); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "active", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const comment = await issueService(db).addComment( conversation.issueId, "Safe Telegram provider response", { userId: "owner-user" }, { authorType: "user" }, ); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Telegram runtime"); providerRuntime.postError = error; await service.publishComment(endpoint.id, conversation.id, comment.id); const [publication] = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, comment.id)); expect(publication).toMatchObject({ state: expectedPublicationState, attempts: 1, redactedError: error.message, nextAttemptAt: null, }); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: expectedEndpointStatus, }); await expect( db .select({ state: chatConversations.state }) .from(chatConversations) .where(eq(chatConversations.id, conversation.id)), ).resolves.toEqual([{ state: expectedConversationState }]); await expect( db .select({ availability: chatEndpointResources.availability }) .from(chatEndpointResources) .where(eq(chatEndpointResources.id, resource.id)), ).resolves.toEqual([{ availability: expectedResourceAvailability }]); await expect( db .select({ status: toolConnections.status, enabled: toolConnections.enabled, healthStatus: toolConnections.healthStatus, }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)), ).resolves.toEqual([expectedConnection]); expect(runtime.endpoints.has(endpoint.id)).toBe(runtimeRetained); await service.shutdown(); }, ); it("orders a Telegram destination failure before concurrent membership recovery", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture); const chatId = "-10091234567"; const [resource] = await db .insert(chatEndpointResources) .values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "chat", providerResourceId: chatId, label: "Telegram recovery race", availability: "available", enabled: true, }) .returning(); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: makeThread({ channelId: chatId, id: `telegram:${chatId}`, name: "Telegram recovery race", }).thread, message: makeMessage({ id: `${chatId}:1`, raw: { message_id: 1 }, text: "@paperclip establish a recovery race fixture", userId: "91234567", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id, chatId); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const comment = await issueService(db).addComment( conversation.issueId, "Safe response during membership recovery", { userId: "owner-user" }, { authorType: "user" }, ); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Telegram runtime"); let markPostEntered!: () => void; let releasePost!: () => void; const postEntered = new Promise((resolve) => { markPostEntered = resolve; }); const postRelease = new Promise((resolve) => { releasePost = resolve; }); providerRuntime.postHook = async () => { markPostEntered(); await postRelease; }; providerRuntime.postError = Object.assign( new Error("Permission denied: cannot sendMessage in telegram"), { name: "PermissionError", adapter: "telegram", code: "PERMISSION_DENIED", action: "sendMessage", }, ); const publicationPromise = service.publishComment( endpoint.id, conversation.id, comment.id, ); await postEntered; const originalTransaction = db.transaction.bind(db); let recoveryPromise!: Promise; let interceptedFailureTransaction = false; let failureFinalizedWhileLeaseHeld = false; const transactionSpy = vi .spyOn(db, "transaction") .mockImplementation((async ( ...args: Parameters ) => { if (!interceptedFailureTransaction) { interceptedFailureTransaction = true; const lease = await db .select({ id: chatEndpointLeases.id }) .from(chatEndpointLeases) .where( and( eq(chatEndpointLeases.endpointId, endpoint.id), eq(chatEndpointLeases.leaseKey, "credentials"), ), ); failureFinalizedWhileLeaseHeld = lease.length === 1; // This branch simulates the pre-fix ordering deterministically: if // failure finalization escaped the lease, allow the newer provider // recovery to commit first so the stale failure would overwrite it. if (!failureFinalizedWhileLeaseHeld) await recoveryPromise; } return originalTransaction(...args); }) as typeof db.transaction); recoveryPromise = service.handleWebhook( endpoint.publicId, "telegram", new Request("https://paperclip.example/telegram", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ update_id: 91234568, my_chat_member: { chat: { id: Number(chatId), type: "supergroup", title: "Telegram recovery race", }, new_chat_member: { status: "member" }, }, }), }), ); releasePost(); try { const [, response] = await Promise.all([ publicationPromise, recoveryPromise, ]); expect(response.status).toBe(202); } finally { transactionSpy.mockRestore(); } expect(interceptedFailureTransaction).toBe(true); expect(failureFinalizedWhileLeaseHeld).toBe(true); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.commentId, comment.id)), ).resolves.toEqual([{ state: "cancelled" }]); await expect( db .select({ state: chatConversations.state }) .from(chatConversations) .where(eq(chatConversations.id, conversation.id)), ).resolves.toEqual([{ state: "active" }]); await expect( db .select({ availability: chatEndpointResources.availability }) .from(chatEndpointResources) .where(eq(chatEndpointResources.id, resource.id)), ).resolves.toEqual([{ availability: "available" }]); await service.shutdown(); }); it("orders a Telegram authentication failure before concurrent reconnect", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture); const chatId = "-10092345678"; await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "chat", providerResourceId: chatId, label: "Telegram reconnect race", availability: "available", enabled: true, }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: makeThread({ channelId: chatId, id: `telegram:${chatId}`, name: "Telegram reconnect race", }).thread, message: makeMessage({ id: `${chatId}:1`, raw: { message_id: 1 }, text: "@paperclip establish a reconnect race fixture", userId: "92345678", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id, chatId); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const comment = await issueService(db).addComment( conversation.issueId, "Safe response during credential reconnect", { userId: "owner-user" }, { authorType: "user" }, ); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Telegram runtime"); let markPostEntered!: () => void; let releasePost!: () => void; const postEntered = new Promise((resolve) => { markPostEntered = resolve; }); const postRelease = new Promise((resolve) => { releasePost = resolve; }); providerRuntime.postHook = async () => { markPostEntered(); await postRelease; }; providerRuntime.postError = Object.assign(new Error("Unauthorized"), { name: "AuthenticationError", adapter: "telegram", code: "AUTH_FAILED", }); const publicationPromise = service.publishComment( endpoint.id, conversation.id, comment.id, ); await postEntered; const originalTransaction = db.transaction.bind(db); let reconnectPromise!: ReturnType; let interceptedFailureTransaction = false; let failureFinalizedWhileLeaseHeld = false; const transactionSpy = vi .spyOn(db, "transaction") .mockImplementation((async ( ...args: Parameters ) => { if (!interceptedFailureTransaction) { interceptedFailureTransaction = true; const lease = await db .select({ id: chatEndpointLeases.id }) .from(chatEndpointLeases) .where( and( eq(chatEndpointLeases.endpointId, endpoint.id), eq(chatEndpointLeases.leaseKey, "credentials"), ), ); failureFinalizedWhileLeaseHeld = lease.length === 1; if (!failureFinalizedWhileLeaseHeld) await reconnectPromise; } return originalTransaction(...args); }) as typeof db.transaction); reconnectPromise = service.configure( endpoint.id, { action: "reconnect" }, "owner-user", ); releasePost(); try { await Promise.all([publicationPromise, reconnectPromise]); } finally { transactionSpy.mockRestore(); } expect(interceptedFailureTransaction).toBe(true); expect(failureFinalizedWhileLeaseHeld).toBe(true); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.commentId, comment.id)), ).resolves.toEqual([{ state: "failed" }]); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "verifying", healthMessage: "Waiting for a test conversation", }); await expect( db .select({ status: toolConnections.status, enabled: toolConnections.enabled, healthStatus: toolConnections.healthStatus, }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)), ).resolves.toEqual([ { status: "active", enabled: true, healthStatus: "healthy" }, ]); expect(runtime.endpoints.has(endpoint.id)).toBe(true); await service.shutdown(); }); it("publishes a closed-choice question, settles its Slack card, and delivers its exact continuation response", async () => { const fixture = await seedCompany(); const continuationRunId = randomUUID(); const resolveNativeQuestion = vi.fn(async () => "not_native" as const); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture, { resolveNativeQuestion, wakeup: async (agentId, options) => { if (options.contextSnapshot?.source !== "issue.interaction.respond") { return { accepted: true }; } const [created] = await db .insert(heartbeatRuns) .values({ id: continuationRunId, companyId: fixture.companyId, agentId, status: "succeeded", contextSnapshot: options.contextSnapshot, }) .onConflictDoNothing() .returning(); return ( created ?? db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, continuationRunId)) .then((rows) => rows[0]) ); }, }); if (!callbacks.onAction) throw new Error("Slack question action callback was not registered"); const channel = makeThread({ channelId: "C-QUESTION", id: "slack:C-QUESTION:4500.1", name: "questions", }); const externalUserId = `U-QUESTION-${randomUUID()}`; await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "4500.1", text: "@maya help me choose a priority", mentioned: true, userId: externalUserId, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const sourceRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: sourceRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", // The prompt itself is the external causal edge. A same-agent internal // run may ask in an already-bound task, and its provider answer must // return the continuation to that exact published prompt conversation. contextSnapshot: { issueId: conversation.issueId, taskId: conversation.issueId, source: "automation", }, }); const interaction = await issueThreadInteractionService(db).create( { id: conversation.issueId, companyId: fixture.companyId, }, { kind: "ask_user_questions", continuationPolicy: "wake_assignee", sourceRunId, title: "Choose the priority", payload: { version: 1, title: "Choose the priority", questions: [ { id: "priority", prompt: "Which priority should we use?", selectionMode: "single", required: true, // The shared contract defaults omission to a closed question. // Native provider buttons must match that canonical shape. options: [ { id: "high", label: "High" }, { id: "normal", label: "Normal" }, ], }, ], }, }, { agentId: fixture.assignedAgentId, runId: sourceRunId }, ); await service.processPendingPublications(); const publication = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), eq(chatPublications.issueId, conversation.issueId), ), ) .then((rows) => rows.find((row) => row.idempotencyKey.startsWith("interaction:")), ); if (!publication?.providerMessageId) throw new Error("Question publication was not delivered"); expect(publication).toMatchObject({ state: "published", payload: { interactionId: interaction.id, progressState: "waiting_for_input", card: { kind: "question", title: "Which priority should we use?", }, }, }); const callbackActions = publication.payload.card?.actions?.filter( (action) => action.type === "callback", ); expect(callbackActions).toHaveLength(2); const highAction = callbackActions?.find( (action) => action.label === "High", ); if (!highAction || highAction.type !== "callback") throw new Error("High-priority callback was not projected"); const linkedUserId = `question-user-${randomUUID()}`; const now = new Date(); await db.insert(authUsers).values({ id: linkedUserId, name: "Question User", email: `${linkedUserId}@example.com`, emailVerified: true, createdAt: now, updatedAt: now, }); await db.insert(companyMemberships).values({ companyId: fixture.companyId, principalType: "user", principalId: linkedUserId, status: "active", membershipRole: "operator", }); const principal = await db .select() .from(chatExternalPrincipals) .where(eq(chatExternalPrincipals.externalId, externalUserId)) .then((rows) => rows[0]); const intent = await service.createLinkIntent( endpoint.id, principal.id, 1_800, ); const token = new URL(intent.confirmationUrl).searchParams.get("token"); if (!token) throw new Error("Question-user identity token was absent"); await service.confirmIdentityLink(token, linkedUserId); const actionEvent = ( overrides: Partial<{ actionId: string; messageId: string; threadId: string; userId: string; value: string; }> = {}, ) => ({ endpointId: endpoint.id, provider: "slack" as const, event: { actionId: overrides.actionId ?? highAction.actionId, adapter: {} as never, messageId: overrides.messageId ?? publication.providerMessageId!, openModal: async () => undefined, raw: { type: "block_actions" }, thread: channel.thread, // The pinned Slack adapter uses the clicked Block Kit message ts here, // even though the authoritative Paperclip conversation is rooted at // the original mention (and a DM root has no ts at all). threadId: overrides.threadId ?? `slack:C-QUESTION:${publication.providerMessageId}`, user: { userId: overrides.userId ?? externalUserId, userName: "question-user", fullName: "Question User", isBot: false, isMe: false, isSystem: false, }, value: overrides.value ?? interaction.id, }, }); const unlinkedAction = actionEvent({ userId: `U-UNLINKED-${randomUUID()}`, }); await callbacks.onAction(unlinkedAction); await callbacks.onAction(unlinkedAction); await vi.waitFor(() => expect(channel.postEphemeral).toHaveBeenCalledTimes(1), ); expect(channel.postEphemeral).toHaveBeenCalledWith( unlinkedAction.event.user.userId, "This action is no longer available. Open the linked Paperclip task or ask an operator to link this account.", { fallbackToDM: false }, ); const deniedSlackActions = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "action"), ), ); expect(deniedSlackActions).toEqual([ expect.objectContaining({ conversationId: null, principalId: expect.any(String), state: "filtered", attempts: 1, redactedError: "External action denied by Paperclip authorization", normalizedEvent: { providerEventId: expect.stringMatching( /^action-denied:[a-f0-9]{64}$/, ), kind: "action", authorization: { outcome: "denied" }, }, }), ]); expect(JSON.stringify(deniedSlackActions[0])).not.toContain( highAction.actionId, ); expect(JSON.stringify(deniedSlackActions[0])).not.toContain(interaction.id); expect(await service.listActivity(endpoint.id)).toEqual( expect.arrayContaining([ expect.objectContaining({ id: deniedSlackActions[0]!.id, kind: "delivery", status: "filtered", summary: "action ignored", detail: "External action denied by Paperclip authorization", replayable: false, }), ]), ); channel.postEphemeral.mockRejectedValueOnce( Object.assign(new Error("injected Slack ephemeral failure"), { name: "ValidationError", }), ); await callbacks.onAction( actionEvent({ userId: `U-UNLINKED-FALLBACK-${randomUUID()}` }), ); await vi.waitFor(() => expect(channel.post).toHaveBeenCalledWith( "This Paperclip action is no longer available.", ), ); expect(channel.postEphemeral).toHaveBeenCalledTimes(2); await db .update(companyMemberships) .set({ membershipRole: "viewer" }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalId, linkedUserId), ), ); await callbacks.onAction(actionEvent({ actionId: "pcq:viewer-forbidden" })); await db .update(companyMemberships) .set({ membershipRole: "operator" }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalId, linkedUserId), ), ); const otherChannel = makeThread({ channelId: "C-QUESTION-OTHER", id: "slack:C-QUESTION-OTHER:4501.1", name: "other-question", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: otherChannel.thread, message: makeMessage({ id: "4501.1", text: "@maya a separate task", mentioned: true, userId: externalUserId, }), trigger: "mention", }); await callbacks.onAction(actionEvent({ threadId: otherChannel.thread.id })); await callbacks.onAction(actionEvent({ messageId: "outbound-forged" })); await callbacks.onAction(actionEvent({ actionId: "pcq:forged" })); await callbacks.onAction( actionEvent({ threadId: `slack:C-QUESTION:${publication.providerMessageId}-forged-value`, value: randomUUID(), }), ); // Model the provider accepting the interactive card while Paperclip loses // the response before it can persist the provider message binding. The // signed callback and opaque issued token must reconcile that ambiguity // without requiring a duplicate send or disabling the real button. await db .delete(chatMessageLinks) .where(eq(chatMessageLinks.publicationId, publication.id)); await db .update(chatPublications) .set({ state: "delivery_unknown", providerMessageId: null }) .where(eq(chatPublications.id, publication.id)); const providerAcknowledgement = callbacks.onAction(actionEvent()); await expect( Promise.race([ providerAcknowledgement.then(() => "acknowledged" as const), new Promise<"timed_out">((resolve) => setTimeout(() => resolve("timed_out"), 250), ), ]), ).resolves.toBe("acknowledged"); await providerAcknowledgement; expect(resolveNativeQuestion).not.toHaveBeenCalled(); await expect( db .select({ providerMessageId: chatPublications.providerMessageId, state: chatPublications.state, }) .from(chatPublications) .where(eq(chatPublications.id, publication.id)), ).resolves.toEqual([ { providerMessageId: publication.providerMessageId, state: "published", }, ]); await service.processPendingPublications(); await vi.waitFor(async () => { await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), eq( chatPublications.idempotencyKey, `interaction-resolution:${interaction.id}:${endpoint.id}`, ), ), ), ).resolves.toEqual([{ state: "published" }]); }); const resolutionPublication = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), eq( chatPublications.idempotencyKey, `interaction-resolution:${interaction.id}:${endpoint.id}`, ), ), ) .then((rows) => rows[0]); expect(resolutionPublication).toMatchObject({ state: "published", providerMessageId: publication.providerMessageId, payload: { interactionId: interaction.id, text: "Answered: High.", card: { kind: "question", title: "Which priority should we use?", body: "Answered: High.", }, }, }); expect(resolutionPublication?.payload.card?.actions ?? []).toHaveLength(0); const providerRuntime = runtime.endpoints.get(endpoint.id); expect(providerRuntime?.edits).toHaveLength(1); expect(providerRuntime?.edits[0]).toMatchObject({ messageId: publication.providerMessageId, }); expect(providerRuntime?.edits[0]?.text).toContain("Answered: High."); expect(providerRuntime?.edits[0]?.text).not.toContain(highAction.actionId); expect(channel.postEphemeral).not.toHaveBeenCalledWith( externalUserId, "Answered: High.", { fallbackToDM: false, }, ); const providerEffectsBeforeExactRedelivery = await db .select({ id: chatActions.id }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "provider_effect"), ), ); expect(providerEffectsBeforeExactRedelivery.length).toBeGreaterThan(0); // Action callbacks acknowledge after the denial and its provider effect are // durable, while the actual ephemeral send runs asynchronously. Drain those // already-staged notices before using the transport call count to prove that // the exact redelivery below does not send another notice. await vi.waitFor(async () => { const providerEffects = await db .select({ status: chatActions.status }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "provider_effect"), ), ); expect(providerEffects).toHaveLength( providerEffectsBeforeExactRedelivery.length, ); expect( providerEffects.every(({ status }) => status === "processed"), ).toBe(true); }); const ephemeralCountBeforeExactRedelivery = channel.postEphemeral.mock.calls.length; const nativeResolutionCountBeforeExactRedelivery = resolveNativeQuestion.mock.calls.length; await callbacks.onAction(actionEvent()); expect(channel.postEphemeral).toHaveBeenCalledTimes( ephemeralCountBeforeExactRedelivery, ); expect(resolveNativeQuestion).toHaveBeenCalledTimes( nativeResolutionCountBeforeExactRedelivery, ); await expect( db .select({ id: chatActions.id }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "provider_effect"), ), ), ).resolves.toHaveLength(providerEffectsBeforeExactRedelivery.length); const [storedInteraction] = await db .select() .from(issueThreadInteractions) .where(eq(issueThreadInteractions.id, interaction.id)); expect(storedInteraction).toMatchObject({ status: "answered", resolvedByUserId: linkedUserId, result: { version: 1, answers: [{ questionId: "priority", optionIds: ["high"] }], }, }); expect( await db .select() .from(issueQuestionResponseDeliveries) .where( eq(issueQuestionResponseDeliveries.interactionId, interaction.id), ), ).toHaveLength(1); const actions = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "question_answer"), ), ); expect(actions).toHaveLength(2); expect( actions.find((action) => action.status === "processed"), ).toMatchObject({ status: "processed", kind: "question_answer", payload: { publicationId: publication.id, interactionId: interaction.id, questionId: "priority", optionId: "high", }, }); expect(actions.find((action) => action.status === "expired")).toMatchObject( { kind: "question_answer", result: { code: "interaction_resolved_by_sibling_action" }, }, ); const answeredActivity = await db .select() .from(activityLog) .where( and( eq(activityLog.companyId, fixture.companyId), eq(activityLog.action, "issue.thread_interaction_answered"), eq(activityLog.entityId, conversation.issueId), ), ); expect(answeredActivity).toHaveLength(1); expect(answeredActivity[0]).toMatchObject({ actorType: "user", actorId: linkedUserId, details: { source: "external_chat", endpointId: endpoint.id, publicationId: publication.id, interactionId: interaction.id, }, }); const continuationPresentationAuthorization = await resolveChatRunPresentationAuthorizationReason(db, { companyId: fixture.companyId, issueId: conversation.issueId, runId: continuationRunId, }); expect(continuationPresentationAuthorization).toBe( "allow_chat_run_presentation", ); const continuationComment = await issueService(db).addComment( conversation.issueId, "SLACK-COLOR-High", { agentId: fixture.assignedAgentId, runId: continuationRunId }, { authorType: "agent", // Heartbeat's final-presentation resolver uses this narrow reason for // a continuation run that resolves back to this published prompt. authorizationReason: continuationPresentationAuthorization, }, ); await service.processPendingPublications(); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.commentId, continuationComment.id)), ).resolves.toEqual([ expect.objectContaining({ conversationId: conversation.id, endpointId: endpoint.id, state: "published", }), ]); expect(runtime.endpoints.get(endpoint.id)?.posts).toEqual( expect.arrayContaining([ expect.objectContaining({ threadId: channel.thread.id, text: "SLACK-COLOR-High", }), ]), ); await db .update(heartbeatRuns) .set({ resultJson: { presentationDecision: { chosenSource: "final_agent_message", commentAction: "create", commentId: continuationComment.id, }, }, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, continuationRunId)); await expect(enqueueChatRunMilestones(db)).resolves.toBe(0); await expect( db .select() .from(chatPublications) .where( like( chatPublications.idempotencyKey, `run:${continuationRunId}:completed:%`, ), ), ).resolves.toHaveLength(0); const unrelatedRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: unrelatedRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: { issueId: conversation.issueId, taskId: conversation.issueId, interactionId: interaction.id, sourceRunId, source: "issue.interaction.respond", }, }); await expect( resolveChatRunPresentationAuthorizationReason(db, { companyId: fixture.companyId, issueId: conversation.issueId, runId: unrelatedRunId, }), ).resolves.toBe("internal_agent_write"); const unrelatedComment = await issueService(db).addComment( conversation.issueId, "This unrelated run must remain internal", { agentId: fixture.assignedAgentId, runId: unrelatedRunId }, { authorType: "agent", authorizationReason: "paperclip_runner_protocol", }, ); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.commentId, unrelatedComment.id)), ).resolves.toHaveLength(0); await db .update(issueQuestionResponseDeliveries) .set({ status: "delivering", targetRunId: null }) .where(eq(issueQuestionResponseDeliveries.interactionId, interaction.id)); const inFlightNativeComment = await issueService(db).addComment( conversation.issueId, "SLACK-NATIVE-IN-FLIGHT-COLOR-High", { agentId: fixture.assignedAgentId, runId: sourceRunId }, { authorType: "agent", authorizationReason: "paperclip_runner_protocol", }, ); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.commentId, inFlightNativeComment.id)), ).resolves.toHaveLength(0); await db .update(issueQuestionResponseDeliveries) .set({ status: "delivered", targetRunId: sourceRunId }) .where(eq(issueQuestionResponseDeliveries.interactionId, interaction.id)); const resumedSourceComment = await issueService(db).addComment( conversation.issueId, "SLACK-RESUMED-SOURCE-COLOR-High", { agentId: fixture.assignedAgentId, runId: sourceRunId }, { authorType: "agent", authorizationReason: "paperclip_runner_protocol", }, ); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.commentId, resumedSourceComment.id)), ).resolves.toHaveLength(0); const steeredRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: steeredRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", // Direct question-response steering may target an already-running run // whose snapshot belongs to another chat turn. The durable delivery // receipt below must take precedence over this unrelated chat context. contextSnapshot: { issueId: conversation.issueId, taskId: conversation.issueId, endpointId: randomUUID(), source: "chat:telegram", }, }); await db .update(issueQuestionResponseDeliveries) .set({ status: "delivered", targetRunId: steeredRunId }) .where(eq(issueQuestionResponseDeliveries.interactionId, interaction.id)); const steeredComment = await addSelectedChatFinal({ agentId: fixture.assignedAgentId, body: "SLACK-STEERED-COLOR-High", companyId: fixture.companyId, issueId: conversation.issueId, runId: steeredRunId, }); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.commentId, steeredComment.id)), ).resolves.toEqual([ expect.objectContaining({ conversationId: conversation.id, endpointId: endpoint.id, }), ]); const reassignedRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: reassignedRunId, companyId: fixture.companyId, agentId: fixture.replacementAgentId, status: "succeeded", contextSnapshot: { issueId: conversation.issueId, taskId: conversation.issueId, interactionId: interaction.id, sourceRunId, source: "issue.interaction.respond", }, }); await db .update(issueQuestionResponseDeliveries) .set({ status: "delivered", targetRunId: reassignedRunId }) .where(eq(issueQuestionResponseDeliveries.interactionId, interaction.id)); const reassignedComment = await issueService(db).addComment( conversation.issueId, "A reassigned agent must not speak through Maya's Slack identity", { agentId: fixture.replacementAgentId, runId: reassignedRunId }, { authorType: "agent", authorizationReason: "paperclip_runner_protocol", }, ); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.commentId, reassignedComment.id)), ).resolves.toHaveLength(0); const staleNoticeCount = channel.postEphemeral.mock.calls.filter( ([userId, text]) => userId === externalUserId && text === "This action is no longer available. Open the linked Paperclip task or ask an operator to link this account.", ).length; await callbacks.onAction(actionEvent()); expect( channel.postEphemeral.mock.calls.filter( ([userId, text]) => userId === externalUserId && text === "This action is no longer available. Open the linked Paperclip task or ask an operator to link this account.", ), ).toHaveLength(staleNoticeCount); expect( await db .select() .from(issueQuestionResponseDeliveries) .where( eq(issueQuestionResponseDeliveries.interactionId, interaction.id), ), ).toHaveLength(1); await db .update(chatPublications) .set({ state: "delivery_unknown" }) .where(eq(chatPublications.id, resolutionPublication.id)); await service.resolvePublication( endpoint.id, resolutionPublication.id, "retry_anyway", linkedUserId, ); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.id, resolutionPublication.id)), ).resolves.toEqual([{ state: "published" }]); await db .update(chatPublications) .set({ state: "delivery_unknown", providerMessageId: null }) .where(eq(chatPublications.id, publication.id)); await expect( service.resolvePublication( endpoint.id, publication.id, "retry_anyway", linkedUserId, ), ).rejects.toMatchObject({ status: 409, details: { code: "chat_terminal_interaction_publication_not_replayable", }, }); await service.shutdown(); }); it("resolves a signed Slack modal correction after SDK context consumption exactly once", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredSlackEndpoint(fixture); let pinned: ReturnType | undefined; let providerServer: Server | undefined; try { const channelId = "C-SIGNED-MODAL"; const threadTs = "1788.100"; const externalUserId = `U-MODAL-${randomUUID()}`; const channel = makeThread({ channelId, id: `slack:${channelId}:${threadTs}`, name: "signed-modal", }); // Existing fixture helpers establish the endpoint, conversation and // published question. From block_actions onward, signed envelopes pass // through the real adapter/SDK/runtime and unmocked service callbacks. // Provider HTTP and the scheduler remain deterministic test boundaries. await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: threadTs, text: "@maya collect deployment details", mentioned: true, userId: externalUserId, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id, externalUserId); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected signed modal conversation"); const [principal] = await db .select() .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, fixture.companyId), eq(chatExternalPrincipals.provider, "slack"), eq(chatExternalPrincipals.externalId, externalUserId), ), ); const linkedUserId = `signed-modal-user-${randomUUID()}`; const now = new Date(); await db.insert(authUsers).values({ id: linkedUserId, name: "Modal Operator", email: `${linkedUserId}@example.com`, emailVerified: true, createdAt: now, updatedAt: now, }); await db.insert(companyMemberships).values({ companyId: fixture.companyId, principalType: "user", principalId: linkedUserId, status: "active", membershipRole: "operator", }); const intent = await service.createLinkIntent( endpoint.id, principal.id, 1_800, ); const identityToken = new URL(intent.confirmationUrl).searchParams.get( "token", ); if (!identityToken) throw new Error("Expected modal identity token"); await service.confirmIdentityLink(identityToken, linkedUserId); const interaction = await issueThreadInteractionService(db).create( { id: conversation.issueId, companyId: fixture.companyId }, { kind: "ask_user_questions", continuationPolicy: "wake_assignee", title: "Deployment details", payload: { version: 1, title: "Deployment details", submitLabel: "Continue", questions: [ { id: "environment", prompt: "Where should I deploy?", selectionMode: "single", required: true, allowOther: false, options: [ { id: "staging", label: "Staging" }, { id: "production", label: "Production" }, ], }, { id: "reason", prompt: "What should the release note say?", selectionMode: "single", required: true, allowOther: true, options: [ { id: "__paperclip_text__", label: "Type an answer", freeText: true, }, ], }, ], }, }, { agentId: fixture.assignedAgentId }, ); runtime.endpoints.get(endpoint.id)!.postResultIds.push("1788.200"); await service.processPendingPublications(); const publication = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), eq(chatPublications.conversationId, conversation.id), ), ) .then((rows) => rows.find((row) => row.payload.interactionId === interaction.id), ); const openAction = publication?.payload.card?.actions?.find( (action) => action.type === "callback", ); if (!publication?.providerMessageId || openAction?.type !== "callback") { throw new Error("Expected published question form action"); } const configuration = runtime.configurations.get(endpoint.id)!; if ( configuration.providerConfig.provider !== "slack" || !callbacks.onModalSubmit ) { throw new Error("Expected Slack provider and service modal callback"); } const credentials = configuration.providerConfig.credentials; type SlackView = { callback_id: string; private_metadata: string; blocks: Array<{ block_id: string; element: { type: string; action_id: string; options?: Array<{ text: { text: string }; value: string }>; }; }>; }; const views: SlackView[] = []; const providerMethods: string[] = []; const unexpectedProviderMethods: string[] = []; providerServer = createServer(async (request, response) => { let body = ""; for await (const chunk of request) body += String(chunk); const params = new URLSearchParams(body); const method = request.url?.split("/").at(-1) ?? ""; providerMethods.push(method); let result: unknown; if (method === "conversations.replies") { result = { ok: true, messages: [ { type: "message", ts: publication.providerMessageId, thread_ts: threadTs, channel: channelId, user: credentials.botUserId, text: "Deployment details", }, ], }; } else if (method === "users.info") { result = { ok: true, user: { id: params.get("user"), name: "maya", real_name: "Maya", is_bot: true, profile: { display_name: "Maya" }, }, }; } else if (method === "views.open") { views.push(JSON.parse(params.get("view")!) as SlackView); result = { ok: true, view: { id: "V-SIGNED-MODAL" } }; } else { unexpectedProviderMethods.push(method); result = { ok: false, error: "unexpected_test_provider_method" }; } response.writeHead(200, { "content-type": "application/json" }); response.end(JSON.stringify(result)); }); await new Promise((resolve) => providerServer!.listen(0, "127.0.0.1", resolve), ); const onSubmit = vi.fn(callbacks.onModalSubmit); pinned = createChatSdkEndpointRuntime({ ...configuration, callbacks: { ...callbacks, onModalSubmit: onSubmit }, logger: "silent", providerConfig: { ...configuration.providerConfig, credentials: { ...credentials, apiUrl: `http://127.0.0.1:${(providerServer.address() as AddressInfo).port}/api/`, }, }, }); await pinned.initialize(); const signed = (payload: unknown) => { const body = new URLSearchParams({ payload: JSON.stringify(payload), }).toString(); const timestamp = String(Math.floor(Date.now() / 1_000)); const signature = createHmac("sha256", credentials.signingSecret!) .update(`v0:${timestamp}:${body}`) .digest("hex"); return new Request("https://paperclip.test/webhooks/slack", { method: "POST", headers: { "content-type": "application/x-www-form-urlencoded", "x-slack-request-timestamp": timestamp, "x-slack-signature": `v0=${signature}`, }, body, }); }; const wireUser = { id: externalUserId, username: "operator", name: "Modal Operator", }; const opened = await pinned.handleWebhook( signed({ type: "block_actions", team: { id: "T-PAPERCLIP" }, user: wireUser, channel: { id: channelId }, container: { type: "message", channel_id: channelId, message_ts: publication.providerMessageId, thread_ts: threadTs, }, message: { ts: publication.providerMessageId, thread_ts: threadTs }, actions: [{ action_id: openAction.actionId, value: interaction.id }], trigger_id: "synthetic-signed-modal-trigger", }), ); expect(opened.status).toBe(200); expect(views).toHaveLength(1); const view = views[0]!; const metadata = JSON.parse(view.private_metadata) as { c: string; m: string; }; expect(metadata).toEqual({ c: expect.any(String), m: view.callback_id }); const select = view.blocks.find( (block) => block.element.type === "static_select", )!; const text = view.blocks.find( (block) => block.element.type === "plain_text_input", )!; const production = select.element.options!.find( (option) => option.text.text === "Production", )!; expect(production.value).not.toBe("production"); const submit = (value: string) => signed({ type: "view_submission", team: { id: "T-PAPERCLIP" }, user: wireUser, view: { id: "V-SIGNED-MODAL", callback_id: view.callback_id, private_metadata: view.private_metadata, state: { values: { [select.block_id]: { [select.element.action_id]: { type: "static_select", selected_option: production, }, }, [text.block_id]: { [text.element.action_id]: { type: "plain_text_input", value }, }, }, }, }, }); const sdkContextRows = () => db .select({ id: chatSdkState.id }) .from(chatSdkState) .where( and( eq(chatSdkState.companyId, fixture.companyId), eq(chatSdkState.endpointId, endpoint.id), eq( chatSdkState.stateKey, `cache:${createHash("sha256").update(`modal-context:slack:${metadata.c}`).digest("hex")}`, ), ), ); const submissionState = async () => ({ interactions: await db .select() .from(issueThreadInteractions) .where(eq(issueThreadInteractions.id, interaction.id)), tokens: await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.providerActionId, view.callback_id), ), ), deliveries: await db .select() .from(issueQuestionResponseDeliveries) .where( eq(issueQuestionResponseDeliveries.interactionId, interaction.id), ), audits: await db .select() .from(activityLog) .where( and( eq(activityLog.companyId, fixture.companyId), eq(activityLog.entityId, conversation.issueId), eq(activityLog.action, "issue.thread_interaction_answered"), ), ), }); await expect(sdkContextRows()).resolves.toHaveLength(1); const before = await submissionState(); expect(before.interactions).toEqual([ expect.objectContaining({ status: "pending" }), ]); expect(before.tokens).toEqual([ expect.objectContaining({ status: "issued" }), ]); expect(before.deliveries).toEqual([]); expect(before.audits).toEqual([]); const wakeupsBefore = wakeup.mock.calls.length; const invalid = await pinned.handleWebhook(submit("")); expect(invalid.status).toBe(200); expect(await invalid.json()).toEqual({ response_action: "errors", errors: { [text.block_id]: "Enter a response" }, }); expect(onSubmit.mock.calls[0]![0].event).toMatchObject({ relatedThread: { id: conversation.externalThreadId }, relatedMessage: { id: publication.providerMessageId }, }); await expect(sdkContextRows()).resolves.toEqual([]); expect(await submissionState()).toEqual(before); expect(wakeup).toHaveBeenCalledTimes(wakeupsBefore); const setRole = (membershipRole: "viewer" | "operator") => db .update(companyMemberships) .set({ membershipRole, updatedAt: new Date() }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalType, "user"), eq(companyMemberships.principalId, linkedUserId), ), ); await setRole("viewer"); const denied = await pinned.handleWebhook( submit("Add regional failover"), ); expect(denied.status).toBe(200); expect(await denied.json()).toEqual({ response_action: "errors", errors: { [select.block_id]: "This form is no longer authorized. Close it and open the linked Paperclip task.", }, }); expect(await submissionState()).toEqual(before); expect(wakeup).toHaveBeenCalledTimes(wakeupsBefore); const denials = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), sql`${chatDeliveries.normalizedEvent}->'modal'->>'code' = 'chat_modal_principal_not_authorized'`, ), ); expect(denials).toHaveLength(1); expect(denials[0]).toMatchObject({ state: "filtered", conversationId: conversation.id, }); await setRole("operator"); const corrected = await pinned.handleWebhook( submit(" Add regional failover "), ); expect(corrected.status).toBe(200); expect(await corrected.json()).toEqual({ response_action: "clear" }); await vi.waitFor(async () => { expect(wakeup).toHaveBeenCalledTimes(wakeupsBefore + 1); expect((await submissionState()).deliveries).toEqual([ expect.objectContaining({ status: "fallback_queued", deliveryMode: "wake_fallback", }), ]); }); expect(wakeup).toHaveBeenLastCalledWith( fixture.assignedAgentId, expect.objectContaining({ idempotencyKey: `question-response:${interaction.id}`, requestedByActorType: "user", requestedByActorId: linkedUserId, contextSnapshot: expect.objectContaining({ issueId: conversation.issueId, interactionId: interaction.id, source: "issue.interaction.respond", }), }), ); const accepted = await submissionState(); expect(accepted.interactions).toEqual([ expect.objectContaining({ companyId: fixture.companyId, issueId: conversation.issueId, status: "answered", resolvedByUserId: linkedUserId, result: { version: 1, summaryMarkdown: null, answers: [ { questionId: "environment", optionIds: ["production"] }, { questionId: "reason", optionIds: [], otherText: "Add regional failover", }, ], }, }), ]); expect(accepted.tokens).toEqual([ expect.objectContaining({ status: "processed", conversationId: conversation.id, principalId: principal.id, result: { code: "question_form_answered", interactionId: interaction.id, }, }), ]); expect(accepted.deliveries).toHaveLength(1); expect(accepted.audits).toEqual([ expect.objectContaining({ actorId: linkedUserId, details: expect.objectContaining({ source: "external_chat_modal", endpointId: endpoint.id, conversationId: conversation.id, publicationId: publication.id, }), }), ]); const duplicate = await pinned.handleWebhook( submit(" Add regional failover "), ); expect(duplicate.status).toBe(200); expect(await duplicate.json()).toEqual({ response_action: "clear" }); expect(await submissionState()).toEqual(accepted); expect(wakeup).toHaveBeenCalledTimes(wakeupsBefore + 1); expect(onSubmit).toHaveBeenCalledTimes(4); for (const [event] of onSubmit.mock.calls.slice(1)) { expect(event).toMatchObject({ endpointId: endpoint.id, provider: "slack", event: { callbackId: view.callback_id, privateMetadata: view.callback_id, user: { userId: externalUserId }, }, }); expect(event.event.relatedThread).toBeUndefined(); expect(event.event.relatedMessage).toBeUndefined(); } await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ id: conversation.id, issueId: conversation.issueId, externalThreadId: channel.thread.id, }), ]); expect( providerMethods.filter((method) => method === "views.open"), ).toHaveLength(1); expect(unexpectedProviderMethods).toEqual([]); } finally { try { await pinned?.shutdown(); } finally { try { await retirePublicationFixture(service, endpoint.id); } finally { if (providerServer) { await new Promise((resolve, reject) => { providerServer!.close((error) => error ? reject(error) : resolve(), ); providerServer!.closeAllConnections(); }); } } } } }); it.each([ { provider: "slack" as const, label: "Slack" }, { provider: "microsoft-teams" as const, label: "Microsoft Teams" }, ])( "round-trips a $label question modal and durably acknowledges denied or replayed tokens", async ({ provider }) => { const fixture = await seedCompany(); let pauseNextFormOpen = false; let releaseFormOpenAuthorization!: () => void; let signalFormOpenAuthorization!: () => void; const formOpenAuthorizationEntered = new Promise((resolve) => { signalFormOpenAuthorization = resolve; }); const formOpenAuthorizationReleased = new Promise((resolve) => { releaseFormOpenAuthorization = resolve; }); const questionFormOpenAuthorizationBarrier = async () => { if (!pauseNextFormOpen) return; signalFormOpenAuthorization(); await formOpenAuthorizationReleased; }; const context = provider === "slack" ? await configuredSlackEndpoint(fixture, { questionFormOpenAuthorizationBarrier, }) : await (async () => { const created = createService( new FakeChatSdkRuntime(), (async () => new Response( JSON.stringify({ access_token: "teams-modal-access" }), { status: 200, headers: { "content-type": "application/json" }, }, )) as typeof globalThis.fetch, { questionFormOpenAuthorizationBarrier, }, ); const endpoint = await created.service.create( fixture.companyId, { provider: "microsoft-teams", assignedAgentId: fixture.assignedAgentId, }, "owner-user", ); await created.service.configure( endpoint.id, { action: "configure", credentials: { clientId: "00000000-0000-4000-8000-000000000311", tenantId: "00000000-0000-4000-8000-000000000322", clientSecret: "teams-modal-secret", }, }, "owner-user", ); const callbacks = created.runtime.configurations.get( endpoint.id, )?.callbacks; if (!callbacks) throw new Error("Expected Teams modal callbacks"); return { ...created, endpoint, callbacks }; })(); const { callbacks, endpoint, runtime, service } = context; if (!callbacks.onAction || !callbacks.onModalSubmit) { throw new Error("Expected question action and modal callbacks"); } const teamsEndpointRuntime = provider === "microsoft-teams" ? runtime.endpoints.get(endpoint.id) : null; if (provider === "microsoft-teams" && !teamsEndpointRuntime) { throw new Error("Expected Teams endpoint runtime"); } const teamsRouteCount = () => teamsEndpointRuntime?.recordedMicrosoftTeamsRoutes.length ?? 0; const externalUserId = `${provider}-modal-user-${randomUUID()}`; const teamsConversationId = "19:modal-conversation@thread.tacv2"; const teamsServiceUrl = "https://smba.trafficmanager.net/amer/"; const channel = provider === "slack" ? makeThread({ channelId: "C-MODAL-ROUNDTRIP", id: `slack:C-MODAL-ROUNDTRIP:${randomUUID()}`, name: "modal-roundtrip", }) : makeThread({ channelId: `teams:${Buffer.from(teamsConversationId).toString("base64url")}:${Buffer.from(teamsServiceUrl).toString("base64url")}`, id: `teams:${Buffer.from(`${teamsConversationId};messageid=modal-root`).toString("base64url")}:${Buffer.from(teamsServiceUrl).toString("base64url")}`, name: "modal-roundtrip", }); const callbackThread = provider === "microsoft-teams" ? { ...channel.thread, channelId: `teams:${Buffer.from(teamsConversationId).toString("base64url")}`, id: `teams:${Buffer.from(`${teamsConversationId};messageid=modal-root`).toString("base64url")}`, } : channel.thread; await deliverMessage({ callbacks, endpointId: endpoint.id, provider, thread: channel.thread, message: makeMessage({ id: `modal-root-${randomUUID()}`, text: "@maya collect the deployment details", mentioned: true, userId: externalUserId, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id, externalUserId); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const linkedUserId = `modal-paperclip-user-${randomUUID()}`; const now = new Date(); await db.insert(authUsers).values({ id: linkedUserId, name: "Modal User", email: `${linkedUserId}@example.com`, emailVerified: true, createdAt: now, updatedAt: now, }); await db.insert(companyMemberships).values({ companyId: fixture.companyId, principalType: "user", principalId: linkedUserId, status: "active", membershipRole: "operator", }); const principal = await db .select() .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, fixture.companyId), eq(chatExternalPrincipals.provider, provider), eq(chatExternalPrincipals.externalId, externalUserId), ), ) .then((rows) => rows[0]); const intent = await service.createLinkIntent( endpoint.id, principal.id, 1_800, ); const identityToken = new URL(intent.confirmationUrl).searchParams.get( "token", ); if (!identityToken) throw new Error("Modal identity token was absent"); await service.confirmIdentityLink(identityToken, linkedUserId); const interaction = await issueThreadInteractionService(db).create( { id: conversation.issueId, companyId: fixture.companyId }, { kind: "ask_user_questions", continuationPolicy: "wake_assignee", title: "Deployment details", payload: { version: 1, title: "Deployment details", submitLabel: "Continue", questions: [ { id: "environment", prompt: "Where should I deploy?", selectionMode: "single", required: true, allowOther: false, options: [ { id: "staging", label: "Staging" }, { id: "production", label: "Production" }, ], }, { id: "reason", prompt: "What should the release note say?", selectionMode: "single", required: true, allowOther: true, options: [ { id: "__paperclip_text__", label: "Type an answer", freeText: true, }, ], }, ], }, }, { agentId: fixture.assignedAgentId }, ); await service.processPendingPublications(); const publication = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), eq(chatPublications.issueId, conversation.issueId), ), ) .then((rows) => rows.find((row) => row.payload.interactionId === interaction.id), ); if (!publication?.providerMessageId) { throw new Error("Question form publication was not delivered"); } const openAction = publication.payload.card?.actions?.find( (action) => action.type === "callback", ); if (!openAction || openAction.type !== "callback") { throw new Error("Question form opener was not projected"); } const modalUser = { userId: externalUserId, userName: "modal-user", fullName: "Modal User", isBot: false, isMe: false, isSystem: false, }; const actionEvent = (actionId: string) => ({ endpointId: endpoint.id, provider, event: { actionId, adapter: {} as never, messageId: publication.providerMessageId!, openModal: vi.fn(async () => ({ viewId: "modal-view" })), raw: provider === "microsoft-teams" ? { serviceUrl: teamsServiceUrl } : {}, thread: channel.thread, threadId: callbackThread.id, triggerId: `modal-trigger-${randomUUID()}`, user: modalUser, value: interaction.id, }, }); const routeCountBeforeDeniedOpen = teamsRouteCount(); await callbacks.onAction( actionEvent(`pcf:${"A".repeat(22)}`) as Parameters< NonNullable >[0], ); await vi.waitFor(() => expect(channel.postEphemeral).toHaveBeenCalledWith( modalUser.userId, "This action is no longer available. Open the linked Paperclip task or ask an operator to link this account.", { fallbackToDM: false }, ), ); expect(teamsRouteCount()).toBe(routeCountBeforeDeniedOpen); if (provider === "microsoft-teams") { pauseNextFormOpen = true; const racedOpen = actionEvent(openAction.actionId); const opening = callbacks.onAction( racedOpen as Parameters>[0], ); await formOpenAuthorizationEntered; await db .update(companyMemberships) .set({ membershipRole: "viewer", updatedAt: new Date() }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalType, "user"), eq(companyMemberships.principalId, linkedUserId), ), ); releaseFormOpenAuthorization(); await opening; expect(racedOpen.event.openModal).not.toHaveBeenCalled(); expect(teamsRouteCount()).toBe(routeCountBeforeDeniedOpen); await db .update(companyMemberships) .set({ membershipRole: "operator", updatedAt: new Date() }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalType, "user"), eq(companyMemberships.principalId, linkedUserId), ), ); pauseNextFormOpen = false; } if (provider === "slack") { const lockProbeOpen = actionEvent(openAction.actionId); let releaseProviderOpen!: () => void; const providerOpenEntered = vi.fn(); lockProbeOpen.event.openModal = vi.fn(async () => { providerOpenEntered(); await new Promise((resolve) => { releaseProviderOpen = resolve; }); return { viewId: "modal-lock-probe" }; }); const opening = callbacks.onAction( lockProbeOpen as Parameters< NonNullable >[0], ); await vi.waitFor(() => expect(providerOpenEntered).toHaveBeenCalledTimes(1), ); const membershipUpdate = db .update(companyMemberships) .set({ membershipRole: "viewer", updatedAt: new Date() }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalType, "user"), eq(companyMemberships.principalId, linkedUserId), ), ) .returning({ id: companyMemberships.id }) .then(() => "updated" as const); const updateOutcome = await Promise.race([ membershipUpdate, new Promise<"timed_out">((resolve) => setTimeout(() => resolve("timed_out"), 500), ), ]); releaseProviderOpen(); await opening; await membershipUpdate; expect(updateOutcome).toBe("updated"); expect(lockProbeOpen.event.openModal).toHaveBeenCalledTimes(1); await db .update(companyMemberships) .set({ membershipRole: "operator", updatedAt: new Date() }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalType, "user"), eq(companyMemberships.principalId, linkedUserId), ), ); const failedOpen = actionEvent(openAction.actionId); failedOpen.event.openModal = vi.fn(async () => { throw Object.assign(new Error("Slack Web API rejected modal"), { code: "slack_webapi_platform_error", data: { error: "expired_trigger_id", ok: false }, }); }); await callbacks.onAction( failedOpen as Parameters>[0], ); await callbacks.onAction( failedOpen as Parameters>[0], ); expect(failedOpen.event.openModal).toHaveBeenCalledTimes(1); await vi.waitFor(() => expect(channel.postEphemeral).toHaveBeenCalledWith( modalUser.userId, "Paperclip could not open this form. Try the action again or open the linked Paperclip task.", { fallbackToDM: false }, ), ); const modalOpenFailures = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), sql`${chatDeliveries.normalizedEvent}->'modal'->>'phase' = 'open'`, ), ); expect(modalOpenFailures).toEqual([ expect.objectContaining({ state: "filtered", attempts: 1, redactedError: expect.stringContaining( "Provider modal open failed", ), normalizedEvent: expect.objectContaining({ modal: { phase: "open", outcome: "failed", disposition: "failed", }, }), }), ]); expect(JSON.stringify(modalOpenFailures)).not.toContain( failedOpen.event.triggerId, ); } const validOpen = actionEvent(openAction.actionId); await callbacks.onAction( validOpen as Parameters>[0], ); expect(validOpen.event.openModal).toHaveBeenCalledTimes(1); expect(teamsRouteCount()).toBeGreaterThanOrEqual( routeCountBeforeDeniedOpen + (provider === "microsoft-teams" ? 1 : 0), ); const routeCountAfterValidOpen = teamsRouteCount(); const modal = validOpen.event.openModal.mock.calls[0]?.[0] as { callbackId: string; children: Array<{ id: string; options?: Array<{ label: string; value: string }>; type: string; }>; privateMetadata?: string; }; const selectField = modal.children.find( (child) => child.type === "select", ); const textField = modal.children.find( (child) => child.type === "text_input", ); const productionValue = selectField?.options?.find( (option) => option.label === "Production", )?.value; if (!selectField || !textField || !productionValue) { throw new Error("Question modal fields were incomplete"); } const modalEvent = (callbackId: string) => ({ endpointId: endpoint.id, provider, event: { adapter: {} as never, callbackId, ...(provider === "slack" ? { privateMetadata: modal.privateMetadata } : {}), raw: provider === "microsoft-teams" ? { serviceUrl: teamsServiceUrl } : {}, relatedMessage: { id: publication.providerMessageId } as never, relatedThread: callbackThread, user: modalUser, values: { [selectField.id]: productionValue, [textField.id]: "Add regional failover", }, viewId: "modal-view", }, }); const forgedSubmit = modalEvent(`pcfs:${"A".repeat(22)}`); await expect( callbacks.onModalSubmit( forgedSubmit as Parameters< NonNullable >[0], ), ).resolves.toEqual({ action: "clear" }); await expect( callbacks.onModalSubmit( forgedSubmit as Parameters< NonNullable >[0], ), ).resolves.toEqual({ action: "clear" }); await expect( db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), sql`${chatDeliveries.normalizedEvent}->'modal'->>'code' = 'chat_modal_token_not_current'`, ), ), ).resolves.toHaveLength(1); expect(teamsRouteCount()).toBe(routeCountAfterValidOpen); const validSubmit = modalEvent(modal.callbackId); const invalidSubmit = { ...validSubmit, event: { ...validSubmit.event, values: { ...validSubmit.event.values, [textField.id]: "" }, }, }; const wakeupsBeforeInvalidForm = context.wakeup.mock.calls.length; const invalidFormResponse = await callbacks.onModalSubmit( invalidSubmit as Parameters< NonNullable >[0], ); if (provider === "microsoft-teams") { expect(invalidFormResponse).toMatchObject({ action: "update", modal: { callbackId: modal.callbackId, privateMetadata: modal.callbackId, children: expect.arrayContaining([ expect.objectContaining({ type: "select", id: selectField.id, initialOption: productionValue, }), expect.objectContaining({ type: "text_input", id: textField.id, initialValue: "", }), expect.objectContaining({ type: "text", content: "What should the release note say?: Enter a response", }), ]), }, }); } else { expect(invalidFormResponse).toEqual({ action: "errors", errors: { [textField.id]: "Enter a response" }, }); } expect(context.wakeup).toHaveBeenCalledTimes(wakeupsBeforeInvalidForm); await expect( db .select({ status: issueThreadInteractions.status }) .from(issueThreadInteractions) .where(eq(issueThreadInteractions.id, interaction.id)), ).resolves.toEqual([{ status: "pending" }]); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.providerActionId, modal.callbackId), ), ), ).resolves.toEqual([{ status: "issued" }]); await expect( db .select({ id: issueQuestionResponseDeliveries.id }) .from(issueQuestionResponseDeliveries) .where( eq(issueQuestionResponseDeliveries.interactionId, interaction.id), ), ).resolves.toEqual([]); await expect( db .select({ id: activityLog.id }) .from(activityLog) .where( and( eq(activityLog.companyId, fixture.companyId), eq(activityLog.entityId, conversation.issueId), eq(activityLog.action, "issue.thread_interaction_answered"), ), ), ).resolves.toEqual([]); await db .update(companyMemberships) .set({ membershipRole: "viewer", updatedAt: new Date() }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalType, "user"), eq(companyMemberships.principalId, linkedUserId), ), ); const deniedSubmitResponse = { action: "errors", errors: { [selectField.id]: "This form is no longer authorized. Close it and open the linked Paperclip task.", }, }; await expect( callbacks.onModalSubmit( validSubmit as Parameters< NonNullable >[0], ), ).resolves.toEqual(deniedSubmitResponse); await expect( callbacks.onModalSubmit( validSubmit as Parameters< NonNullable >[0], ), ).resolves.toEqual(deniedSubmitResponse); await expect( db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), sql`${chatDeliveries.normalizedEvent}->'modal'->>'code' = 'chat_modal_principal_not_authorized'`, ), ), ).resolves.toHaveLength(1); expect(teamsRouteCount()).toBe(routeCountAfterValidOpen); await db .update(companyMemberships) .set({ membershipRole: "operator", updatedAt: new Date() }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalType, "user"), eq(companyMemberships.principalId, linkedUserId), ), ); // Both callbacks load the issued token. Hold the duplicate until the // first callback commits its answer, exercising stale preflight state // rather than relying on database latency to produce the race. const originalLoadToken = chatQuestionForms.loadChatQuestionFormSubmissionToken; let issuedReplayToken: Awaited> = null; let tokenLoads = 0; let releaseFirstSubmit!: () => void; let releaseReplaySubmit!: () => void; const firstSubmitReleased = new Promise((resolve) => { releaseFirstSubmit = resolve; }); const replaySubmitReleased = new Promise((resolve) => { releaseReplaySubmit = resolve; }); const tokenLoadSpy = vi .spyOn(chatQuestionForms, "loadChatQuestionFormSubmissionToken") .mockImplementation(async (...args) => { const loaded = await originalLoadToken(...args); if (args[1].callbackId === modal.callbackId && tokenLoads < 2) { const ordinal = ++tokenLoads; expect(loaded?.status).toBe("issued"); issuedReplayToken = loaded; await (ordinal === 1 ? firstSubmitReleased : replaySubmitReleased); } return loaded; }); const firstSubmit = callbacks.onModalSubmit( validSubmit as Parameters< NonNullable >[0], ); const replaySubmit = callbacks.onModalSubmit( validSubmit as Parameters< NonNullable >[0], ); // Observe rejections immediately; finally joins both callbacks even when // the red assertion fails, so no held callback can leak into another test. const submissionsSettled = Promise.allSettled([ firstSubmit, replaySubmit, ]); try { await vi.waitFor(() => expect(tokenLoads).toBe(2)); releaseFirstSubmit(); await vi.waitFor(async () => { const [current] = await db .select({ status: issueThreadInteractions.status }) .from(issueThreadInteractions) .where(eq(issueThreadInteractions.id, interaction.id)); expect(current?.status).toBe("answered"); }); releaseReplaySubmit(); await expect(Promise.all([firstSubmit, replaySubmit])).resolves.toEqual( [{ action: "clear" }, { action: "clear" }], ); } finally { releaseFirstSubmit(); releaseReplaySubmit(); await submissionsSettled; tokenLoadSpy.mockRestore(); } expect(teamsRouteCount()).toBeGreaterThanOrEqual( routeCountAfterValidOpen + (provider === "microsoft-teams" ? 1 : 0), ); await vi.waitFor( async () => { // The publication worker intentionally drains a bounded global batch. // A full-suite database can contain more than 25 older eligible rows, // so keep advancing FIFO until this interaction's resolution owns the // existing provider-message link. await service.processPendingPublications(); const linked = await db .select({ publicationId: chatMessageLinks.publicationId }) .from(chatMessageLinks) .where( and( eq(chatMessageLinks.endpointId, endpoint.id), eq(chatMessageLinks.conversationId, conversation.id), eq( chatMessageLinks.providerMessageId, publication.providerMessageId, ), ), ); expect(linked).toHaveLength(1); expect(linked[0]?.publicationId).not.toBe(publication.id); const resolutionState = linked[0]?.publicationId ? await db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.id, linked[0].publicationId)) .then((rows) => rows[0]?.state ?? null) : null; expect(resolutionState).toBe("published"); }, { timeout: 5_000 }, ); const [answered] = await db .select() .from(issueThreadInteractions) .where(eq(issueThreadInteractions.id, interaction.id)); expect(answered).toMatchObject({ status: "answered", resolvedByUserId: linkedUserId, result: { version: 1, answers: [ { questionId: "environment", optionIds: ["production"] }, { questionId: "reason", optionIds: [], otherText: "Add regional failover", }, ], }, }); await expect( callbacks.onModalSubmit( validSubmit as Parameters< NonNullable >[0], ), ).resolves.toEqual({ action: "clear" }); if (!issuedReplayToken) throw new Error("Expected captured issued token"); const staleReplay = async () => { const staleLoadSpy = vi .spyOn(chatQuestionForms, "loadChatQuestionFormSubmissionToken") .mockResolvedValueOnce(issuedReplayToken); try { return await callbacks.onModalSubmit!( validSubmit as Parameters< NonNullable >[0], ); } finally { staleLoadSpy.mockRestore(); } }; const [processedToken] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.providerActionId, modal.callbackId), ), ); expect(processedToken?.status).toBe("processed"); // A terminal interaction alone cannot authorize stale-token recovery. // The exact processed receipt and its original resolving user must match. await db .update(chatActions) .set({ result: { code: "question_form_answered", interactionId: randomUUID(), }, }) .where(eq(chatActions.id, processedToken.id)); await expect(staleReplay()).resolves.toEqual(deniedSubmitResponse); await db .update(chatActions) .set({ result: processedToken.result }) .where(eq(chatActions.id, processedToken.id)); await db .update(companyMemberships) .set({ membershipRole: "viewer" }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalType, "user"), eq(companyMemberships.principalId, linkedUserId), ), ); await expect(staleReplay()).resolves.toEqual(deniedSubmitResponse); await db .update(companyMemberships) .set({ membershipRole: "operator" }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalType, "user"), eq(companyMemberships.principalId, linkedUserId), ), ); await db .update(chatIdentityLinks) .set({ status: "revoked" }) .where( and( eq(chatIdentityLinks.endpointId, endpoint.id), eq(chatIdentityLinks.principalId, principal.id), ), ); await expect(staleReplay()).resolves.toEqual(deniedSubmitResponse); await db .update(chatIdentityLinks) .set({ status: "linked" }) .where( and( eq(chatIdentityLinks.endpointId, endpoint.id), eq(chatIdentityLinks.principalId, principal.id), ), ); const relinkedUserId = `modal-relinked-user-${randomUUID()}`; await db.insert(authUsers).values({ id: relinkedUserId, name: "Different Modal User", email: `${relinkedUserId}@example.com`, emailVerified: true, createdAt: now, updatedAt: now, }); await db.insert(companyMemberships).values({ companyId: fixture.companyId, principalType: "user", principalId: relinkedUserId, status: "active", membershipRole: "operator", }); await db .update(chatIdentityLinks) .set({ paperclipUserId: relinkedUserId }) .where( and( eq(chatIdentityLinks.endpointId, endpoint.id), eq(chatIdentityLinks.principalId, principal.id), ), ); await expect(staleReplay()).resolves.toEqual(deniedSubmitResponse); await expect( callbacks.onModalSubmit( validSubmit as Parameters< NonNullable >[0], ), ).resolves.toEqual(deniedSubmitResponse); await db .update(chatIdentityLinks) .set({ paperclipUserId: linkedUserId }) .where( and( eq(chatIdentityLinks.endpointId, endpoint.id), eq(chatIdentityLinks.principalId, principal.id), ), ); await expect(staleReplay()).resolves.toEqual({ action: "clear" }); expect( await db .select() .from(issueQuestionResponseDeliveries) .where( eq(issueQuestionResponseDeliveries.interactionId, interaction.id), ), ).toHaveLength(1); expect(await service.listActivity(endpoint.id)).toEqual( expect.arrayContaining([ expect.objectContaining({ kind: "delivery", status: "filtered", detail: "External chat modal submission denied by Paperclip", replayable: false, }), ]), ); }, ); it("keeps a chat-origin run presentation internal when its external question owns the reply", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const channel = makeThread({ channelId: "C-PRESENTATION-QUESTION", id: "slack:C-PRESENTATION-QUESTION:4700.1", name: "presentation-question", }); const providerMessageId = "4700.1"; await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: providerMessageId, text: "@maya ask me before choosing", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const conversation = await db .select() .from(chatConversations) .where( and( eq(chatConversations.endpointId, endpoint.id), eq(chatConversations.externalThreadId, channel.thread.id), ), ) .then((rows) => rows[0]); if (!conversation) throw new Error("Expected Slack conversation"); const sourceRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: sourceRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "slack", providerMessageId, }), resultJson: { presentationDecision: { chosenSource: "final_agent_message", commentAction: "create", }, }, }); await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${sourceRunId}:working:${endpoint.id}`, payload: { text: "Maya is working…", progressState: "working" }, state: "pending", }); await service.processPendingPublications(); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack provider runtime"); const workingPublication = await db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `run:${sourceRunId}:working:${endpoint.id}`, ), ) .then((rows) => rows[0]); if (!workingPublication?.providerMessageId) { throw new Error("Expected Slack working publication"); } expect(workingPublication).toMatchObject({ state: "published", providerMessageId: expect.any(String), }); const interaction = await issueThreadInteractionService(db).create( { id: conversation.issueId, companyId: fixture.companyId }, { kind: "ask_user_questions", continuationPolicy: "wake_assignee", sourceRunId, title: "Choose a color", payload: { version: 1, title: "Choose a color", questions: [ { id: "color", prompt: "Which color should we use?", selectionMode: "single", required: true, options: [ { id: "red", label: "Red" }, { id: "blue", label: "Blue" }, ], }, ], }, }, { agentId: fixture.assignedAgentId, runId: sourceRunId }, ); // The native interaction owns this run's provider-visible response slot as // soon as its prompt is queued. A protocol-authored waiting summary from // the same still-running source must remain an internal task comment; if it // were also queued, publication ordering could replace the prompt with the // summary before the provider action becomes usable. await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `interaction:${interaction.id}:${endpoint.id}`, ), ), ).resolves.toEqual([{ state: "pending" }]); const protocolWaitingSummary = await issueService(db).addComment( conversation.issueId, "I need your answer before I can continue.", { agentId: fixture.assignedAgentId, runId: sourceRunId }, { authorType: "agent", authorizationReason: "paperclip_runner_protocol", }, ); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.commentId, protocolWaitingSummary.id)), ).resolves.toHaveLength(0); const sourcePresentationAuthorization = await resolveChatRunPresentationAuthorizationReason(db, { companyId: fixture.companyId, issueId: conversation.issueId, runId: sourceRunId, }); expect(sourcePresentationAuthorization).toBe("internal_agent_write"); const internalPresentation = await issueService(db).addComment( conversation.issueId, "Internal interaction summary: continuationPolicy wake_assignee", { agentId: fixture.assignedAgentId, runId: sourceRunId }, { authorType: "agent", authorizationReason: sourcePresentationAuthorization, }, ); await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.id, internalPresentation.id)), ).resolves.toEqual([ { body: "Internal interaction summary: continuationPolicy wake_assignee", }, ]); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.commentId, internalPresentation.id)), ).resolves.toHaveLength(0); // External answer isolation cancels the parked source run before starting // its dedicated continuation. That technical handoff must not turn into a // provider-visible "stopped" message beside the still-actionable prompt. await db .update(heartbeatRuns) .set({ status: "cancelled", errorCode: "external_chat_continuation", updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, sourceRunId)); await enqueueChatRunMilestones(db); await expect( db .select() .from(chatPublications) .where( like( chatPublications.idempotencyKey, `run:${sourceRunId}:completed:%`, ), ), ).resolves.toHaveLength(0); await expect( db .select() .from(chatPublications) .where( like(chatPublications.idempotencyKey, `run:${sourceRunId}:failed:%`), ), ).resolves.toHaveLength(0); await db .update(heartbeatRuns) .set({ status: "succeeded", errorCode: null, updatedAt: new Date() }) .where(eq(heartbeatRuns.id, sourceRunId)); const postCountBeforePrompt = providerRuntime.posts.length; await service.processPendingPublications(); const originalPrompt = await db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `interaction:${interaction.id}:${endpoint.id}`, ), ) .then((rows) => rows[0]); expect(originalPrompt).toMatchObject({ state: "published", providerMessageId: workingPublication.providerMessageId, payload: { interactionId: interaction.id, progressState: "waiting_for_input", card: { kind: "question", title: "Which color should we use?" }, }, }); expect(providerRuntime.posts).toHaveLength(postCountBeforePrompt); expect(providerRuntime.edits.at(-1)).toMatchObject({ threadId: channel.thread.id, messageId: workingPublication.providerMessageId, text: expect.stringContaining("Which color should we use?"), }); await expect( db .select({ publicationId: chatMessageLinks.publicationId }) .from(chatMessageLinks) .where( and( eq(chatMessageLinks.endpointId, endpoint.id), eq(chatMessageLinks.conversationId, conversation.id), eq( chatMessageLinks.providerMessageId, workingPublication.providerMessageId, ), ), ), ).resolves.toEqual([{ publicationId: originalPrompt.id }]); const renderedProviderMessages = new Map([ [workingPublication.providerMessageId, providerRuntime.posts[0]?.text], ]); for (const edit of providerRuntime.edits) { renderedProviderMessages.set(edit.messageId, edit.text); } const renderedText = JSON.stringify([...renderedProviderMessages.values()]); expect(renderedText).toContain("Which color should we use?"); expect(renderedText).not.toContain("Maya is working"); expect(renderedText).not.toContain("continuationPolicy"); // A fast provider answer can settle the interaction before heartbeat's // presentation pass. The already provider-visible original prompt still // consumes the source run's external final, while the separate continuation // run remains eligible to publish its actual answer. await db .update(issueThreadInteractions) .set({ status: "answered", result: { version: 1, answers: [{ questionId: "color", optionIds: ["red"] }], }, resolvedAt: new Date(), updatedAt: new Date(), }) .where(eq(issueThreadInteractions.id, interaction.id)); await expect( resolveChatRunPresentationAuthorizationReason(db, { companyId: fixture.companyId, issueId: conversation.issueId, runId: sourceRunId, }), ).resolves.toBe("internal_agent_write"); const continuationRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: continuationRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: { issueId: conversation.issueId, taskId: conversation.issueId, source: "issue.interaction.respond", interactionId: interaction.id, sourceRunId, }, }); await db.insert(issueQuestionResponseDeliveries).values({ companyId: fixture.companyId, issueId: conversation.issueId, interactionId: interaction.id, sourceRunId, targetRunId: continuationRunId, correlationId: `chat-presentation-question:${interaction.id}`, payloadSha256: createHash("sha256").update(interaction.id).digest("hex"), status: "delivered", deliveryMode: "steered", }); const continuationAuthorization = await resolveChatRunPresentationAuthorizationReason(db, { companyId: fixture.companyId, issueId: conversation.issueId, runId: continuationRunId, }); expect(continuationAuthorization).toBe("allow_chat_run_presentation"); const continuationComment = await issueService(db).addComment( conversation.issueId, "COLOR-CONTINUATION-Red", { agentId: fixture.assignedAgentId, runId: continuationRunId }, { authorType: "agent", authorizationReason: continuationAuthorization, }, ); await service.processPendingPublications(); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.commentId, continuationComment.id)), ).resolves.toEqual([{ state: "published" }]); expect(JSON.stringify(providerRuntime.posts)).toContain( "COLOR-CONTINUATION-Red", ); await service.shutdown(); }); it("replaces Telegram queued progress with its native confirmation and rejects late working progress", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture); const externalUserId = "771234599"; const dm = makeThread({ channelId: externalUserId, id: `telegram:${externalUserId}`, isDM: true, name: "Telegram interaction presentation", }); const providerMessageId = `${externalUserId}:1`; await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: providerMessageId, text: "Ask before publishing the release", userId: externalUserId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, externalUserId); await service.test(endpoint.id, "owner-user"); const conversation = await db .select() .from(chatConversations) .where( and( eq(chatConversations.endpointId, endpoint.id), eq(chatConversations.externalThreadId, dm.thread.id), ), ) .then((rows) => rows[0]); if (!conversation) throw new Error("Expected Telegram conversation"); const sourceRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: sourceRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "telegram", providerMessageId, }), resultJson: { presentationDecision: { chosenSource: "final_agent_message", commentAction: "create", }, }, }); await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${sourceRunId}:queued:${endpoint.id}`, payload: { text: "Maya is queued.", progressState: "queued" }, state: "pending", }); await service.processPendingPublications(); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Telegram provider runtime"); const queuedPublication = await db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `run:${sourceRunId}:queued:${endpoint.id}`, ), ) .then((rows) => rows[0]); if (!queuedPublication?.providerMessageId) { throw new Error("Expected Telegram queued publication"); } const interaction = await issueThreadInteractionService(db).create( { id: conversation.issueId, companyId: fixture.companyId }, { kind: "request_confirmation", continuationPolicy: "wake_assignee", sourceRunId, title: "Release check", payload: { version: 1, prompt: "Publish the release now?", acceptLabel: "Publish", rejectLabel: "Wait", }, }, { agentId: fixture.assignedAgentId, runId: sourceRunId }, ); const authorization = await resolveChatRunPresentationAuthorizationReason( db, { companyId: fixture.companyId, issueId: conversation.issueId, runId: sourceRunId, }, ); expect(authorization).toBe("internal_agent_write"); const internalPresentation = await issueService(db).addComment( conversation.issueId, "Internal Telegram confirmation summary", { agentId: fixture.assignedAgentId, runId: sourceRunId }, { authorType: "agent", authorizationReason: authorization }, ); await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.id, internalPresentation.id)), ).resolves.toEqual([{ body: "Internal Telegram confirmation summary" }]); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.commentId, internalPresentation.id)), ).resolves.toHaveLength(0); await enqueueChatRunMilestones(db); await expect( db .select() .from(chatPublications) .where( like( chatPublications.idempotencyKey, `run:${sourceRunId}:completed:%`, ), ), ).resolves.toHaveLength(0); const postCountBeforePrompt = providerRuntime.posts.length; await service.processPendingPublications(); const promptPublication = await db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `interaction:${interaction.id}:${endpoint.id}`, ), ) .then((rows) => rows[0]); expect(promptPublication).toMatchObject({ state: "published", providerMessageId: queuedPublication.providerMessageId, payload: { interactionId: interaction.id, progressState: "waiting_for_input", card: { kind: "confirmation", title: "Publish the release now?" }, }, }); expect(providerRuntime.posts).toHaveLength(postCountBeforePrompt); expect(providerRuntime.edits.at(-1)).toMatchObject({ threadId: dm.thread.id, messageId: queuedPublication.providerMessageId, text: expect.stringContaining("Publish the release now?"), }); await expect( db .select({ publicationId: chatMessageLinks.publicationId }) .from(chatMessageLinks) .where( and( eq(chatMessageLinks.endpointId, endpoint.id), eq(chatMessageLinks.conversationId, conversation.id), eq( chatMessageLinks.providerMessageId, queuedPublication.providerMessageId, ), ), ), ).resolves.toEqual([{ publicationId: promptPublication.id }]); const renderedProviderMessages = new Map([ [queuedPublication.providerMessageId, providerRuntime.posts[0]?.text], ]); for (const edit of providerRuntime.edits) { renderedProviderMessages.set(edit.messageId, edit.text); } const renderedText = JSON.stringify([...renderedProviderMessages.values()]); expect(renderedText).toContain("Publish the release now?"); expect(renderedText).not.toContain("Maya is queued"); expect(renderedText).not.toContain( "Internal Telegram confirmation summary", ); const editsBeforeLateWorking = providerRuntime.edits.length; const postsBeforeLateWorking = providerRuntime.posts.length; // The milestone scan above raced a still-running source behind its newly // queued interaction. Drain that actual late placeholder rather than // inserting a duplicate fixture row. await service.processPendingPublications(); await expect( db .select({ attempts: chatPublications.attempts, redactedError: chatPublications.redactedError, state: chatPublications.state, }) .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `run:${sourceRunId}:working:${endpoint.id}`, ), ), ).resolves.toEqual([ { attempts: 0, redactedError: "Run progress was superseded by its provider interaction", state: "cancelled", }, ]); expect(providerRuntime.posts).toHaveLength(postsBeforeLateWorking); expect(providerRuntime.edits).toHaveLength(editsBeforeLateWorking); await expect( db .select({ publicationId: chatMessageLinks.publicationId }) .from(chatMessageLinks) .where( and( eq(chatMessageLinks.endpointId, endpoint.id), eq(chatMessageLinks.conversationId, conversation.id), eq( chatMessageLinks.providerMessageId, queuedPublication.providerMessageId, ), ), ), ).resolves.toEqual([{ publicationId: promptPublication.id }]); await service.shutdown(); }); it("acknowledges denied Telegram actions after one durable, payload-free notice", async () => { const fixture = await seedCompany(); const botId = Number.parseInt( fixture.companyId.replaceAll("-", "").slice(0, 12), 16, ); const botToken = `${botId}:telegram-denial-webhook-test`; const apiCalls: Array<{ body: string; method: string }> = []; let webhookSecret = ""; const telegramFetch = vi.fn( async (input: string | URL | Request, init?: RequestInit) => { const method = new URL(String(input)).pathname.split("/").at(-1) ?? ""; const body = typeof init?.body === "string" ? init.body : ""; apiCalls.push({ body, method }); if (method === "getMe") { return new Response( JSON.stringify({ ok: true, result: { id: botId, username: "paperclip_denial_test_bot", first_name: "Paperclip Denial Test", }, }), { status: 200, headers: { "content-type": "application/json" } }, ); } if (method === "getWebhookInfo") { return new Response( JSON.stringify({ ok: true, result: { url: "" } }), { status: 200, headers: { "content-type": "application/json" }, }, ); } if (method === "setWebhook") { webhookSecret = String( (JSON.parse(body) as { secret_token?: unknown }).secret_token ?? "", ); return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } if (method === "setMyCommands") { return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } if (method === "answerCallbackQuery") { return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } if (method === "sendMessage") { return new Response( JSON.stringify({ ok: true, result: { message_id: 9001, from: { id: botId, is_bot: true }, date: Math.floor(Date.now() / 1_000), chat: { id: 417200359, type: "private", first_name: "Telegram User", }, text: "This Paperclip action is no longer available.", }, }), { status: 200, headers: { "content-type": "application/json" } }, ); } throw new Error(`Unexpected Telegram API call: ${method}`); }, ) as typeof globalThis.fetch; const previousFetch = globalThis.fetch; globalThis.fetch = telegramFetch; const service = chatChannelService(db, { fetch: telegramFetch, heartbeat: { wakeup: receiptBackedWakeup(vi.fn(async () => ({ accepted: true }))), }, publicBaseUrl: "https://paperclip.example", }); try { const endpoint = await service.create( fixture.companyId, { provider: "telegram", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await service.configure( endpoint.id, { action: "configure", credentials: { botToken } }, "owner-user", ); expect(webhookSecret).not.toBe(""); await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, endpoint.id)); const actionId = "pcq:telegram-webhook-secret-token"; const callbackData = telegramChatSdkCallbackData(actionId); const providerRequest = ( updateId: number, callbackId: string, data = callbackData, ) => new Request( `https://paperclip.example/api/chat-webhooks/${endpoint.publicId}/telegram`, { method: "POST", headers: { "content-type": "application/json", "x-telegram-bot-api-secret-token": webhookSecret, }, body: JSON.stringify({ update_id: updateId, callback_query: { id: callbackId, data, from: { id: 417200359, is_bot: false, first_name: "Telegram User", username: "telegram-user", }, message: { message_id: 444, from: { id: botId, is_bot: true }, date: Math.floor(Date.now() / 1_000), chat: { id: 417200359, type: "private", first_name: "Telegram User", }, text: "Choose a priority", }, }, }), }, ); const first = await service.handleWebhook( endpoint.publicId, "telegram", providerRequest(7001, "denied-callback-1"), ); expect(first.status).toBe(200); await expect(first.text()).resolves.toBe("OK"); const providerRetry = await service.handleWebhook( endpoint.publicId, "telegram", providerRequest(7001, "denied-callback-1"), ); expect(providerRetry.status).toBe(200); await expect(providerRetry.text()).resolves.toBe("OK"); await vi.waitFor(() => { expect( apiCalls.filter(({ method }) => method === "sendMessage"), ).toHaveLength(1); }); await vi.waitFor(async () => { const effects = await db .select({ status: chatActions.status }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "provider_effect"), ), ); expect(effects).toEqual([{ status: "processed" }]); }); const unrecordedActionId = "pcq:durable-denial-insert-failure"; const unrecordedCallbackData = telegramChatSdkCallbackData(unrecordedActionId); const transactionSpy = vi .spyOn(db, "transaction") .mockRejectedValueOnce( new Error("injected denied-action persistence failure"), ); let unrecorded: Response; try { unrecorded = await service.handleWebhook( endpoint.publicId, "telegram", providerRequest( 7003, "denied-callback-persistence-failure", unrecordedCallbackData, ), ); } finally { transactionSpy.mockRestore(); } expect(unrecorded.status).toBe(503); expect(unrecorded.headers.get("retry-after")).toBe("1"); const notices = apiCalls.filter(({ method }) => method === "sendMessage"); expect(notices).toHaveLength(1); expect(JSON.parse(notices[0]!.body)).toMatchObject({ chat_id: "417200359", text: "This Paperclip action is no longer available. Open the linked task or ask an operator to link this account.", }); expect(notices[0]!.body).not.toContain(actionId); expect(notices[0]!.body).not.toContain(callbackData); expect(notices[0]!.body).not.toContain(unrecordedActionId); expect(notices[0]!.body).not.toContain(unrecordedCallbackData); expect(notices[0]!.body).not.toContain(webhookSecret); const denials = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "action"), ), ); expect(denials).toHaveLength(1); expect(denials[0]).toMatchObject({ state: "filtered", attempts: 1, redactedError: "External action denied by Paperclip authorization", normalizedEvent: { providerEventId: expect.stringMatching( /^action-denied:[a-f0-9]{64}$/, ), kind: "action", authorization: { outcome: "denied" }, }, }); const serializedDenial = JSON.stringify(denials[0]); expect(serializedDenial).not.toContain(actionId); expect(serializedDenial).not.toContain(callbackData); expect(serializedDenial).not.toContain(webhookSecret); } finally { await service.shutdown(); globalThis.fetch = previousFetch; } }); it("uses compact single-use Telegram action tokens and filters forged, expired, and oversized callbacks", async () => { const fixture = await seedCompany(); let pauseNextQuestionResolution = false; let releaseQuestionResolution!: () => void; let signalQuestionResolution!: () => void; const questionResolutionEntered = new Promise((resolve) => { signalQuestionResolution = resolve; }); const questionResolutionReleased = new Promise((resolve) => { releaseQuestionResolution = resolve; }); const { callbacks, endpoint, service } = await configuredTelegramEndpoint( fixture, { questionResolutionPersistBarrier: async () => { if (!pauseNextQuestionResolution) return; signalQuestionResolution(); await questionResolutionReleased; }, }, ); const externalUserId = "771234567"; const channel = makeThread({ channelId: externalUserId, id: `telegram:${externalUserId}`, isDM: true, name: "Maya direct message", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: channel.thread, message: makeMessage({ id: "tg-question-1", text: "Help me choose a priority", userId: externalUserId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const linkedUserId = `telegram-question-user-${randomUUID()}`; const now = new Date(); await db.insert(authUsers).values({ id: linkedUserId, name: "Telegram Question User", email: `${linkedUserId}@example.com`, emailVerified: true, createdAt: now, updatedAt: now, }); await db.insert(companyMemberships).values({ companyId: fixture.companyId, principalType: "user", principalId: linkedUserId, status: "active", membershipRole: "operator", }); const principal = await db .select() .from(chatExternalPrincipals) .where(eq(chatExternalPrincipals.externalId, externalUserId)) .then((rows) => rows[0]); const intent = await service.createLinkIntent( endpoint.id, principal.id, 1_800, ); const identityToken = new URL(intent.confirmationUrl).searchParams.get( "token", ); if (!identityToken) throw new Error("Telegram identity token was absent"); await service.confirmIdentityLink(identityToken, linkedUserId); async function publishQuestion(title: string) { const interaction = await issueThreadInteractionService(db).create( { id: conversation.issueId, companyId: fixture.companyId, }, { kind: "ask_user_questions", continuationPolicy: "wake_assignee", title, payload: { version: 1, title, questions: [ { id: "priority", prompt: "Which priority should we use?", selectionMode: "single", required: true, allowOther: false, options: [ { id: "high", label: "High" }, { id: "normal", label: "Normal" }, ], }, ], }, }, { agentId: fixture.assignedAgentId }, ); // This suite intentionally leaves some retryable fixtures behind. Drain // a bounded full-page backlog so this helper does not assume its question // is among the global oldest 25. A delivery scheduled by the preceding // answer may concurrently claim this exact row, so await its durable // published state instead of treating that in-flight claim as failure. await service.processPendingPublications(1_000); let publication: typeof chatPublications.$inferSelect | undefined; await vi.waitFor(async () => { publication = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), eq(chatPublications.issueId, conversation.issueId), ), ) .then((rows) => rows.find((row) => row.payload.interactionId === interaction.id), ); expect(publication).toMatchObject({ state: "published", providerMessageId: expect.any(String), }); }); if (!publication?.providerMessageId) throw new Error("Telegram question publication was not delivered"); const action = publication.payload.card?.actions?.find( (candidate) => candidate.type === "callback" && candidate.label === "High", ); if (!action || action.type !== "callback") throw new Error("Telegram callback action was not projected"); const token = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.providerActionId, action.actionId), ), ) .then((rows) => rows[0]); if (!token) throw new Error("Telegram action token was not persisted"); return { interaction, publication, action, token }; } const first = await publishQuestion("Choose the initial priority"); const callbackData = telegramChatSdkCallbackData(first.action.actionId); expect(Buffer.byteLength(callbackData, "utf8")).toBe(39); expect(Buffer.byteLength(callbackData, "utf8")).toBeLessThanOrEqual( TELEGRAM_CALLBACK_DATA_LIMIT_BYTES, ); expect(first.token).toMatchObject({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, principalId: null, kind: "question_answer", providerActionId: first.action.actionId, status: "issued", payload: { version: 1, publicationId: first.publication.id, interactionId: first.interaction.id, questionId: "priority", optionId: "high", expiresAt: expect.any(String), }, }); const actionEvent = (input: { actionId: string; callbackData: string; messageId?: string; }) => ({ endpointId: endpoint.id, provider: "telegram" as const, event: { actionId: input.actionId, adapter: {} as never, messageId: input.messageId ?? first.publication.providerMessageId!, openModal: async () => undefined, raw: { id: `callback-${randomUUID()}`, data: input.callbackData, from: { id: Number(externalUserId), first_name: "Telegram User" }, }, thread: channel.thread, threadId: channel.thread.id, user: { userId: externalUserId, userName: "telegram-user", fullName: "Telegram User", isBot: false, isMe: false, isSystem: false, }, value: undefined, }, }); const forgedActionId = "pcq:AAAAAAAAAAAAAAAAAAAAAA"; const forgedTelegramAction = actionEvent({ actionId: forgedActionId, callbackData: telegramChatSdkCallbackData(forgedActionId), }); await callbacks.onAction(forgedTelegramAction); // These hand-built events prove action authorization, not authenticated // Telegram response provenance. Their denials must not publish a fallback. await service.processPendingProviderEffects(); expect(channel.post).not.toHaveBeenCalled(); expect(JSON.stringify(channel.post.mock.calls)).not.toContain( forgedActionId, ); expect(JSON.stringify(channel.post.mock.calls)).not.toContain( telegramChatSdkCallbackData(forgedActionId), ); channel.post.mockClear(); await callbacks.onAction(forgedTelegramAction); expect(channel.post).not.toHaveBeenCalled(); const deniedTelegramActions = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "action"), ), ); expect(deniedTelegramActions).toEqual([ expect.objectContaining({ conversationId: null, principalId: principal.id, state: "filtered", attempts: 1, redactedError: "External action denied by Paperclip authorization", normalizedEvent: { providerEventId: expect.stringMatching( /^action-denied:[a-f0-9]{64}$/, ), kind: "action", authorization: { outcome: "denied" }, }, }), ]); expect(JSON.stringify(deniedTelegramActions[0])).not.toContain( forgedActionId, ); expect(JSON.stringify(deniedTelegramActions[0])).not.toContain( telegramChatSdkCallbackData(forgedActionId), ); expect(await service.listActivity(endpoint.id)).toEqual( expect.arrayContaining([ expect.objectContaining({ id: deniedTelegramActions[0]!.id, kind: "delivery", status: "filtered", summary: "action ignored", detail: "External action denied by Paperclip authorization", replayable: false, }), ]), ); const oversizedCallbackData = "x".repeat( TELEGRAM_CALLBACK_DATA_LIMIT_BYTES + 1, ); expect(Buffer.byteLength(oversizedCallbackData, "utf8")).toBe(65); await callbacks.onAction( actionEvent({ actionId: first.action.actionId, callbackData: oversizedCallbackData, }), ); await db .update(chatActions) .set({ payload: { ...first.token.payload, optionId: "forged-option" } }) .where(eq(chatActions.id, first.token.id)); await callbacks.onAction( actionEvent({ actionId: first.action.actionId, callbackData, }), ); await db .update(chatActions) .set({ payload: first.token.payload }) .where(eq(chatActions.id, first.token.id)); await callbacks.onAction( actionEvent({ actionId: first.action.actionId, callbackData, }), ); const [answered] = await db .select() .from(issueThreadInteractions) .where(eq(issueThreadInteractions.id, first.interaction.id)); expect(answered).toMatchObject({ status: "answered", resolvedByUserId: linkedUserId, result: { version: 1, answers: [{ questionId: "priority", optionIds: ["high"] }], }, }); await callbacks.onAction( actionEvent({ actionId: first.action.actionId, callbackData, }), ); expect( await db .select() .from(issueQuestionResponseDeliveries) .where( eq( issueQuestionResponseDeliveries.interactionId, first.interaction.id, ), ), ).toHaveLength(1); await service.processPendingProviderEffects(); expect(channel.post).not.toHaveBeenCalled(); channel.post.mockClear(); const authorization = await publishQuestion( "Choose a permission-sensitive priority", ); const authorizationCallbackData = telegramChatSdkCallbackData( authorization.action.actionId, ); await db .update(companyMemberships) .set({ membershipRole: "viewer" }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalId, linkedUserId), ), ); await callbacks.onAction( actionEvent({ actionId: authorization.action.actionId, callbackData: authorizationCallbackData, messageId: authorization.publication.providerMessageId!, }), ); await service.processPendingProviderEffects(); expect(channel.post).not.toHaveBeenCalled(); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, authorization.token.id)), ).resolves.toEqual([{ status: "issued" }]); await expect( db .select({ status: issueThreadInteractions.status }) .from(issueThreadInteractions) .where(eq(issueThreadInteractions.id, authorization.interaction.id)), ).resolves.toEqual([{ status: "pending" }]); await db .update(companyMemberships) .set({ membershipRole: "operator" }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalId, linkedUserId), ), ); channel.post.mockClear(); const authorizationRace = await publishQuestion( "Choose a priority while membership changes", ); const authorizationRaceCallbackData = telegramChatSdkCallbackData( authorizationRace.action.actionId, ); pauseNextQuestionResolution = true; const racedCallback = callbacks.onAction( actionEvent({ actionId: authorizationRace.action.actionId, callbackData: authorizationRaceCallbackData, messageId: authorizationRace.publication.providerMessageId!, }), ); await questionResolutionEntered; await db .update(companyMemberships) .set({ membershipRole: "viewer", updatedAt: new Date() }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalId, linkedUserId), ), ); releaseQuestionResolution(); await racedCallback; await service.processPendingProviderEffects(); expect(channel.post).not.toHaveBeenCalled(); await expect( db .select({ status: issueThreadInteractions.status }) .from(issueThreadInteractions) .where( eq(issueThreadInteractions.id, authorizationRace.interaction.id), ), ).resolves.toEqual([{ status: "pending" }]); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, authorizationRace.token.id)), ).resolves.toEqual([{ status: "issued" }]); await db .update(companyMemberships) .set({ membershipRole: "operator", updatedAt: new Date() }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalId, linkedUserId), ), ); channel.post.mockClear(); const expired = await publishQuestion("Choose an expired priority"); await db .update(chatActions) .set({ payload: { ...expired.token.payload, expiresAt: new Date(Date.now() - 1_000).toISOString(), }, }) .where(eq(chatActions.id, expired.token.id)); const expiredCallbackData = telegramChatSdkCallbackData( expired.action.actionId, ); await callbacks.onAction( actionEvent({ actionId: expired.action.actionId, callbackData: expiredCallbackData, messageId: expired.publication.providerMessageId!, }), ); await service.processPendingProviderEffects(); expect(channel.post).not.toHaveBeenCalled(); expect(JSON.stringify(channel.post.mock.calls)).not.toContain( expired.action.actionId, ); expect(JSON.stringify(channel.post.mock.calls)).not.toContain( expiredCallbackData, ); const [expiredToken] = await db .select() .from(chatActions) .where(eq(chatActions.id, expired.token.id)); expect(expiredToken).toMatchObject({ status: "expired", result: { code: "question_action_token_expired" }, }); const [stillPending] = await db .select() .from(issueThreadInteractions) .where(eq(issueThreadInteractions.id, expired.interaction.id)); expect(stillPending.status).toBe("pending"); }); it("renders safe Telegram confirmations as opaque one-use buttons and wakes the resolved task", async () => { const fixture = await seedCompany(); let injectSettlementCrash = true; const settlementBarrier = vi.fn(async () => { if (!injectSettlementCrash) return; injectSettlementCrash = false; throw new Error( "injected post-interaction confirmation settlement crash", ); }); const { callbacks, endpoint, service, wakeup } = await configuredTelegramEndpoint(fixture, { confirmationResolutionPersistBarrier: settlementBarrier, }); if (!callbacks.onAction) throw new Error("Telegram action callback was not registered"); const externalUserId = "771234568"; const dm = makeThread({ channelId: externalUserId, id: `telegram:${externalUserId}`, isDM: true, name: "Telegram confirmation DM", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: `${externalUserId}:1`, text: "Prepare a release confirmation", userId: externalUserId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, externalUserId); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Telegram confirmation conversation"); const linkedUserId = `telegram-confirmation-user-${randomUUID()}`; const now = new Date(); await db.insert(authUsers).values({ id: linkedUserId, name: "Telegram Confirmation User", email: `${linkedUserId}@example.com`, emailVerified: true, createdAt: now, updatedAt: now, }); await db.insert(companyMemberships).values({ companyId: fixture.companyId, principalType: "user", principalId: linkedUserId, status: "active", membershipRole: "operator", }); const principal = await db .select() .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, fixture.companyId), eq(chatExternalPrincipals.provider, "telegram"), eq(chatExternalPrincipals.externalId, externalUserId), ), ) .then((rows) => rows[0]); if (!principal) throw new Error("Expected Telegram confirmation principal"); const linkIntent = await service.createLinkIntent( endpoint.id, principal.id, 1_800, ); const linkToken = new URL(linkIntent.confirmationUrl).searchParams.get( "token", ); if (!linkToken) throw new Error("Telegram identity token was absent"); await service.confirmIdentityLink(linkToken, linkedUserId); const interaction = await issueThreadInteractionService(db).create( { id: conversation.issueId, companyId: fixture.companyId, }, { kind: "request_confirmation", continuationPolicy: "wake_assignee", title: "Release check", payload: { version: 1, prompt: "Proceed with Telegram production qualification?", acceptLabel: "Proceed", rejectLabel: "Stop", }, }, { agentId: fixture.assignedAgentId }, ); await service.processPendingPublications(); const publication = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), eq( chatPublications.idempotencyKey, `interaction:${interaction.id}:${endpoint.id}`, ), ), ) .then((rows) => rows[0]); expect(publication).toMatchObject({ state: "published", payload: { interactionId: interaction.id, card: { kind: "confirmation", title: "Proceed with Telegram production qualification?", actions: [ expect.objectContaining({ type: "callback", label: "Proceed", style: "primary", }), expect.objectContaining({ type: "callback", label: "Stop", style: "danger", }), ], }, }, }); if (!publication.providerMessageId) throw new Error("Telegram confirmation was not delivered"); const confirmationActions = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "confirmation_response"), ), ); expect(confirmationActions).toHaveLength(2); expect( confirmationActions.every( (action) => action.providerActionId.startsWith("pcq:") && Buffer.byteLength( telegramChatSdkCallbackData(action.providerActionId), "utf8", ) <= TELEGRAM_CALLBACK_DATA_LIMIT_BYTES, ), ).toBe(true); const acceptAction = confirmationActions.find( (action) => action.payload.decision === "accept", ); if (!acceptAction) throw new Error("Telegram accept action was not persisted"); wakeup.mockClear(); const actionEvent = { endpointId: endpoint.id, provider: "telegram" as const, event: { actionId: acceptAction.providerActionId, adapter: {} as never, messageId: publication.providerMessageId, openModal: async () => undefined, raw: { id: `callback-${randomUUID()}`, data: telegramChatSdkCallbackData(acceptAction.providerActionId), from: { id: Number(externalUserId), first_name: "Telegram User" }, }, thread: dm.thread, threadId: dm.thread.id, user: { userId: externalUserId, userName: "telegram-user", fullName: "Telegram User", isBot: false, isMe: false, isSystem: false, }, value: undefined, }, }; await expect(callbacks.onAction(actionEvent)).rejects.toThrow( "injected post-interaction confirmation settlement crash", ); expect(settlementBarrier).toHaveBeenCalledTimes(1); await expect( db .select({ status: issueThreadInteractions.status, resolvedByUserId: issueThreadInteractions.resolvedByUserId, }) .from(issueThreadInteractions) .where(eq(issueThreadInteractions.id, interaction.id)), ).resolves.toEqual([ { status: "accepted", resolvedByUserId: linkedUserId }, ]); await expect( db .select({ decision: sql`${chatActions.payload}->>'decision'`, status: chatActions.status, }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "confirmation_response"), ), ), ).resolves.toEqual( expect.arrayContaining([ { decision: "accept", status: "processing" }, { decision: "reject", status: "expired" }, ]), ); expect(wakeup).not.toHaveBeenCalled(); const crashWindowPublication = await db .select({ state: chatPublications.state }) .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `interaction-resolution:${interaction.id}:${endpoint.id}`, ), ) .then((rows) => rows[0]); expect(crashWindowPublication).toBeDefined(); expect(crashWindowPublication?.state).not.toBe("published"); expect( (service.runtime as unknown as FakeChatSdkRuntime).endpoints.get( endpoint.id, )?.edits, ).toHaveLength(0); const recoveryRuntime = new FakeChatSdkRuntime(); const recovery = createService( recoveryRuntime, fakeTelegramFetch() as typeof globalThis.fetch, { wakeup: async () => { expect( recoveryRuntime.endpoints.get(endpoint.id)?.edits, ).toHaveLength(1); return { accepted: true }; }, }, ); await recovery.service.processPendingPublications(); await vi.waitFor(async () => { await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), eq( chatPublications.idempotencyKey, `interaction-resolution:${interaction.id}:${endpoint.id}`, ), ), ), ).resolves.toEqual([{ state: "published" }]); }); const resolutionPublication = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), eq( chatPublications.idempotencyKey, `interaction-resolution:${interaction.id}:${endpoint.id}`, ), ), ) .then((rows) => rows[0]); expect(resolutionPublication).toMatchObject({ state: "published", providerMessageId: publication.providerMessageId, payload: { interactionId: interaction.id, card: { kind: "confirmation", title: "Proceed with Telegram production qualification?", body: "Accepted", }, }, }); const providerRuntime = recovery.runtime.endpoints.get(endpoint.id); expect(providerRuntime?.edits).toHaveLength(1); expect(providerRuntime?.edits[0]).toMatchObject({ messageId: publication.providerMessageId, }); expect(providerRuntime?.edits[0]?.text).toContain("Accepted"); expect(providerRuntime?.edits[0]?.text).not.toContain( acceptAction.providerActionId, ); await expect( db .select({ decision: sql`${chatActions.payload}->>'decision'`, status: chatActions.status, }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "confirmation_response"), ), ) .orderBy(asc(chatActions.createdAt), asc(chatActions.id)), ).resolves.toEqual( expect.arrayContaining([ { decision: "accept", status: "processed" }, { decision: "reject", status: "expired" }, ]), ); expect(recovery.wakeup).toHaveBeenCalledTimes(1); expect(recovery.wakeup).toHaveBeenCalledWith( fixture.assignedAgentId, expect.objectContaining({ allowRunCoalescing: false, idempotencyKey: `interaction:${interaction.id}:accepted`, reason: "issue_commented", payload: expect.objectContaining({ interactionId: interaction.id, interactionStatus: "accepted", }), }), ); await expect( db .select() .from(activityLog) .where( and( eq(activityLog.action, "issue.thread_interaction_accepted"), eq( sql`${activityLog.details}->>'interactionId'`, interaction.id, ), eq(sql`${activityLog.details}->>'source'`, "external_chat"), ), ), ).resolves.toHaveLength(1); const providerPostCountBeforeRedelivery = dm.post.mock.calls.length; const providerEffectCountBeforeRedelivery = await db .select({ id: chatActions.id }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "provider_effect"), ), ) .then((rows) => rows.length); await callbacks.onAction(actionEvent); await new Promise((resolve) => setImmediate(resolve)); expect(dm.post).toHaveBeenCalledTimes(providerPostCountBeforeRedelivery); await expect( db .select({ id: chatActions.id }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "provider_effect"), ), ) .then((rows) => rows.length), ).resolves.toBe(providerEffectCountBeforeRedelivery); await recovery.service.processPendingPublications(); expect(recovery.wakeup).toHaveBeenCalledTimes(1); const confirmationWake = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "interaction_wakeup"), ), ) .then((rows) => rows[0]); if (!confirmationWake) throw new Error("Expected durable confirmation wake"); await db .update(chatActions) .set({ status: "processing", updatedAt: new Date(Date.now() - 60_000) }) .where(eq(chatActions.id, confirmationWake.id)); await db.insert(agentWakeupRequests).values({ companyId: fixture.companyId, agentId: fixture.assignedAgentId, source: "automation", reason: "issue_commented", status: "queued", idempotencyKey: `interaction:${interaction.id}:accepted`, }); recovery.wakeup.mockClear(); await recovery.service.processPendingPublications(); expect(recovery.wakeup).not.toHaveBeenCalled(); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, confirmationWake.id)), ).resolves.toEqual([ { status: "processed", result: { code: "interaction_wakeup_already_durable" }, }, ]); const boardRace = await issueThreadInteractionService(db).create( { id: conversation.issueId, companyId: fixture.companyId }, { kind: "request_confirmation", continuationPolicy: "wake_assignee", payload: { version: 1, prompt: "Let the board resolve after Telegram claims this action?", }, }, { agentId: fixture.assignedAgentId }, ); await service.processPendingPublications(); const boardRaceActions = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "confirmation_response"), eq( sql`${chatActions.payload}->>'interactionId'`, boardRace.id, ), ), ); const boardRaceAccept = boardRaceActions.find( (action) => action.payload.decision === "accept", ); if (!boardRaceAccept) throw new Error("Expected board-race Telegram accept action"); await db .update(chatActions) .set({ principalId: principal.id, status: "processing", updatedAt: new Date(), }) .where( and( eq(chatActions.id, boardRaceAccept.id), eq(chatActions.status, "issued"), ), ); const boardIssue = await db .select() .from(issues) .where( and( eq(issues.companyId, fixture.companyId), eq(issues.id, conversation.issueId), ), ) .then((rows) => rows[0]); if (!boardIssue) throw new Error("Expected board-race issue"); await issueThreadInteractionService(db).acceptInteraction( boardIssue, boardRace.id, {}, { userId: linkedUserId }, ); await db.insert(agentWakeupRequests).values({ companyId: fixture.companyId, agentId: fixture.replacementAgentId, source: "automation", reason: "issue_commented", status: "queued", idempotencyKey: `interaction:${boardRace.id}:accepted`, }); recovery.wakeup.mockClear(); await recovery.service.processPendingPublications(); expect(recovery.wakeup).not.toHaveBeenCalled(); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where( and( eq(chatActions.kind, "interaction_wakeup"), eq( sql`${chatActions.payload}->>'interactionId'`, boardRace.id, ), ), ), ).resolves.toEqual([ { status: "processed", result: { code: "interaction_wakeup_coalesced_after_reassignment", }, }, ]); await expect( db .select({ decision: sql`${chatActions.payload}->>'decision'`, status: chatActions.status, }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "confirmation_response"), eq( sql`${chatActions.payload}->>'interactionId'`, boardRace.id, ), ), ), ).resolves.toEqual( expect.arrayContaining([ { decision: "accept", status: "processed" }, { decision: "reject", status: "expired" }, ]), ); await expect( db .select({ state: chatPublications.state, payload: chatPublications.payload, }) .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `interaction-resolution:${boardRace.id}:${endpoint.id}`, ), ), ).resolves.toEqual([ expect.objectContaining({ state: "published", payload: expect.objectContaining({ interactionId: boardRace.id }), }), ]); await expect( db .select() .from(activityLog) .where( and( eq(activityLog.action, "issue.thread_interaction_accepted"), eq( sql`${activityLog.details}->>'interactionId'`, boardRace.id, ), eq(sql`${activityLog.details}->>'source'`, "external_chat"), ), ), ).resolves.toHaveLength(0); const failingWakeup = vi.fn(async () => { throw new Error("injected permanent confirmation wake failure"); }); const retryingWakeService = createService( new FakeChatSdkRuntime(), fakeTelegramFetch() as typeof globalThis.fetch, { wakeup: failingWakeup }, ); // These rows exercise the recovery worker itself. Insert them directly so // the original endpoint service's interaction-created subscription cannot // race this deliberately failing worker and queue the continuation first. const [retryingInteraction] = await db .insert(issueThreadInteractions) .values({ companyId: fixture.companyId, issueId: conversation.issueId, kind: "request_confirmation", continuationPolicy: "none", createdByAgentId: fixture.assignedAgentId, status: "accepted", resolvedByUserId: linkedUserId, resolvedAt: new Date(), payload: { version: 1, prompt: "Exercise interaction wake retry exhaustion", }, }) .returning(); const retryingWakeId = randomUUID(); await db.insert(chatActions).values({ id: retryingWakeId, companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, principalId: principal.id, kind: "interaction_wakeup", providerActionId: `interaction_wakeup:${randomUUID()}`, payload: { version: 1, interactionId: retryingInteraction.id, interactionKind: "request_confirmation", interactionStatus: "accepted", issueId: conversation.issueId, agentId: fixture.assignedAgentId, sourceCommentId: null, sourceRunId: null, requestedByUserId: linkedUserId, }, status: "issued", }); try { await retryingWakeService.service.processPendingPublications(1_000); expect(failingWakeup).toHaveBeenCalledTimes(1); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, retryingWakeId)), ).resolves.toEqual([ { status: "issued", result: { code: "interaction_wakeup_failed", attemptCount: 1 }, }, ]); await retryingWakeService.service.processPendingPublications(1_000); expect(failingWakeup).toHaveBeenCalledTimes(1); for (let attempt = 2; attempt <= 5; attempt += 1) { await db .update(chatActions) .set({ updatedAt: new Date(Date.now() - 31_000) }) .where(eq(chatActions.id, retryingWakeId)); await retryingWakeService.service.processPendingPublications(1_000); } expect(failingWakeup).toHaveBeenCalledTimes(5); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, retryingWakeId)), ).resolves.toEqual([ { status: "issued", result: { code: "interaction_wakeup_failed", attemptCount: 5 }, }, ]); await db .update(chatActions) .set({ updatedAt: new Date(Date.now() - 31_000) }) .where(eq(chatActions.id, retryingWakeId)); await retryingWakeService.service.processPendingPublications(1_000); expect(failingWakeup).toHaveBeenCalledTimes(6); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, retryingWakeId)), ).resolves.toEqual([ { status: "issued", result: { code: "interaction_wakeup_failed", attemptCount: 6 }, }, ]); } finally { try { await retryingWakeService.service.shutdown(); } finally { // This intentionally retryable synthetic action must not become due // during a later test's global drain. Remove only this fixture row; // production shutdown must preserve durable retry work. await db .delete(chatActions) .where( and( eq(chatActions.id, retryingWakeId), eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "interaction_wakeup"), ), ); } } await expect( db .select({ id: chatActions.id }) .from(chatActions) .where(eq(chatActions.id, retryingWakeId)), ).resolves.toEqual([]); const [deferredInteraction] = await db .insert(issueThreadInteractions) .values({ companyId: fixture.companyId, issueId: conversation.issueId, kind: "request_confirmation", continuationPolicy: "none", createdByAgentId: fixture.assignedAgentId, status: "accepted", resolvedByUserId: linkedUserId, resolvedAt: new Date(), payload: { version: 1, prompt: "Exercise a scheduler-deferred interaction wake", }, }) .returning(); const deferredWakeId = randomUUID(); await db.insert(chatActions).values({ id: deferredWakeId, companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, principalId: principal.id, kind: "interaction_wakeup", providerActionId: `interaction_wakeup:${randomUUID()}`, payload: { version: 1, interactionId: deferredInteraction.id, interactionKind: "request_confirmation", interactionStatus: "accepted", issueId: conversation.issueId, agentId: fixture.assignedAgentId, sourceCommentId: null, sourceRunId: null, requestedByUserId: linkedUserId, }, status: "issued", }); const deferredWakeup = vi .fn() .mockResolvedValueOnce(null) .mockResolvedValueOnce({ accepted: true }); const deferredWakeService = createService( new FakeChatSdkRuntime(), fakeTelegramFetch() as typeof globalThis.fetch, { wakeup: deferredWakeup }, ); await deferredWakeService.service.processPendingPublications(); expect(deferredWakeup).toHaveBeenCalledTimes(1); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, deferredWakeId)), ).resolves.toEqual([ { status: "issued", result: { code: "interaction_wakeup_deferred" }, }, ]); await deferredWakeService.service.processPendingPublications(); expect(deferredWakeup).toHaveBeenCalledTimes(1); await db .update(chatActions) .set({ updatedAt: new Date(Date.now() - 31_000) }) .where(eq(chatActions.id, deferredWakeId)); await deferredWakeService.service.processPendingPublications(); expect(deferredWakeup).toHaveBeenCalledTimes(2); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, deferredWakeId)), ).resolves.toEqual([ { status: "processed", result: { code: "interaction_wakeup_queued" }, }, ]); await deferredWakeService.service.shutdown(); const governed = await issueThreadInteractionService(db).create( { id: conversation.issueId, companyId: fixture.companyId }, { kind: "request_confirmation", continuationPolicy: "wake_assignee", payload: { version: 1, prompt: "Explain why this should be rejected", rejectRequiresReason: true, }, }, { agentId: fixture.assignedAgentId }, ); await service.processPendingPublications(); const guardedPublication = await db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `interaction:${governed.id}:${endpoint.id}`, ), ) .then((rows) => rows[0]); expect( guardedPublication.payload.card?.actions?.some( (action) => action.type === "callback", ) ?? false, ).toBe(false); expect( await db .select() .from(chatActions) .where( and( eq(chatActions.kind, "confirmation_response"), eq( sql`${chatActions.payload}->>'interactionId'`, governed.id, ), ), ), ).toHaveLength(0); }); it("publishes complex question sets as non-executable Paperclip fallbacks", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service } = await configuredSlackEndpoint(fixture); const channel = makeThread({ channelId: "C-COMPLEX-QUESTION", id: "slack:C-COMPLEX-QUESTION:4550.1", name: "complex-questions", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "4550.1", text: "@maya ask for several inputs", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const interaction = await issueThreadInteractionService(db).create( { id: conversation.issueId, companyId: fixture.companyId }, { kind: "ask_user_questions", payload: { version: 1, questions: [ { id: "regions", prompt: "Which regions?", selectionMode: "multi", required: true, allowOther: false, options: [ { id: "us", label: "US" }, { id: "eu", label: "EU" }, ], }, { id: "notes", prompt: "Any constraints?", selectionMode: "single", required: false, allowOther: true, options: [ { id: "other", label: "Describe them", freeText: true }, ], }, ], }, }, { agentId: fixture.assignedAgentId }, ); const publications = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), eq(chatPublications.issueId, conversation.issueId), ), ); const publication = publications.find( (candidate) => candidate.payload.interactionId === interaction.id, ); expect(publication).toBeDefined(); if (!publication) throw new Error("Expected interaction publication"); expect(publication.payload).toMatchObject({ interactionId: interaction.id, }); expect( publication.payload.card?.actions?.filter( (action) => action.type === "callback", ) ?? [], ).toEqual([]); expect(publication.payload.text).toContain( "Open the task in Paperclip to respond", ); expect(callbacks.onModalSubmit).toBeTypeOf("function"); expect(callbacks.onModalClose).toBeUndefined(); expect(callbacks.onReaction).toBeTypeOf("function"); }); it("publishes GitHub questions as link-only cards with no executable callback", async () => { const fixture = await seedCompany(); const previousPublicUrl = process.env.PAPERCLIP_PUBLIC_URL; process.env.PAPERCLIP_PUBLIC_URL = "https://paperclip.example"; try { const { callbacks, endpoint, runtime, service } = await configuredGitHubEndpoint(fixture); const thread = makeThread({ channelId: "paperclipai/paperclip", id: "github:paperclipai/paperclip:issue:451", name: "paperclipai/paperclip", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id: "45101", text: "@maya ask me for a release decision", mentioned: true, }), trigger: "mention", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const interaction = await issueThreadInteractionService(db).create( { id: conversation!.issueId, companyId: fixture.companyId }, { kind: "ask_user_questions", payload: { version: 1, questions: [ { id: "release", prompt: "Ship this release?", selectionMode: "single", required: true, allowOther: false, options: [ { id: "ship", label: "Ship" }, { id: "hold", label: "Hold" }, ], }, ], }, }, { agentId: fixture.assignedAgentId }, ); await service.processPendingPublications(); const [publication] = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), eq( chatPublications.idempotencyKey, `interaction:${interaction.id}:${endpoint.id}`, ), ), ); expect(publication).toMatchObject({ state: "published", payload: { interactionId: interaction.id, card: { actions: [ { type: "link", label: "Open in Paperclip", url: `https://paperclip.example/issues/${conversation!.issueId}`, }, ], }, }, }); expect(callbacks.onAction).toBeUndefined(); expect( await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "question_answer"), ), ), ).toHaveLength(0); expect( JSON.stringify(runtime.endpoints.get(endpoint.id)?.posts), ).toContain(`https://paperclip.example/issues/${conversation!.issueId}`); } finally { if (previousPublicUrl === undefined) delete process.env.PAPERCLIP_PUBLIC_URL; else process.env.PAPERCLIP_PUBLIC_URL = previousPublicUrl; } }); it("returns a successful GitHub link-question continuation as one exact final reply", async () => { const fixture = await seedCompany(); const previousPublicUrl = process.env.PAPERCLIP_PUBLIC_URL; process.env.PAPERCLIP_PUBLIC_URL = "https://paperclip.example"; try { const continuationRunId = randomUUID(); const wakeup = vi.fn(async (agentId, options) => { if (options.contextSnapshot?.source !== "issue.interaction.respond") { return { accepted: true }; } const [created] = await db .insert(heartbeatRuns) .values({ id: continuationRunId, companyId: fixture.companyId, agentId, status: "running", contextSnapshot: options.contextSnapshot, }) .onConflictDoNothing() .returning(); return ( created ?? db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, continuationRunId)) .then((rows) => rows[0]) ); }); const { callbacks, endpoint, runtime, service } = await configuredGitHubEndpoint(fixture, { wakeup }); const thread = makeThread({ channelId: "paperclipai/paperclip", id: "github:paperclipai/paperclip:issue:455", name: "paperclipai/paperclip", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id: "45501", text: "@maya ask for and apply the release color", mentioned: true, }), trigger: "mention", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); if (!conversation) throw new Error("Expected GitHub conversation"); const sourceRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: sourceRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "github", providerMessageId: "45501", }), }); const interaction = await issueThreadInteractionService(db).create( { id: conversation.issueId, companyId: fixture.companyId }, { kind: "ask_user_questions", continuationPolicy: "wake_assignee", sourceRunId, title: "Choose the release color", payload: { version: 1, questions: [ { id: "color", prompt: "Which release color should we use?", selectionMode: "single", required: true, allowOther: false, options: [ { id: "blue", label: "Blue" }, { id: "green", label: "Green" }, ], }, ], }, }, { agentId: fixture.assignedAgentId, runId: sourceRunId }, ); await service.processPendingPublications(); const promptPublication = await db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `interaction:${interaction.id}:${endpoint.id}`, ), ) .then((rows) => rows[0]); expect(promptPublication).toMatchObject({ state: "published", payload: { interactionId: interaction.id, progressState: "waiting_for_input", card: { actions: [ expect.objectContaining({ label: "Open in Paperclip", type: "link", }), ], }, }, }); await issueThreadInteractionService(db).answerQuestions( { id: conversation.issueId, companyId: fixture.companyId }, interaction.id, { answers: [{ questionId: "color", optionIds: ["blue"] }] }, { userId: "owner-user" }, ); await questionResponseDeliveryService(db, { heartbeat: { cancelRun: vi.fn(), wakeup, } as never, }).deliver(interaction.id); await service.processPendingPublications(); await vi.waitFor(() => expect( wakeup.mock.calls.some( ([, options]) => options.contextSnapshot?.source === "issue.interaction.respond", ), ).toBe(true), ); await enqueueChatRunMilestones(db); await service.processPendingPublications(); const workingPublication = await db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `run:${continuationRunId}:working:${endpoint.id}`, ), ) .then((rows) => rows[0]); expect(workingPublication).toMatchObject({ state: "published", providerMessageId: expect.any(String), }); await db .update(heartbeatRuns) .set({ status: "succeeded", updatedAt: new Date() }) .where(eq(heartbeatRuns.id, continuationRunId)); const authorizationReason = await resolveChatRunPresentationAuthorizationReason(db, { companyId: fixture.companyId, issueId: conversation.issueId, runId: continuationRunId, }); expect(authorizationReason).toBe("allow_chat_run_presentation"); const finalComment = await issueService(db).addComment( conversation.issueId, "GITHUB-COLOR-Blue", { agentId: fixture.assignedAgentId, runId: continuationRunId }, { authorType: "agent", authorizationReason }, ); await service.processPendingPublications(); const finalPublication = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, finalComment.id)) .then((rows) => rows[0]); expect(finalPublication).toMatchObject({ conversationId: conversation.id, endpointId: endpoint.id, state: "published", payload: { text: "GITHUB-COLOR-Blue" }, providerMessageId: workingPublication.providerMessageId, }); const providerRuntime = runtime.endpoints.get(endpoint.id); expect( providerRuntime?.posts.filter( (post) => post.text === "GITHUB-COLOR-Blue", ), ).toHaveLength(0); expect( providerRuntime?.edits.filter( (edit) => edit.text === "GITHUB-COLOR-Blue", ), ).toEqual([ expect.objectContaining({ messageId: finalPublication.providerMessageId, threadId: thread.thread.id, }), ]); const runPublications = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), eq(chatPublications.conversationId, conversation.id), like(chatPublications.idempotencyKey, `run:${continuationRunId}:%`), ), ); expect(runPublications).toEqual([ expect.objectContaining({ providerMessageId: finalPublication.providerMessageId, state: "published", }), ]); await service.shutdown(); } finally { if (previousPublicUrl === undefined) delete process.env.PAPERCLIP_PUBLIC_URL; else process.env.PAPERCLIP_PUBLIC_URL = previousPublicUrl; } }); it("returns a failed link-only interaction continuation to its GitHub thread", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredGitHubEndpoint(fixture); const thread = makeThread({ channelId: "paperclipai/paperclip", id: "github:paperclipai/paperclip:issue:454", name: "paperclipai/paperclip", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id: "45401", text: "@maya pause for a release confirmation", mentioned: true, }), trigger: "mention", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); if (!conversation) throw new Error("Expected GitHub conversation"); const sourceCommentId = await db .select({ commentId: chatMessageLinks.commentId }) .from(chatMessageLinks) .where( and( eq(chatMessageLinks.conversationId, conversation.id), eq(chatMessageLinks.direction, "inbound"), isNotNull(chatMessageLinks.commentId), ), ) .then((rows) => rows[0]?.commentId ?? null); if (!sourceCommentId) throw new Error("Expected GitHub source comment"); const interaction = await issueThreadInteractionService(db).create( { id: conversation.issueId, companyId: fixture.companyId }, { kind: "request_confirmation", continuationPolicy: "wake_assignee", sourceCommentId, payload: { version: 1, prompt: "Ship this release?" }, }, { agentId: fixture.assignedAgentId }, ); await service.processPendingPublications(); wakeup.mockClear(); const issue = await db .select() .from(issues) .where(eq(issues.id, conversation.issueId)) .then((rows) => rows[0]); if (!issue) throw new Error("Expected GitHub-backed issue"); await issueThreadInteractionService(db).acceptInteraction( issue, interaction.id, {}, { userId: "owner-user" }, ); await service.processPendingPublications(); expect(wakeup).toHaveBeenCalledOnce(); const continuation = wakeup.mock.calls[0]?.[1]; expect(continuation).toMatchObject({ contextSnapshot: { issueId: conversation.issueId, source: "chat:github", sourceCommentId, wakeCommentId: sourceCommentId, wakeCommentIds: [sourceCommentId], }, payload: { issueId: conversation.issueId, sourceCommentId, wakeCommentId: sourceCommentId, wakeCommentIds: [sourceCommentId], }, }); const wakeCommentId = ( continuation?.contextSnapshot as { wakeCommentId?: string } | undefined )?.wakeCommentId; if (!wakeCommentId) throw new Error("Expected GitHub wake comment link"); await expect( db .select({ commentId: chatMessageLinks.commentId }) .from(chatMessageLinks) .where( and( eq(chatMessageLinks.conversationId, conversation.id), eq(chatMessageLinks.direction, "inbound"), eq(chatMessageLinks.commentId, wakeCommentId), ), ), ).resolves.toHaveLength(1); const failedRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: failedRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "failed", errorCode: "continuation_failed", contextSnapshot: continuation!.contextSnapshot, }); await expect( enqueueChatRunMilestones(db, { publicBaseUrl: "https://paperclip.example", }), ).resolves.toBe(1); await service.processPendingPublications(); const failedPublication = await db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `run:${failedRunId}:failed:${endpoint.id}`, ), ) .then((rows) => rows[0]); expect(failedPublication).toMatchObject({ conversationId: conversation.id, state: "published", }); expect(failedPublication?.payload.text).toContain( "stopped before completing this turn", ); expect( runtime.endpoints .get(endpoint.id) ?.posts.some((post) => post.text.includes("stopped before completing this turn"), ), ).toBe(true); await service.shutdown(); }); it("publishes truthful GitHub attachment fallbacks without provider file bytes", async () => { for (const testCase of [ { publicBaseUrl: "https://board.paperclip.example", expectedFallback: (issueId: string) => `File saved on the Paperclip task: report.txt. This GitHub App connection cannot upload file bytes into comments. Download it: https://board.paperclip.example/issues/${issueId}`, }, { publicBaseUrl: "http://127.0.0.1:3103", expectedFallback: () => "File saved on the private Paperclip task: report.txt. This GitHub App connection cannot upload file bytes into comments.", }, ]) { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, runtime, service } = await configuredGitHubEndpoint(fixture, { publicBaseUrl: testCase.publicBaseUrl, storage: storage.storage, }); const thread = makeThread({ channelId: "paperclipai/paperclip", id: "github:paperclipai/paperclip:issue:452", name: "paperclipai/paperclip", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id: "45201", text: "@maya create a downloadable report", mentioned: true, }), trigger: "mention", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation!.issueId, provider: "github", providerMessageId: "45201", }), }); const comment = await addSelectedChatFinal({ agentId: fixture.assignedAgentId, body: "The report is ready.", companyId: fixture.companyId, issueId: conversation!.issueId, runId, }); const stored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${conversation!.issueId}`, originalFilename: "report.txt", contentType: "text/plain", body: Buffer.from("report contents", "utf8"), }); await issueService(db).createAttachment({ issueId: conversation!.issueId, issueCommentId: comment.id, provider: stored.provider, objectKey: stored.objectKey, contentType: stored.contentType, byteSize: stored.byteSize, sha256: stored.sha256, originalFilename: stored.originalFilename, createdByAgentId: fixture.assignedAgentId, createdByRunId: runId, }); await service.processPendingPublications(); const posts = runtime.endpoints.get(endpoint.id)?.posts ?? []; expect( posts.some((post) => post.text.includes(testCase.expectedFallback(conversation!.issueId)), ), ).toBe(true); expect( posts.every((post) => !post.text.includes("Shared report.txt.")), ).toBe(true); expect(posts.every((post) => post.files === undefined)).toBe(true); expect(storage.storage.getObject).not.toHaveBeenCalled(); if (testCase.publicBaseUrl.startsWith("http://127.0.0.1")) { expect( posts.every((post) => !post.text.includes(testCase.publicBaseUrl)), ).toBe(true); } await service.shutdown(); } }); it("publishes one truthful notice when a selected PNG receives a definite provider rejection", async () => { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture, { storage: storage.storage, }); const channel = makeThread({ channelId: "C-ATTACHMENT-REJECTED", id: "slack:C-ATTACHMENT-REJECTED:4521.1", name: "attachment-rejected", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "4521.1", text: "@maya start an attachment failure test", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Slack file conversation"); const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); const stored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${conversation.issueId}`, originalFilename: "provider-rejected.png", contentType: "image/png", body: png, }); const attachment = await issueService(db).createAttachment({ issueId: conversation.issueId, provider: stored.provider, objectKey: stored.objectKey, contentType: stored.contentType, byteSize: stored.byteSize, sha256: stored.sha256, originalFilename: stored.originalFilename, createdByUserId: "owner-user", }); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack provider runtime"); providerRuntime.posts.length = 0; let postAttempt = 0; providerRuntime.postHook = async () => { postAttempt += 1; if (postAttempt !== 2) return; throw Object.assign(new Error("Slack rejected the PNG upload"), { adapter: "slack", code: "slack_webapi_platform_error", data: { error: "invalid_arguments" }, }); }; const failed = await service.publishBoardMessage( endpoint.id, conversation.id, "The selected PNG should follow.", "selected-png-provider-rejection", "owner-user", [attachment.id], ); providerRuntime.postHook = undefined; expect(failed).toMatchObject({ state: "failed" }); const selectedAttachmentNotice = "Paperclip could not send an attachment. The file remains on its Paperclip task for an operator to retry." + ` Open task: https://paperclip.example/issues/${conversation.issueId}`; const publications = await db .select() .from(chatPublications) .where( and( eq(chatPublications.companyId, fixture.companyId), eq(chatPublications.conversationId, conversation.id), or( eq(chatPublications.commentId, failed.commentId!), like( chatPublications.idempotencyKey, "attachment-failure-notice:%", ), ), ), ) .orderBy(asc(chatPublications.createdAt)); expect(publications).toEqual([ expect.objectContaining({ state: "published", payload: { text: "The selected PNG should follow." }, }), expect.objectContaining({ id: failed.id, state: "failed", payload: expect.objectContaining({ attachmentIds: [attachment.id] }), }), expect.objectContaining({ commentId: null, state: "published", payload: { text: selectedAttachmentNotice, }, }), ]); expect(providerRuntime.posts).toEqual([ { threadId: channel.thread.id, text: "The selected PNG should follow.", }, { threadId: channel.thread.id, text: selectedAttachmentNotice, }, ]); expect(storage.storage.getObject).toHaveBeenCalledOnce(); await service.processPendingPublications(); expect(providerRuntime.posts).toHaveLength(2); await service.shutdown(); }); it("preserves the raw webhook request and returns the provider adapter response", async () => { const fixture = await seedCompany(); const { endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected configured fake provider runtime"); const payload = JSON.stringify({ type: "event_callback", event_id: "Ev-123", }); // Both requests use one explicitly ready listener, as in the publication // status fixture below. Implicit per-request Supertest listeners can reset // before Express receives the request under the loaded macOS suite. const server = createServer(webhookApp(service)); await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", () => { server.off("error", reject); resolve(); }); }); try { const response = await request(server) .post(`/api/chat-webhooks/${endpoint.publicId}/slack`) .set("content-type", "application/json") .set("x-slack-signature", "v0=test-signature") .send(payload) .expect(202); expect(response.text).toBe("accepted"); expect(response.headers["x-chat-test"]).toBe("accepted"); expect( providerRuntime.webhookRequest?.headers.get("x-slack-signature"), ).toBe("v0=test-signature"); await expect(providerRuntime.webhookRequest?.text()).resolves.toBe( payload, ); await request(server) .post(`/api/chat-webhooks/${endpoint.publicId}/irc`) .set("content-type", "application/json") .send("{}") .expect(400); } finally { await new Promise((resolve, reject) => { server.close((error) => (error ? reject(error) : resolve())); }); } }); it("retires superseded credentials and clears endpoint-owned secrets on removal", async () => { const fixture = await seedCompany(); const { service } = createService(); const endpoint = await service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId, }, "owner-user", ); await service.configure( endpoint.id, { action: "configure", credentials: { botToken: "xoxb-first", signingSecret: "first-secret" }, }, "owner-user", ); const [firstConnection] = await db .select() .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); const firstIds = new Set( firstConnection.credentialSecretRefs.map((ref) => ref.secretId), ); expect(firstIds.size).toBe(2); await service.configure( endpoint.id, { action: "reconnect", credentials: { botToken: "xoxb-second", signingSecret: "second-secret", }, }, "owner-user", ); const [rotatedConnection] = await db .select() .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(rotatedConnection.credentialSecretRefs).toHaveLength(2); expect( rotatedConnection.credentialSecretRefs.every( (ref) => !firstIds.has(ref.secretId), ), ).toBe(true); const afterRotation = await db .select() .from(companySecrets) .where(eq(companySecrets.companyId, fixture.companyId)); expect(afterRotation.map((secret) => secret.id).sort()).toEqual( rotatedConnection.credentialSecretRefs.map((ref) => ref.secretId).sort(), ); await service.configure(endpoint.id, { action: "remove" }, "owner-user"); const [removedConnection] = await db .select() .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(removedConnection.credentialSecretRefs).toEqual([]); expect( await db .select() .from(companySecrets) .where(eq(companySecrets.companyId, fixture.companyId)), ).toHaveLength(0); }); it.each([ { provider: "github" as const, label: "GitHub" }, { provider: "microsoft-teams" as const, label: "Microsoft Teams" }, ])( "removes a $label endpoint without deleting its task or audit history", async ({ provider }) => { const fixture = await seedCompany(); const configured = provider === "github" ? await configuredGitHubEndpoint(fixture) : await configuredTeamsEndpoint(fixture); const { callbacks, endpoint, runtime, service } = configured; const thread = provider === "github" ? makeThread({ channelId: "paperclipai/paperclip", id: "github:paperclipai/paperclip:issue:97", name: "paperclipai/paperclip", }) : makeThread({ channelId: "teams-personal-cleanup", id: "teams:personal-cleanup", isDM: true, name: "Teams cleanup DM", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider, thread: thread.thread, message: makeMessage({ id: `${provider}-cleanup-root`, raw: provider === "microsoft-teams" ? { from: { aadObjectId: "097269c2-77cb-4d6e-9be4-9a6362969699" }, } : undefined, text: provider === "github" ? "@maya preserve this GitHub task" : "Preserve this Teams task", mentioned: provider === "github", userId: provider === "github" ? "97001" : "29:teams-cleanup-user", }), trigger: provider === "github" ? "mention" : "direct_message", }); const [conversation] = await service.listConversations(endpoint.id); expect(conversation).toBeDefined(); const credentialCount = await db .select({ id: companySecrets.id }) .from(companySecrets) .where(eq(companySecrets.companyId, fixture.companyId)) .then((rows) => rows.length); expect(credentialCount).toBe(provider === "github" ? 4 : 3); const deliveryCount = await db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)) .then((rows) => rows.length); expect(deliveryCount).toBeGreaterThan(0); await service.configure(endpoint.id, { action: "remove" }, "owner-user"); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "archived", }); expect(runtime.endpoints.has(endpoint.id)).toBe(false); await expect( db .select({ refs: toolConnections.credentialSecretRefs, status: toolConnections.status, }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)), ).resolves.toEqual([{ refs: [], status: "archived" }]); await expect( db .select({ id: companySecrets.id }) .from(companySecrets) .where(eq(companySecrets.companyId, fixture.companyId)), ).resolves.toHaveLength(0); await expect( db .select({ state: chatConversations.state }) .from(chatConversations) .where(eq(chatConversations.id, conversation!.id)), ).resolves.toEqual([{ state: "endpoint_removed" }]); await expect( db .select({ id: issues.id }) .from(issues) .where(eq(issues.id, conversation!.issueId)), ).resolves.toHaveLength(1); await expect( db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)), ).resolves.toHaveLength(deliveryCount); }, ); it("retries credential cleanup after archival made a remove partially complete", async () => { const fixture = await seedCompany(); const { endpoint, service } = await configuredSlackEndpoint(fixture); await db .update(chatEndpoints) .set({ status: "archived", archivedAt: new Date() }) .where(eq(chatEndpoints.id, endpoint.id)); await db .update(toolConnections) .set({ status: "archived", enabled: false }) .where(eq(toolConnections.id, endpoint.connectionId)); await expect( service.configure(endpoint.id, { action: "remove" }, "owner-user"), ).resolves.toMatchObject({ status: "archived", }); const [cleaned] = await db .select({ refs: toolConnections.credentialSecretRefs }) .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(cleaned!.refs).toEqual([]); await expect( db .select() .from(companySecrets) .where(eq(companySecrets.companyId, fixture.companyId)), ).resolves.toHaveLength(0); await expect( service.configure(endpoint.id, { action: "remove" }, "owner-user"), ).rejects.toMatchObject({ status: 409, details: { code: "chat_endpoint_already_removed" }, }); }); it("marks provider setup as needing attention when Telegram webhook registration fails", async () => { const fixture = await seedCompany(); const providerFetch = vi.fn(async (input: string | URL | Request) => { const url = String(input); if (url.endsWith("/getMe")) { return new Response( JSON.stringify({ ok: true, result: { id: 42, username: "maya_e2e_bot", first_name: "Maya" }, }), { status: 200, headers: { "content-type": "application/json" }, }, ); } if (url.endsWith("/getWebhookInfo")) { return new Response(JSON.stringify({ ok: true, result: { url: "" } }), { status: 200, headers: { "content-type": "application/json" }, }); } if (url.endsWith("/setWebhook")) { return new Response( JSON.stringify({ ok: false, description: "webhook unavailable" }), { status: 400, headers: { "content-type": "application/json" }, }, ); } throw new Error(`Unexpected provider request: ${url}`); }) as unknown as typeof globalThis.fetch; const { service, runtime } = createService( new FakeChatSdkRuntime(), providerFetch, ); const endpoint = await service.create( fixture.companyId, { provider: "telegram", assignedAgentId: fixture.assignedAgentId, }, "owner-user", ); await expect( service.configure( endpoint.id, { action: "configure", credentials: { botToken: "telegram-test-token" }, }, "owner-user", ), ).rejects.toMatchObject({ status: 422 }); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "attention", healthMessage: "Provider setup needs attention", lastError: "Telegram could not register the webhook: webhook unavailable", }); const [connection] = await db .select() .from(toolConnections) .where(eq(toolConnections.id, endpoint.connectionId)); expect(connection).toMatchObject({ healthStatus: "degraded", healthMessage: "Provider setup failed", }); expect(runtime.endpoints.has(endpoint.id)).toBe(false); }); it("redacts Telegram bot tokens from provider setup errors and endpoint health", async () => { const fixture = await seedCompany(); const botToken = "123456789:ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghi"; const providerFetch = vi.fn(async (input: string | URL | Request) => { const url = String(input); if (url.endsWith("/getMe")) { return new Response( JSON.stringify({ ok: true, result: { id: 43, username: "redaction_test_bot", first_name: "Redaction", }, }), { status: 200, headers: { "content-type": "application/json" }, }, ); } if (url.endsWith("/getWebhookInfo")) throw new Error(`network failed for ${url}; token ${botToken}`); throw new Error(`Unexpected provider request: ${url}`); }) as unknown as typeof globalThis.fetch; const { service } = createService(new FakeChatSdkRuntime(), providerFetch); const endpoint = await service.create( fixture.companyId, { provider: "telegram", assignedAgentId: fixture.assignedAgentId, }, "owner-user", ); let setupError: unknown; try { await service.configure( endpoint.id, { action: "configure", credentials: { botToken }, }, "owner-user", ); } catch (error) { setupError = error; } expect(setupError).toMatchObject({ status: 422, details: { code: "chat_provider_setup_failed" }, }); const configured = await service.get(endpoint.id); const serializedFailure = JSON.stringify({ setupError, configured }); expect(serializedFailure).not.toContain(botToken); expect(serializedFailure).not.toContain(encodeURIComponent(botToken)); expect(configured).toMatchObject({ status: "attention", healthMessage: "Provider setup needs attention", lastError: "network failed for https://api.telegram.org/bot***REDACTED***/getWebhookInfo; token ***REDACTED***", }); }); it("reads the entire publication batch without side effects and rejects foreign bindings", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredSlackEndpoint(fixture); const channel = makeThread({ channelId: "C-BATCH-STATUS", id: "slack:C-BATCH-STATUS:4410.1", name: "batch-status", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "4410.1", text: "@maya status test", mentioned: true, }), trigger: "mention", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const comment = await issueService(db).addComment( conversation.issueId, "Batch status fixture", { userId: "owner-user" }, ); const [textPart, filePart] = await db .insert(chatPublications) .values([ { companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, commentId: comment.id, idempotencyKey: "status-text", state: "published", payload: { text: "Batch status fixture" }, providerMessageId: "status-provider-id", }, { companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, commentId: comment.id, idempotencyKey: "status-file", state: "pending", payload: { text: "File fixture" }, }, ]) .returning(); const app = routesApp(db, fixture.companyId, service); const path = `/api/chat-endpoints/${endpoint.id}/conversations/${conversation.id}/publications/${textPart.id}/status`; const initialPosts = runtime.endpoints.get(endpoint.id)?.posts.length; const initialWakes = wakeup.mock.calls.length; for (const state of [ "pending", "streaming", "retry", "failed", "delivery_unknown", "cancelled", "published", ] as const) { await db .update(chatPublications) .set({ state }) .where(eq(chatPublications.id, filePart.id)); const response = await request(app).get(path).expect(200); expect(response.body).toMatchObject({ total: 2, published: state === "published" ? 2 : 1, publication: { state }, }); if (state !== "published") expect(response.body.publication.id).toBe(filePart.id); expect(Object.keys(response.body.publication).sort()).toEqual([ "attempts", "id", "nextAttemptAt", "providerUrl", "publishedAt", "redactedError", "state", ]); } expect(runtime.endpoints.get(endpoint.id)?.posts.length).toBe(initialPosts); expect(wakeup.mock.calls.length).toBe(initialWakes); await request(app) .get(path.replace(conversation.id, randomUUID())) .expect(404); await request(app).get(path.replace(textPart.id, randomUUID())).expect(404); await request(app) .get(path.replace(textPart.id, "invalid-publication")) .expect(400); const foreign = await seedCompany(); const foreignEndpoint = await service.create( foreign.companyId, { provider: "slack", assignedAgentId: foreign.assignedAgentId }, "owner-user", ); await request(app) .get(path.replace(endpoint.id, foreignEndpoint.id)) .expect(404); await request(routesApp(db, foreign.companyId, service)) .get(path) .expect(404); }); it("publishes only explicitly selected board attachments and stays idempotent across retries", async () => { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture, { storage: storage.storage, }); const channel = makeThread({ channelId: "C-BOARD-SEND", id: "slack:C-BOARD-SEND:4400.1", name: "board-send", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "4400.1", text: "@maya start a board-send task", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const stored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${conversation.issueId}`, originalFilename: "selected-result.txt", contentType: "text/plain", body: Buffer.from("selected result", "utf8"), }); const attachment = await issueService(db).createAttachment({ issueId: conversation.issueId, provider: stored.provider, objectKey: stored.objectKey, contentType: stored.contentType, byteSize: stored.byteSize, sha256: stored.sha256, originalFilename: stored.originalFilename, createdByUserId: "owner-user", }); const first = await service.publishBoardMessage( endpoint.id, conversation.id, "Visible board update", "same-browser-request-1234", "owner-user", [attachment.id], ); const second = await service.publishBoardMessage( endpoint.id, conversation.id, "Visible board update", "same-browser-request-1234", "owner-user", [attachment.id], ); expect(second.id).toBe(first.id); const comments = await db .select() .from(issueComments) .where( and( eq(issueComments.issueId, conversation.issueId), eq(issueComments.body, "Visible board update"), ), ); const publications = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, comments[0].id)) .orderBy(asc(chatPublications.createdAt)); expect(comments).toHaveLength(1); expect(publications).toHaveLength(2); expect(publications).toEqual([ expect.objectContaining({ state: "published", payload: { text: "Visible board update" }, }), expect.objectContaining({ id: first.id, state: "published", payload: expect.objectContaining({ attachmentIds: [attachment.id] }), }), ]); await expect( db .select({ issueCommentId: issueAttachments.issueCommentId }) .from(issueAttachments) .where(eq(issueAttachments.id, attachment.id)), ).resolves.toEqual([{ issueCommentId: comments[0].id }]); expect(runtime.endpoints.get(endpoint.id)?.posts).toEqual([ { threadId: channel.thread.id, text: "Visible board update" }, { threadId: channel.thread.id, text: "", files: [ expect.objectContaining({ filename: "selected-result.txt", mimeType: "text/plain", data: Buffer.from("selected result", "utf8"), }), ], }, ]); const blockedStored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${conversation.issueId}`, originalFilename: "blocked-result.txt", contentType: "text/plain", body: Buffer.from("blocked result", "utf8"), }); const blockedAttachment = await issueService(db).createAttachment({ issueId: conversation.issueId, provider: blockedStored.provider, objectKey: blockedStored.objectKey, contentType: blockedStored.contentType, byteSize: blockedStored.byteSize, sha256: blockedStored.sha256, originalFilename: blockedStored.originalFilename, createdByUserId: "owner-user", }); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack runtime"); providerRuntime.postError = new Error("socket reset after write"); const blocked = await service.publishBoardMessage( endpoint.id, conversation.id, "Potentially accepted board update", "ambiguous-browser-request-1234", "owner-user", [blockedAttachment.id], ); const blockedRetry = await service.publishBoardMessage( endpoint.id, conversation.id, "Potentially accepted board update", "ambiguous-browser-request-1234", "owner-user", [blockedAttachment.id], ); expect(blockedRetry.id).toBe(blocked.id); expect(blocked).toMatchObject({ state: "delivery_unknown" }); await expect( db .select({ id: chatPublications.id, state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.commentId, blocked.commentId!)) .orderBy(asc(chatPublications.createdAt)), ).resolves.toEqual([ { id: blocked.id, state: "delivery_unknown" }, { id: expect.any(String), state: "pending" }, ]); }); describe("Board send terminal attachment rejection", () => { async function rejectionFixture() { const company = await seedCompany(); const storage = createStorageService(); const context = await configuredSlackEndpoint(company, { storage: storage.storage, }); const channel = makeThread({ channelId: "C-BOARD-REJECTION", id: "slack:C-BOARD-REJECTION:4410.1", }); await deliverMessage({ callbacks: context.callbacks, endpointId: context.endpoint.id, thread: channel.thread, message: makeMessage({ id: "4410.1", text: "@maya start a rejection test", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(context.service, context.endpoint.id); await context.service.test(context.endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, context.endpoint.id)); const stored = await storage.storage.putFile({ companyId: company.companyId, namespace: `issues/${conversation.issueId}`, originalFilename: "private-original.txt", contentType: "text/plain", body: Buffer.from("private original file"), }); const attachment = await issueService(db).createAttachment({ issueId: conversation.issueId, ...stored, createdByUserId: "owner-user", }); const owner = await issueService(db).addComment( conversation.issueId, "Original private attachment owner", { userId: "owner-user" }, { authorType: "user", attachmentIds: [attachment.id] }, ); const key = "rejected-browser-send-1234"; const send = ( body = "Rejected private message", ids = [attachment.id], clientKey = key, ) => context.service.publishBoardMessage( context.endpoint.id, conversation.id, body, clientKey, "owner-user", ids, ); const receipts = () => db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, context.endpoint.id), eq(chatActions.kind, "board_send_rejected"), ), ); const publications = () => db .select() .from(chatPublications) .where( and( eq(chatPublications.conversationId, conversation.id), sql`${chatPublications.idempotencyKey} like 'explicit-board:%'`, ), ); const initialPosts = context.runtime.endpoints.get(context.endpoint.id)?.posts.length ?? 0; const details = { code: "chat_board_send_attachments_already_bound", endpointId: context.endpoint.id, conversationId: conversation.id, idempotencyKey: key, attachmentIds: [attachment.id], }; return { ...context, conversation, attachment, owner, send, receipts, publications, details, initialPosts, storage, }; } it("keeps the exact rejected key terminal after normal owner-comment deletion and service reconstruction", async () => { const fixture = await rejectionFixture(); let restarted: ChatChannelService | undefined; try { await expect(fixture.send()).rejects.toMatchObject({ status: 409 }); await issueService(db).removeComment(fixture.owner.id); const [unbound] = await db .select() .from(issueAttachments) .where(eq(issueAttachments.id, fixture.attachment.id)); expect(unbound.issueCommentId).toBeNull(); await fixture.service.shutdown(); restarted = createService(fixture.runtime).service; await expect( restarted.publishBoardMessage( fixture.endpoint.id, fixture.conversation.id, "Changed retry payload", fixture.details.idempotencyKey, "owner-user", [], ), ).rejects.toMatchObject({ status: 409, details: fixture.details }); expect(await fixture.publications()).toEqual([]); expect(await fixture.receipts()).toHaveLength(1); expect( fixture.runtime.endpoints .get(fixture.endpoint.id) ?.posts.slice(fixture.initialPosts) ?? [], ).toEqual([]); const comments = await db .select() .from(issueComments) .where( and( eq(issueComments.issueId, fixture.conversation.issueId), inArray(issueComments.body, [ "Rejected private message", "Changed retry payload", ]), ), ); expect(comments).toEqual([]); } finally { if (restarted) await restarted.shutdown(); await retirePublicationFixture(fixture.service, fixture.endpoint.id); } }); it("commits one content-free scoped receipt for concurrent rejection and allows only a separately keyed corrected send", async () => { const fixture = await rejectionFixture(); try { const results = await Promise.allSettled([ fixture.send(), fixture.send(), ]); for (const result of results) expect(result).toMatchObject({ status: "rejected", reason: { status: 409, details: fixture.details }, }); const receipts = await fixture.receipts(); expect(receipts).toHaveLength(1); expect(receipts[0]).toMatchObject({ companyId: fixture.endpoint.companyId, endpointId: fixture.endpoint.id, conversationId: fixture.conversation.id, status: "processed", }); const serialized = JSON.stringify(receipts); for (const secret of [ "Rejected private message", "private-original.txt", "private original file", fixture.owner.body, ]) expect(serialized).not.toContain(secret); expect(await fixture.publications()).toEqual([]); const corrected = await fixture.send( "Corrected separately keyed send", [], "corrected-browser-send-1234", ); expect(corrected.state).toBe("published"); const replay = await fixture.send( "Different retry text", [fixture.attachment.id], "corrected-browser-send-1234", ); expect(replay.id).toBe(corrected.id); expect(await fixture.publications()).toHaveLength(1); expect(await fixture.receipts()).toHaveLength(1); expect( fixture.runtime.endpoints .get(fixture.endpoint.id) ?.posts.slice(fixture.initialPosts), ).toEqual([ { threadId: "slack:C-BOARD-REJECTION:4410.1", text: "Corrected separately keyed send", }, ]); } finally { await retirePublicationFixture(fixture.service, fixture.endpoint.id); } }); it("returns the exact HTTP rejection with only the bound subset and replays that receipt unchanged", async () => { const fixture = await rejectionFixture(); const app = routesApp(db, fixture.endpoint.companyId, fixture.service); const server = createServer(app); try { await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", () => { server.off("error", reject); resolve(); }); }); const client = request( `http://127.0.0.1:${(server.address() as AddressInfo).port}`, ); const stored = await fixture.storage.storage.putFile({ companyId: fixture.endpoint.companyId, namespace: `issues/${fixture.conversation.issueId}`, originalFilename: "still-unbound.txt", contentType: "text/plain", body: Buffer.from("unbound"), }); const unbound = await issueService(db).createAttachment({ issueId: fixture.conversation.issueId, ...stored, createdByUserId: "owner-user", }); const path = `/api/chat-endpoints/${fixture.endpoint.id}/conversations/${fixture.conversation.id}/publications`; const response = await client .post(path) .send({ body: "Rejected private message", idempotencyKey: fixture.details.idempotencyKey, attachmentIds: [unbound.id, fixture.attachment.id], }) .expect(409); expect(response.body.details).toEqual(fixture.details); await issueService(db).removeComment(fixture.owner.id); const repeated = await client .post(path) .send({ body: "Changed retry payload", idempotencyKey: fixture.details.idempotencyKey, attachmentIds: [unbound.id], }) .expect(409); expect(repeated.body.details).toEqual(fixture.details); expect(await fixture.publications()).toEqual([]); expect(await fixture.receipts()).toHaveLength(1); const [unchanged] = await db .select() .from(issueAttachments) .where(eq(issueAttachments.id, unbound.id)); expect(unchanged.issueCommentId).toBeNull(); expect( fixture.runtime.endpoints .get(fixture.endpoint.id) ?.posts.slice(fixture.initialPosts), ).toEqual([]); } finally { try { await retirePublicationFixture(fixture.service, fixture.endpoint.id); } finally { await new Promise((resolve, reject) => { server.close((error) => (error ? reject(error) : resolve())); server.closeAllConnections(); }); } } }); it("does not classify missing attachments as a safe terminal rejection", async () => { const fixture = await rejectionFixture(); try { await expect( fixture.send("Missing attachment", [randomUUID()]), ).rejects.toMatchObject({ status: 422 }); expect(await fixture.receipts()).toEqual([]); expect(await fixture.publications()).toEqual([]); const corrected = await fixture.send( "Same key after an unclassified failure", [], ); expect(corrected.state).toBe("published"); } finally { await retirePublicationFixture(fixture.service, fixture.endpoint.id); } }); }); describe("Teams file receipt API projection", () => { type Part = { state?: typeof chatPublications.$inferInsert.state; transfer?: Partial; }; async function projectionFixture(active = false) { const company = await seedCompany(); const providerFetch = vi.fn(async () => { throw new Error("Projection GET must not contact a provider"); }); const configured = active ? await configuredTeamsEndpoint(company) : null; const context = configured ?? createService(new FakeChatSdkRuntime(), providerFetch); const endpoint = configured?.endpoint ?? (await context.service.create( company.companyId, { provider: "microsoft-teams", assignedAgentId: company.assignedAgentId, name: "Read-only Teams file receipt fixture", }, "owner-user", )); let issue: typeof issues.$inferSelect | undefined; let conversation: typeof chatConversations.$inferSelect | undefined; if (!configured) { [issue] = await db .insert(issues) .values({ companyId: company.companyId, title: "Seeded file receipt projection", status: "backlog", }) .returning(); [conversation] = await db .insert(chatConversations) .values({ companyId: company.companyId, endpointId: endpoint.id, issueId: issue!.id, externalConversationId: `teams:projection:${randomUUID()}`, externalLabel: "Synthetic personal conversation", isDirectMessage: true, }) .returning(); } if (configured) { const providerConversationId = `19:projection-${randomUUID()}@thread.tacv2`; const serviceUrl = "https://smba.trafficmanager.net/amer/"; const rootMessageId = "1740000000991"; const channel = makeThread({ channelId: `teams:${Buffer.from(providerConversationId).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, id: `teams:${Buffer.from(`${providerConversationId};messageid=${rootMessageId}`).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`, name: "Active projection fixture", }); await deliverMessage({ callbacks: configured.callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: channel.thread, message: makeMessage({ id: rootMessageId, text: "@maya verify this active publication fixture", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(context.service, endpoint.id); await context.service.test(endpoint.id, "owner-user"); [conversation] = await db .select() .from(chatConversations) .where( and( eq(chatConversations.endpointId, endpoint.id), eq(chatConversations.externalThreadId, channel.thread.id), ), ); if (!conversation) throw new Error("Expected admitted active Teams conversation"); [issue] = await db .select() .from(issues) .where(eq(issues.id, conversation.issueId)); await expect(context.service.get(endpoint.id)).resolves.toMatchObject({ status: "active", }); } const [principal] = await db .insert(chatExternalPrincipals) .values({ companyId: company.companyId, provider: "microsoft-teams", providerAccountId: randomUUID(), externalId: `projection-user-${randomUUID()}`, }) .returning(); const batch = async (parts: Part[]) => { const [comment] = await db .insert(issueComments) .values({ companyId: company.companyId, issueId: issue!.id, authorUserId: "owner-user", body: "PRIVATE-COMMENT-PROJECTION-CANARY", }) .returning(); const publications: Array = []; for (const [index, part] of parts.entries()) { const [publication] = await db .insert(chatPublications) .values({ companyId: company.companyId, endpointId: endpoint.id, conversationId: conversation!.id, issueId: issue!.id, commentId: comment!.id, idempotencyKey: `projection:${randomUUID()}:${index}`, payload: { text: "PRIVATE-PAYLOAD-PROJECTION-CANARY" }, state: part.state ?? "published", providerMessageId: `PRIVATE-CARD-PROJECTION-CANARY-${index}`, attempts: 1, createdAt: new Date(Date.now() + index), }) .returning(); publications.push(publication!); if (part.transfer) { await db.insert(chatTeamsFileTransfers).values({ companyId: company.companyId, endpointId: endpoint.id, conversationId: conversation!.id, publicationId: publication!.id, issueId: issue!.id, commentId: comment!.id, attachmentId: randomUUID(), principalId: principal!.id, authorizedUserId: "owner-user", runtimeGeneration: 1, credentialFingerprint: "PRIVATE-FINGERPRINT-PROJECTION-CANARY", conversationGeneration: 1, sourceDigest: "1".repeat(64), authorityDigest: "2".repeat(64), tenantId: randomUUID(), botAppId: randomUUID(), aadObjectId: randomUUID(), providerConversationId: "PRIVATE-CONVERSATION-PROJECTION-CANARY", providerUserId: "PRIVATE-USER-PROJECTION-CANARY", sha256: "3".repeat(64), byteSize: 12, filename: `report-${index}.txt`, tokenSha256: createHash("sha256") .update(randomUUID()) .digest("hex"), phase: "awaiting_consent", version: 2, consentMessageId: `PRIVATE-CARD-PROJECTION-CANARY-${index}`, expiresAt: new Date( Date.now() + (part.transfer.phase === "expired" ? -60_000 : 600_000), ), privateState: { schema: "synthetic-private-state", ciphertext: "PRIVATE-CIPHERTEXT-PROJECTION-CANARY", uploadUrl: "https://private.invalid/upload?token=PRIVATE-TOKEN-PROJECTION-CANARY", contentUrl: "https://private.invalid/PRIVATE-CONTENT-PROJECTION-CANARY", }, ...part.transfer, }); } } return publications; }; const snapshot = async () => ({ publications: await db .select() .from(chatPublications) .where(eq(chatPublications.companyId, company.companyId)) .orderBy(asc(chatPublications.id)), transfers: await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.companyId, company.companyId)) .orderBy(asc(chatTeamsFileTransfers.id)), comments: await db .select() .from(issueComments) .where(eq(issueComments.companyId, company.companyId)) .orderBy(asc(issueComments.id)), actions: await db .select() .from(chatActions) .where(eq(chatActions.companyId, company.companyId)) .orderBy(asc(chatActions.id)), deliveries: await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.companyId, company.companyId)) .orderBy(asc(chatDeliveries.id)), links: await db .select() .from(chatMessageLinks) .where(eq(chatMessageLinks.companyId, company.companyId)) .orderBy(asc(chatMessageLinks.id)), wakes: await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.companyId, company.companyId)) .orderBy(asc(agentWakeupRequests.id)), audit: await db .select() .from(activityLog) .where(eq(activityLog.companyId, company.companyId)) .orderBy(asc(activityLog.id)), }); const statusPath = (publicationId: string) => `/api/chat-endpoints/${endpoint.id}/conversations/${conversation!.id}/publications/${publicationId}/status`; const read = async (publicationId: string) => { const before = await snapshot(); const app = routesApp(db, company.companyId, context.service); // Own one ready loopback listener for both reads. Creating and closing // a separate implicit Supertest listener per GET can reset the TCP // connection before Express receives it under the loaded macOS suite. const server = createServer(app); await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", () => { server.off("error", reject); resolve(); }); }); const { status, activity } = await (async () => { try { const status = await request(server) .get(statusPath(publicationId)) .expect(200); const activity = await request(server) .get(`/api/chat-endpoints/${endpoint.id}/activity`) .expect(200); return { status, activity }; } finally { await new Promise((resolve, reject) => { server.close((error) => (error ? reject(error) : resolve())); }); } })(); expect(status.body).toEqual( await context.service.getPublicationBatchStatus( endpoint.id, conversation!.id, publicationId, ), ); expect(activity.body).toEqual( await context.service.listActivity(endpoint.id), ); const serialized = JSON.stringify([status.body, activity.body]); expect(serialized).not.toMatch( /PROJECTION-CANARY|https?:|ciphertext|privateState|uploadUrl|contentUrl|tokenSha256|consentMessageId|fileInfoMessageId|responseActivityId|providerMessageId/, ); expect(await snapshot()).toEqual(before); expect(providerFetch).not.toHaveBeenCalled(); expect(context.wakeup).not.toHaveBeenCalled(); expect(context.cancelRun).not.toHaveBeenCalled(); expect(context.runtime.endpoints.size).toBe(0); return { status: status.body, activity: activity.body as Array>, }; }; return { ...company, ...context, endpoint, conversation: conversation!, principal: principal!, issue: issue!, batch, snapshot, read, statusPath, }; } // These are persisted-state/API proofs, not provider upload or tenant-auth // proofs. No worker is started; every GET must preserve all seeded rows. it("projects mixed delivered, declined, expired and cancelled file outcomes without making GET effects", async () => { const fixture = await projectionFixture(); try { const parts = await fixture.batch([ {}, { transfer: { phase: "delivered", fileInfoMessageId: "PRIVATE-FILE-RECEIPT-PROJECTION-CANARY", }, }, { transfer: { phase: "declined", responseActivityId: "PRIVATE-DECLINE-PROJECTION-CANARY", }, }, { transfer: { phase: "expired" } }, { transfer: { phase: "cancelled", reason: "cancelled_after_upload" }, }, ]); const { status, activity } = await fixture.read(parts[0]!.id); expect(status).toMatchObject({ total: 5, published: 2, declined: 1, expired: 1, cancelled: 1, settled: 5, awaitingConsent: 0, canDismiss: true, publication: { id: parts[2]!.id, state: "cancelled" }, }); expect( status.parts.map((part: { id: string; state: string }) => [ part.id, part.state, ]), ).toEqual( parts.map((part, index) => [ part.id, index < 2 ? "published" : "cancelled", ]), ); expect( activity .filter((item) => parts.some((part) => part.id === item.id)) .map((item) => item.status) .sort(), ).toEqual([ "cancelled", "cancelled", "cancelled", "published", "published", ]); for (const item of activity) { expect(item.replayable).toBe(false); expect(item.resolutionActions).toEqual([]); } } finally { await fixture.service.shutdown(); } }); it("keeps a cancelled head locked while a consent-card receipt waits for the file tail", async () => { const fixture = await projectionFixture(); try { const [head, tail] = await fixture.batch([ { state: "cancelled" }, { transfer: {} }, ]); const { status, activity } = await fixture.read(head!.id); expect(status).toMatchObject({ total: 2, published: 0, declined: 0, expired: 0, cancelled: 1, settled: 1, awaitingConsent: 1, canDismiss: false, publication: { id: tail!.id, state: "awaiting_consent" }, }); expect(activity.find((item) => item.id === tail!.id)).toMatchObject({ status: "awaiting_consent", replayable: false, resolutionActions: [], fileTransfer: { phase: "awaiting_consent", version: 2, filename: "report-1.txt", }, }); } finally { await fixture.service.shutdown(); } }); it("preserves all-published legacy batches without transfer metadata and excludes other comments", async () => { const fixture = await projectionFixture(); try { const parts = await fixture.batch([{}, {}]); await fixture.batch([{ state: "pending" }]); const { status } = await fixture.read(parts[0]!.id); expect(status).toMatchObject({ total: 2, published: 2, declined: 0, expired: 0, cancelled: 0, settled: 2, awaitingConsent: 0, canDismiss: true, publication: { id: parts[1]!.id, state: "published" }, }); expect(status.parts.map((part: { id: string }) => part.id)).toEqual( parts.map((part) => part.id), ); for (const part of status.parts) expect(part).not.toHaveProperty("fileTransfer"); } finally { await fixture.service.shutdown(); } }); it.each([ ["delivered without file receipt", { phase: "delivered" }], [ "waiting without card receipt", { phase: "awaiting_consent", consentMessageId: null }, ], ["declined without response receipt", { phase: "declined" }], [ "malformed filename", { phase: "delivered", filename: "", fileInfoMessageId: "PRIVATE-FILE-PROJECTION-CANARY", }, ], [ "wrong-stage operator receipt", { phase: "delivered", privateState: { resolution: { schema: "paperclip.teams.file-resolution.v1", action: "mark_delivered", fromPhase: "upload_unknown", }, }, }, ], ] satisfies Array< [string, Partial] >)( "refuses terminal success or actions for %s", async (_label, transfer) => { const fixture = await projectionFixture(); try { const [part] = await fixture.batch([{ transfer }]); const { status, activity } = await fixture.read(part!.id); expect(status).toMatchObject({ total: 1, published: 0, settled: 0, awaitingConsent: 0, canDismiss: false, publication: { id: part!.id, state: "delivery_unknown" }, }); expect(activity).toEqual([ expect.objectContaining({ id: part!.id, status: "delivery_unknown", replayable: false, resolutionActions: [], }), ]); } finally { await fixture.service.shutdown(); } }, ); it.each([ ["consent_unknown", ["cancel"]], ["upload_unknown", ["cancel"]], ["file_info_unknown", ["mark_delivered", "retry_anyway", "cancel"]], ["conflict", []], ] as const)( "offers only the exact %s stage actions", async (phase, actions) => { const fixture = await projectionFixture(); try { const [part] = await fixture.batch([{ transfer: { phase } }]); const { status, activity } = await fixture.read(part!.id); expect(status).toMatchObject({ published: 0, settled: 0, canDismiss: false, publication: { state: "delivery_unknown" }, }); expect(activity).toEqual([ expect.objectContaining({ id: part!.id, replayable: false, resolutionActions: actions, fileTransfer: { provider: "microsoft-teams", phase, version: 2, filename: "report-0.txt", expiresAt: expect.any(String), }, }), ]); } finally { await fixture.service.shutdown(); } }, ); it("requires exact endpoint, conversation, publication and Board company scope", async () => { const fixture = await projectionFixture(); const other = await projectionFixture(); try { const [part] = await fixture.batch([{ transfer: {} }]); const [foreign] = await other.batch([{ transfer: {} }]); const app = routesApp(db, fixture.companyId, fixture.service); const before = await fixture.snapshot(); await request(app).get(fixture.statusPath(foreign!.id)).expect(404); await request(app) .get( fixture .statusPath(part!.id) .replace(fixture.conversation.id, other.conversation.id), ) .expect(404); // Cross-company resources are deliberately concealed as not found. await request(app).get(other.statusPath(part!.id)).expect(404); await request(app) .get(`/api/chat-endpoints/${other.endpoint.id}/activity`) .expect(404); expect(await fixture.snapshot()).toEqual(before); const { status, activity } = await fixture.read(part!.id); expect(status.total).toBe(1); expect(activity.map((item) => item.id)).toEqual([part!.id]); expect(JSON.stringify([status, activity])).not.toContain(foreign!.id); } finally { try { await fixture.service.shutdown(); } finally { await other.service.shutdown(); } } }); it("never projects a transfer from another conversation into Activity", async () => { const fixture = await projectionFixture(); try { const [otherConversation] = await db .insert(chatConversations) .values({ companyId: fixture.companyId, endpointId: fixture.endpoint.id, issueId: fixture.issue.id, externalConversationId: `teams:other:${randomUUID()}`, externalLabel: "Different conversation", }) .returning(); const [part] = await fixture.batch([ { state: "pending", transfer: { conversationId: otherConversation!.id, phase: "file_info_unknown", filename: "WRONG-CONVERSATION-FILE.txt", }, }, ]); const { status, activity } = await fixture.read(part!.id); expect(status).toMatchObject({ publication: { state: "pending" }, canDismiss: false, }); expect(status.publication).not.toHaveProperty("fileTransfer"); expect(activity).toEqual([ expect.objectContaining({ id: part!.id, status: "pending", replayable: false, resolutionActions: [], }), ]); expect(activity[0]).not.toHaveProperty("fileTransfer"); } finally { await fixture.service.shutdown(); } }); it.each([ ["missing", undefined], ["wrong phase", { phase: "upload_unknown", version: 2 }], ["stale version", { phase: "file_info_unknown", version: 1 }], ] as const)( "refuses generic resolution with %s transfer preconditions without changing receipts", async (_label, fileTransfer) => { const fixture = await projectionFixture(); try { const [part] = await fixture.batch([ { state: "delivery_unknown", transfer: { phase: "file_info_unknown" }, }, ]); const before = await fixture.snapshot(); const app = routesApp(db, fixture.companyId, fixture.service); // A missing or stale transfer hint must never fall through to ordinary // message retry, mark-delivered, or cancellation. for (const action of [ "mark_delivered", "retry_anyway", "cancel", ] as const) { const response = await request(app) .post( `/api/chat-endpoints/${fixture.endpoint.id}/publications/${part!.id}/resolve`, ) .send({ action, ...(fileTransfer ? { fileTransfer } : {}) }) .expect(409); expect(response.body).toMatchObject({ code: "chat_file_transfer_resolution_required", }); await expect( fixture.service.resolvePublication( fixture.endpoint.id, part!.id, action, "owner-user", fileTransfer, ), ).rejects.toMatchObject({ status: 409, details: { code: "chat_file_transfer_resolution_required" }, }); expect(await fixture.snapshot()).toEqual(before); } await fixture.read(part!.id); } finally { await fixture.service.shutdown(); } }, ); it("refuses generic replay of a failed transfer without another provider attempt", async () => { const fixture = await projectionFixture(); try { const [part] = await fixture.batch([ { state: "failed", transfer: { phase: "consent_pending" } }, ]); const before = await fixture.snapshot(); const response = await request( routesApp(db, fixture.companyId, fixture.service), ) .post( `/api/chat-endpoints/${fixture.endpoint.id}/publications/${part!.id}/replay`, ) .expect(409); expect(response.body).toMatchObject({ code: "chat_file_transfer_resolution_required", }); await expect( fixture.service.replayPublication(fixture.endpoint.id, part!.id), ).rejects.toMatchObject({ status: 409, details: { code: "chat_file_transfer_resolution_required" }, }); expect(await fixture.snapshot()).toEqual(before); const { activity } = await fixture.read(part!.id); expect(activity).toEqual([ expect.objectContaining({ id: part!.id, replayable: false, resolutionActions: [], }), ]); } finally { await fixture.service.shutdown(); } }); it("does not treat a caller-only transfer hint as ordinary delivery authority", async () => { const fixture = await projectionFixture(); try { const [part] = await fixture.batch([{ state: "delivery_unknown" }]); const before = await fixture.snapshot(); const response = await request( routesApp(db, fixture.companyId, fixture.service), ) .post( `/api/chat-endpoints/${fixture.endpoint.id}/publications/${part!.id}/resolve`, ) .send({ action: "mark_delivered", fileTransfer: { phase: "file_info_unknown", version: 2 }, }) .expect(409); expect(response.body).toMatchObject({ code: "chat_file_transfer_resolution_required", }); expect(await fixture.snapshot()).toEqual(before); await fixture.read(part!.id); } finally { await fixture.service.shutdown(); } }); it.each(["draft", "verifying", "paused", "attention", "archived"] as const)( "does not consume a worker slot for a pending Teams transfer on a %s endpoint", async (status) => { const fixture = await projectionFixture(); try { // Settle unrelated fixture work before arming this exact inactive // candidate. The transfer must not enter either publication worker. await fixture.service.processPendingPublications(); await db .update(chatEndpoints) .set({ status }) .where(eq(chatEndpoints.id, fixture.endpoint.id)); const [part] = await fixture.batch([ { state: "pending", transfer: { phase: "consent_pending" } }, ]); const [endpointBefore] = await db .select() .from(chatEndpoints) .where(eq(chatEndpoints.id, fixture.endpoint.id)); expect(endpointBefore?.status).toBe(status); const before = await fixture.snapshot(); await expect( fixture.service.processPendingPublications(1), ).resolves.toBe(0); await expect( fixture.service.processPendingPublications(1), ).resolves.toBe(0); expect(await fixture.snapshot()).toEqual(before); await expect( db .select() .from(chatEndpoints) .where(eq(chatEndpoints.id, fixture.endpoint.id)), ).resolves.toEqual([endpointBefore]); expect(fixture.runtime.replaceCount).toBe(0); // Also checks zero provider requests, runtime creation, wakeups, and // cancellation calls, with no projection/read-side mutations. await fixture.read(part!.id); } finally { await fixture.service.shutdown(); } }, ); it.each(["pending", "streaming"] as const)( "generic publication worker ignores an existing %s Teams transfer on an active endpoint", async (state) => { const fixture = await projectionFixture(true); try { // Finish setup's ordinary outbox work before arming the transfer. await fixture.service.processPendingPublications(); const ordinaryControl = await fixture.service.publishBoardMessage( fixture.endpoint.id, fixture.conversation.id, "Ordinary publication control before transfer ownership", `teams-worker-control-${randomUUID()}`, "owner-user", ); expect(ordinaryControl.state).toBe("published"); const [part] = await fixture.batch([ { state, transfer: { phase: state === "pending" ? "consent_pending" : "consent_sending", ...(state === "streaming" ? { attemptId: randomUUID(), attemptExpiresAt: new Date(Date.now() + 90_000), } : {}), }, }, ]); if (state === "streaming") await db .update(chatPublications) .set({ updatedAt: new Date(Date.now() - 61_000) }) .where(eq(chatPublications.id, part!.id)); await expect( fixture.service.get(fixture.endpoint.id), ).resolves.toMatchObject({ status: "active", setup: { step: "complete" }, }); const [connection] = await db .select() .from(toolConnections) .where(eq(toolConnections.id, fixture.endpoint.connectionId)); expect(connection).toMatchObject({ status: "active", enabled: true, }); const providerRuntime = fixture.runtime.endpoints.get( fixture.endpoint.id, ); expect(providerRuntime).toBeDefined(); // An actual ordinary provider post succeeded before the transfer was // seeded. An inactive fixture cannot satisfy this proof. expect(providerRuntime!.posts.length).toBeGreaterThan(0); const postsBefore = [...providerRuntime!.posts]; const before = await fixture.snapshot(); await fixture.service.processPendingPublications(); expect(await fixture.snapshot()).toEqual(before); expect(providerRuntime!.posts).toEqual(postsBefore); } finally { await retirePublicationFixture(fixture.service, fixture.endpoint.id); } }, ); }); it.each(["slack", "discord"] as const)( "preserves a published %s file prefix across restart and retries only an explicitly resolved ambiguous upload", async (provider) => { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, runtime, service } = provider === "slack" ? await configuredSlackEndpoint(fixture, { storage: storage.storage }) : await configuredDiscordEndpoint(fixture, { storage: storage.storage, }); let restarted: ChatChannelService | undefined; try { const channel = makeThread({ channelId: provider === "slack" ? "C-PARTIAL-FILES" : "333333333333332811", id: provider === "slack" ? "slack:C-PARTIAL-FILES:4400.81" : "discord:1457808928258658549:333333333333332811:555555555555552811", name: "partial-files-restart", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider, thread: channel.thread, message: makeMessage({ id: provider === "slack" ? "4400.81" : "555555555555552811", text: "@maya prepare the selected file batch", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected partial-file conversation"); const attachmentIds: string[] = []; const bodies = [1, 2, 3].map((index) => Buffer.from(`exact selected file ${index}`, "utf8"), ); for (const [index, body] of bodies.entries()) { const stored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${conversation.issueId}`, originalFilename: `selected-${index + 1}.txt`, contentType: "text/plain", body, }); const attachment = await issueService(db).createAttachment({ issueId: conversation.issueId, provider: stored.provider, objectKey: stored.objectKey, contentType: stored.contentType, byteSize: stored.byteSize, sha256: stored.sha256, originalFilename: stored.originalFilename, createdByUserId: "owner-user", }); attachmentIds.push(attachment.id); } const initialRuntime = runtime.endpoints.get(endpoint.id); if (!initialRuntime) throw new Error("Expected initial provider runtime"); initialRuntime.posts.length = 0; initialRuntime.postResultIds.push("1789000.000001", "1789000.000002"); let initialAttempts = 0; initialRuntime.postHook = async () => { initialAttempts += 1; if (initialAttempts === 3) throw new Error( "socket closed after the second upload request was written", ); }; const text = "The complete answer is saved; the three selected files follow."; const blocked = await service.publishBoardMessage( endpoint.id, conversation.id, text, "partial-file-restart", "owner-user", attachmentIds, ); expect(blocked).toMatchObject({ state: "delivery_unknown", attempts: 1, providerMessageId: null, }); if (!blocked.commentId) throw new Error("Expected durable batch comment"); const readBatch = () => db .select() .from(chatPublications) .where(eq(chatPublications.commentId, blocked.commentId!)) .orderBy(asc(chatPublications.createdAt)); const initialBatch = await readBatch(); expect(initialBatch.map((row) => row.state)).toEqual([ "published", "published", "delivery_unknown", "pending", ]); expect(initialBatch.map((row) => row.attempts)).toEqual([1, 1, 1, 0]); expect( initialBatch.map((row) => row.payload.attachmentIds ?? []), ).toEqual([[], ...attachmentIds.map((id) => [id])]); expect(initialBatch[2]!.id).toBe(blocked.id); expect(initialRuntime.posts).toEqual([ { threadId: channel.thread.id, text }, { threadId: channel.thread.id, text: provider === "slack" ? "" : "Shared selected-1.txt.", files: [ expect.objectContaining({ filename: "selected-1.txt", data: bodies[0], }), ], }, ]); expect(initialAttempts).toBe(3); const prefix = initialBatch.slice(0, 2); const prefixLinks = await db .select() .from(chatMessageLinks) .where( inArray( chatMessageLinks.publicationId, prefix.map((row) => row.id), ), ) .orderBy(asc(chatMessageLinks.id)); expect(prefixLinks).toHaveLength(2); const commentBefore = await db .select() .from(issueComments) .where(eq(issueComments.id, blocked.commentId)); expect(commentBefore).toEqual([ expect.objectContaining({ body: text }), ]); await service.shutdown(); const fresh = createService(new FakeChatSdkRuntime(), undefined, { storage: storage.storage, }); restarted = fresh.service; fresh.runtime.initializeHook = async () => { fresh.runtime.endpoints .get(endpoint.id)! .postResultIds.push("1789000.000003", "1789000.000004"); }; await restarted.processPendingPublications(); await restarted.processPendingPublications(); expect(await readBatch()).toEqual(initialBatch); expect(fresh.runtime.endpoints.get(endpoint.id)?.posts ?? []).toEqual( [], ); const partial = await request( routesApp(db, fixture.companyId, restarted), ) .get( `/api/chat-endpoints/${endpoint.id}/conversations/${conversation.id}/publications/${prefix[0]!.id}/status`, ) .expect(200); expect(partial.body).toMatchObject({ total: 4, published: 2, publication: { id: blocked.id, state: "delivery_unknown", attempts: 1, nextAttemptAt: null, }, }); expect( (await restarted.listActivity(endpoint.id)).find( (row) => row.id === blocked.id, ), ).toMatchObject({ kind: "publication", status: "delivery_unknown", replayable: false, resolutionActions: ["mark_delivered", "retry_anyway", "cancel"], }); await expect( restarted.replayPublication(endpoint.id, blocked.id), ).rejects.toMatchObject({ status: 409, details: { code: "chat_publication_resolution_required" }, }); expect(await readBatch()).toEqual(initialBatch); await restarted.resolvePublication( endpoint.id, blocked.id, "retry_anyway", "owner-user", ); await restarted.processPendingPublications(); const completed = await readBatch(); expect(completed.map((row) => row.state)).toEqual([ "published", "published", "published", "published", ]); expect(completed.map((row) => row.attempts)).toEqual([1, 1, 2, 1]); expect(completed.slice(0, 2)).toEqual(prefix); expect( await db .select() .from(chatMessageLinks) .where( inArray( chatMessageLinks.publicationId, prefix.map((row) => row.id), ), ) .orderBy(asc(chatMessageLinks.id)), ).toEqual(prefixLinks); expect( await db .select() .from(issueComments) .where(eq(issueComments.id, blocked.commentId)), ).toEqual(commentBefore); const resumedRuntime = fresh.runtime.endpoints.get(endpoint.id); expect(resumedRuntime?.posts).toEqual( [2, 3].map((index) => ({ threadId: channel.thread.id, text: provider === "slack" ? "" : `Shared selected-${index}.txt.`, files: [ expect.objectContaining({ filename: `selected-${index}.txt`, data: bodies[index - 1], }), ], })), ); expect(initialRuntime.posts).toHaveLength(2); expect(initialAttempts).toBe(3); if (provider === "slack") expect(resumedRuntime?.slackFilePublicationAttempts).toBe(2); expect( await db .select({ details: activityLog.details }) .from(activityLog) .where( and( eq(activityLog.entityId, blocked.id), eq(activityLog.action, "chat.publication_retry_anyway"), ), ), ).toEqual([ { details: expect.objectContaining({ duplicateRiskAccepted: true, previousState: "delivery_unknown", nextState: "retry", }), }, ]); const finalStatus = await restarted.getPublicationBatchStatus( endpoint.id, conversation.id, prefix[0]!.id, ); expect(finalStatus).toMatchObject({ total: 4, published: 4, publication: { state: "published" }, }); await restarted.processPendingPublications(); expect(resumedRuntime?.posts).toHaveLength(2); expect(await readBatch()).toEqual(completed); } finally { await service.shutdown(); await retirePublicationFixture(restarted ?? service, endpoint.id); } }, ); it("recovers an accepted Slack file receipt without uploading twice or exposing provider identifiers", async () => { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture, { storage: storage.storage }); try { const channel = makeThread({ channelId: "C-SLACK-RECEIPT", id: "slack:C-SLACK-RECEIPT:4400.15", name: "slack-file-receipt", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "4400.15", text: "@maya start a receipt repair test", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Slack receipt conversation"); const comment = await issueService(db).addComment( conversation.issueId, "Accepted Slack file fixture", { userId: "owner-user" }, ); const stored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${conversation.issueId}`, originalFilename: "accepted-once.txt", contentType: "text/plain", body: Buffer.from("accepted exactly once", "utf8"), }); const attachment = await issueService(db).createAttachment({ issueId: conversation.issueId, issueCommentId: comment.id, provider: stored.provider, objectKey: stored.objectKey, contentType: stored.contentType, byteSize: stored.byteSize, sha256: stored.sha256, originalFilename: stored.originalFilename, createdByUserId: "owner-user", }); const [publication] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, commentId: comment.id, idempotencyKey: `slack-file-receipt:${randomUUID()}`, payload: { text: "", attachmentIds: [attachment.id] }, state: "pending", }) .returning(); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack provider runtime"); providerRuntime.posts.length = 0; providerRuntime.slackFileReceiptCaptureRepeats = 2; providerRuntime.slackFilePostAcceptanceError = new Error( "connection closed after Slack accepted the upload", ); // Cross the earlier rate-limit fixture's five-second retry deadline. // A retired fixture must not be adopted by this global worker, even on // a slower full-suite run. This changes Date only, not transport timers. vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(new Date(Date.now() + 6_000)); const processingStartedAt = new Date(); const processed = await service.processPendingPublications(); const extraPublicationDiagnostics = processed === 1 ? undefined : await db .select({ id: chatPublications.id, endpointId: chatPublications.endpointId, key: chatPublications.idempotencyKey, state: chatPublications.state, attempts: chatPublications.attempts, nextAttemptAt: chatPublications.nextAttemptAt, }) .from(chatPublications) .where( sql`${chatPublications.updatedAt} >= ${processingStartedAt.toISOString()}::timestamptz`, ) .limit(20); expect(processed, JSON.stringify(extraPublicationDiagnostics)).toBe(1); expect(providerRuntime.slackFilePublicationAttempts).toBe(1); await expect( db .select({ attempts: chatPublications.attempts, providerMessageId: chatPublications.providerMessageId, state: chatPublications.state, }) .from(chatPublications) .where(eq(chatPublications.id, publication!.id)), ).resolves.toEqual([ { attempts: 1, providerMessageId: null, state: "delivery_unknown" }, ]); const [receipt] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slack_file_upload_receipt"), eq( chatActions.providerActionId, `slack-file-receipt:${publication!.id}:1`, ), ), ); expect(receipt).toMatchObject({ status: "received", payload: expect.objectContaining({ fileIds: ["FTEST1"], publicationAttempt: 1, publicationId: publication!.id, }), }); expect( JSON.stringify(await service.listActivity(endpoint.id)), ).not.toContain("FTEST1"); expect( JSON.stringify(await service.listActivity(endpoint.id)), ).not.toContain("credentialFingerprint"); // Legacy/default-timestamp receipts can retain PostgreSQL microseconds // that JavaScript Date cannot round-trip. Row ownership must come from // the locked status, not a lossy timestamp equality check. await db .update(chatActions) .set({ updatedAt: sql`${chatActions.updatedAt} + interval '0.000123 seconds'`, }) .where(eq(chatActions.id, receipt!.id)); const heldEndpoint = await service.create( fixture.companyId, { provider: "slack", assignedAgentId: fixture.assignedAgentId, name: "Paused receipt backlog", }, "owner-user", ); await db .update(chatEndpoints) .set({ status: "paused", updatedAt: new Date() }) .where(eq(chatEndpoints.id, heldEndpoint.id)); await db.insert(chatActions).values( Array.from({ length: 25 }, (_, index) => ({ companyId: fixture.companyId, endpointId: heldEndpoint.id, kind: "slack_file_upload_receipt", providerActionId: `paused-slack-file-receipt:${index}:${randomUUID()}`, payload: {}, status: "received" as const, result: { code: "slack_file_upload_identity_pending", attempts: 0 }, createdAt: new Date(Date.now() - 60_000), updatedAt: new Date(Date.now() - 60_000), })), ); providerRuntime.slackFilePostAcceptanceError = null; providerRuntime.slackFileReceiptResultIds.push("1788.990001"); await expect( service.processPendingSlackFileUploadReceipts(1), ).resolves.toBe(1); expect(providerRuntime.slackFilePublicationAttempts).toBe(1); expect(providerRuntime.slackFileReceiptLookups).toEqual([ { fileIds: ["FTEST1"], threadId: channel.thread.id, }, ]); await expect( db .select({ providerMessageId: chatPublications.providerMessageId, state: chatPublications.state, }) .from(chatPublications) .where(eq(chatPublications.id, publication!.id)), ).resolves.toEqual([ { providerMessageId: "1788.990001", state: "published" }, ]); await expect( db .select({ providerMessageId: chatMessageLinks.providerMessageId, publicationId: chatMessageLinks.publicationId, }) .from(chatMessageLinks) .where(eq(chatMessageLinks.publicationId, publication!.id)), ).resolves.toEqual([ { providerMessageId: "1788.990001", publicationId: publication!.id, }, ]); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, receipt!.id)), ).resolves.toEqual([{ status: "processed" }]); await expect( service.processPendingSlackFileUploadReceipts(), ).resolves.toBe(0); expect(providerRuntime.slackFileReceiptLookups).toHaveLength(1); // Operator confirmation is a state-only assertion. The later receipt // lookup may add the exact Slack identity, but must not replay bytes or // rewrite the operator's published timestamp. await db .delete(chatMessageLinks) .where(eq(chatMessageLinks.publicationId, publication!.id)); await db .update(chatPublications) .set({ state: "delivery_unknown", providerMessageId: null, publishedAt: null, updatedAt: new Date(), }) .where(eq(chatPublications.id, publication!.id)); await db .update(chatActions) .set({ status: "received", result: { code: "slack_file_upload_identity_pending", attempts: 1 }, updatedAt: new Date(), }) .where(eq(chatActions.id, receipt!.id)); const [conflictingPublication] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, commentId: comment.id, idempotencyKey: `existing-slack-message:${randomUUID()}`, payload: { text: "Existing provider message" }, state: "published", providerMessageId: "1788.990003", publishedAt: new Date(), }) .returning(); await db.insert(chatMessageLinks).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, publicationId: conflictingPublication!.id, commentId: comment.id, providerMessageId: "1788.990003", direction: "outbound", }); providerRuntime.slackFileReceiptResultIds.push("1788.990003"); await expect( service.processPendingSlackFileUploadReceipts(1), ).resolves.toBe(1); await expect( db .select({ result: chatActions.result, status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, receipt!.id)), ).resolves.toEqual([ { result: expect.objectContaining({ code: "slack_file_upload_receipt_message_conflict", retryable: false, }), status: "failed", }, ]); await db .update(chatActions) .set({ status: "received", result: { code: "slack_file_upload_identity_pending", attempts: 1 }, updatedAt: new Date(), }) .where(eq(chatActions.id, receipt!.id)); await service.resolvePublication( endpoint.id, publication!.id, "mark_delivered", "owner-user", ); const operatorPublishedAt = await db .select({ publishedAt: chatPublications.publishedAt }) .from(chatPublications) .where(eq(chatPublications.id, publication!.id)) .then((rows) => rows[0]!.publishedAt); providerRuntime.slackFileReceiptResultIds.push("1788.990002"); await expect( service.processPendingSlackFileUploadReceipts(1), ).resolves.toBe(1); expect(providerRuntime.slackFilePublicationAttempts).toBe(1); await expect( db .select({ providerMessageId: chatPublications.providerMessageId, publishedAt: chatPublications.publishedAt, state: chatPublications.state, }) .from(chatPublications) .where(eq(chatPublications.id, publication!.id)), ).resolves.toEqual([ { providerMessageId: "1788.990002", publishedAt: operatorPublishedAt, state: "published", }, ]); await db .delete(chatMessageLinks) .where(eq(chatMessageLinks.publicationId, publication!.id)); await db .update(chatPublications) .set({ state: "delivery_unknown", providerMessageId: null, publishedAt: null, updatedAt: new Date(), }) .where(eq(chatPublications.id, publication!.id)); await db .update(chatActions) .set({ status: "received", result: { code: "slack_file_upload_identity_pending", attempts: 2 }, updatedAt: new Date(), }) .where(eq(chatActions.id, receipt!.id)); await service.resolvePublication( endpoint.id, publication!.id, "cancel", "owner-user", ); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, receipt!.id)), ).resolves.toEqual([{ status: "cancelled" }]); await expect( service.processPendingSlackFileUploadReceipts(1), ).resolves.toBe(0); } finally { vi.useRealTimers(); await retirePublicationFixture(service, endpoint.id); } }); it("defers Slack file receipt recovery while the original post owns the same attempt", async () => { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture, { storage: storage.storage }); let releasePost!: () => void; const postReleased = new Promise((resolve) => { releasePost = resolve; }); let postEntered!: () => void; const postBlocked = new Promise((resolve) => { postEntered = resolve; }); try { const channel = makeThread({ channelId: "C-SLACK-RECEIPT-STREAMING", id: "slack:C-SLACK-RECEIPT-STREAMING:4400.16", name: "slack-file-receipt-streaming", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "4400.16", text: "@maya start a streaming receipt test", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected streaming conversation"); const comment = await issueService(db).addComment( conversation.issueId, "Streaming Slack file fixture", { userId: "owner-user" }, ); const stored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${conversation.issueId}`, originalFilename: "streaming-once.txt", contentType: "text/plain", body: Buffer.from("still owned by original post", "utf8"), }); const attachment = await issueService(db).createAttachment({ issueId: conversation.issueId, issueCommentId: comment.id, provider: stored.provider, objectKey: stored.objectKey, contentType: stored.contentType, byteSize: stored.byteSize, sha256: stored.sha256, originalFilename: stored.originalFilename, createdByUserId: "owner-user", }); const [publication] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, commentId: comment.id, idempotencyKey: `slack-file-receipt-streaming:${randomUUID()}`, payload: { text: "", attachmentIds: [attachment.id] }, state: "pending", }) .returning(); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack provider runtime"); providerRuntime.slackFilePostAcceptanceHook = async () => { postEntered(); await postReleased; }; const originalWorker = service.processPendingPublications(); await postBlocked; await expect( service.processPendingSlackFileUploadReceipts(), ).resolves.toBe(0); expect(providerRuntime.slackFileReceiptLookups).toHaveLength(0); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.id, publication!.id)), ).resolves.toEqual([{ state: "streaming" }]); releasePost(); await originalWorker; await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.id, publication!.id)), ).resolves.toEqual([{ state: "published" }]); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slack_file_upload_receipt"), eq( chatActions.providerActionId, `slack-file-receipt:${publication!.id}:1`, ), ), ), ).resolves.toEqual([{ status: "processed" }]); } finally { releasePost(); await retirePublicationFixture(service, endpoint.id); } }); it("rechecks a Slack file receipt after a competing worker changes its retry deadline", async () => { const { endpoint, fixture, providerRuntime, receipt, service } = await acceptedUnknownSlackFileReceipt("stale-worker"); try { const [malformed] = await db .insert(chatActions) .values({ companyId: fixture.companyId, endpointId: endpoint.id, kind: "slack_file_upload_receipt", providerActionId: `malformed-slack-file-receipt:${randomUUID()}`, payload: {}, status: "received", result: null, createdAt: new Date(Date.now() - 60_000), updatedAt: new Date(Date.now() - 60_000), }) .returning(); await db .update(chatActions) .set({ updatedAt: sql`${chatActions.updatedAt} + interval '0.000123 seconds'`, }) .where(eq(chatActions.id, malformed!.id)); await expect( service.processPendingSlackFileUploadReceipts(1), ).resolves.toBe(0); await expect( db .select({ result: chatActions.result, status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, malformed!.id)), ).resolves.toEqual([ { result: { code: "slack_file_upload_receipt_payload_invalid", retryable: false, }, status: "failed", }, ]); expect(providerRuntime.slackFileReceiptLookups).toHaveLength(0); const blockerToken = `test-blocker-${randomUUID()}`; await db.insert(chatEndpointLeases).values({ companyId: fixture.companyId, endpointId: endpoint.id, leaseKey: "credentials", token: blockerToken, expiresAt: new Date(Date.now() + 60_000), }); // The endpoint can also have deferred work using this same lease. Track // each of the two receipt workers by its async call chain, not all lease // tokens observed globally, so the barrier proves both selected the row. const workerScope = new AsyncLocalStorage<0 | 1>(); const attemptedWorkerTokens = [new Set(), new Set()]; const originalInsert = db.insert.bind(db); const insertSpy = vi.spyOn(db, "insert").mockImplementation((table) => { const builder = originalInsert(table); if (table === chatEndpointLeases) { const originalValues = builder.values.bind(builder); builder.values = ((values: { endpointId?: string; leaseKey?: string; token?: string; }) => { if ( values.endpointId === endpoint.id && values.leaseKey === "credentials" && values.token && values.token !== blockerToken ) { const worker = workerScope.getStore(); if (worker !== undefined) { attemptedWorkerTokens[worker]!.add(values.token); } } return originalValues(values); }) as typeof builder.values; } return builder; }); const workers = [ workerScope.run(0, () => service.processPendingSlackFileUploadReceipts(1), ), workerScope.run(1, () => service.processPendingSlackFileUploadReceipts(1), ), ]; let outcomes: number[] = []; try { await expect .poll(() => attemptedWorkerTokens.map((tokens) => tokens.size)) .toEqual([1, 1]); expect( new Set(attemptedWorkerTokens.flatMap((tokens) => [...tokens])).size, ).toBe(2); } finally { await db .delete(chatEndpointLeases) .where( and( eq(chatEndpointLeases.endpointId, endpoint.id), eq(chatEndpointLeases.leaseKey, "credentials"), eq(chatEndpointLeases.token, blockerToken), ), ); try { outcomes = await Promise.all(workers); } finally { insertSpy.mockRestore(); } } expect(outcomes.reduce((sum, value) => sum + value, 0)).toBe(1); expect(providerRuntime.slackFileReceiptLookups).toHaveLength(1); await expect( db .select({ result: chatActions.result, status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, receipt.id)), ).resolves.toEqual([ { result: expect.objectContaining({ attempts: 1, code: "slack_file_upload_identity_pending", retryable: true, retryAt: expect.any(String), }), status: "failed", }, ]); expect(providerRuntime.slackFilePublicationAttempts).toBe(1); } finally { await retirePublicationFixture(service, endpoint.id); } }); it("rejects a recovered Slack file identity when destination reach changes during lookup", async () => { const { conversation, endpoint, providerRuntime, publication, receipt, service, } = await acceptedUnknownSlackFileReceipt("revoked-reach"); let releaseLookup!: () => void; const lookupReleased = new Promise((resolve) => { releaseLookup = resolve; }); let lookupEntered!: () => void; const lookupBlocked = new Promise((resolve) => { lookupEntered = resolve; }); try { providerRuntime.slackFileReceiptResultIds.push("1788.991001"); providerRuntime.slackFileReceiptHook = async () => { lookupEntered(); await lookupReleased; }; const recovery = service.processPendingSlackFileUploadReceipts(1); await lookupBlocked; if (!conversation.resourceId) { throw new Error("Expected Slack channel resource"); } await db .update(chatEndpointResources) .set({ enabled: false, updatedAt: new Date() }) .where( and( eq(chatEndpointResources.id, conversation.resourceId), eq(chatEndpointResources.endpointId, endpoint.id), ), ); releaseLookup(); await expect(recovery).resolves.toBe(1); await expect( db .select({ providerMessageId: chatPublications.providerMessageId, state: chatPublications.state, }) .from(chatPublications) .where(eq(chatPublications.id, publication.id)), ).resolves.toEqual([ { providerMessageId: null, state: "delivery_unknown" }, ]); await expect( db .select({ result: chatActions.result, status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, receipt.id)), ).resolves.toEqual([ { result: { attempts: 1, code: "slack_file_upload_receipt_authorization_changed", }, status: "cancelled", }, ]); await expect( db .select({ id: chatMessageLinks.id }) .from(chatMessageLinks) .where(eq(chatMessageLinks.publicationId, publication.id)), ).resolves.toHaveLength(0); expect(providerRuntime.slackFilePublicationAttempts).toBe(1); expect(providerRuntime.slackFileReceiptLookups).toHaveLength(1); } finally { releaseLookup(); await retirePublicationFixture(service, endpoint.id); } }); it("retries pre-transport attachment integrity failures without blocking another conversation", async () => { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture, { storage: storage.storage, }); try { const damagedChannel = makeThread({ channelId: "C-BOARD-DAMAGED-FILE", id: "slack:C-BOARD-DAMAGED-FILE:4400.2", name: "board-damaged-file", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: damagedChannel.thread, message: makeMessage({ id: "4400.2", text: "@maya start the damaged-file task", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: "C-BOARD-HEALTHY-FILE", label: "board-healthy-file", availability: "available", enabled: true, }); const healthyChannel = makeThread({ channelId: "C-BOARD-HEALTHY-FILE", id: "slack:C-BOARD-HEALTHY-FILE:4400.3", name: "board-healthy-file", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: healthyChannel.thread, message: makeMessage({ id: "4400.3", text: "@maya start the healthy-file task", mentioned: true, }), trigger: "mention", }); const conversations = await service.listConversations(endpoint.id); const damagedConversation = conversations.find( (conversation) => conversation.externalThreadId === damagedChannel.thread.id, ); const healthyConversation = conversations.find( (conversation) => conversation.externalThreadId === healthyChannel.thread.id, ); if (!damagedConversation || !healthyConversation) { throw new Error("Expected both board attachment conversations"); } const damagedBody = Buffer.from("registered outbound evidence", "utf8"); const healthyBody = Buffer.from("healthy outbound evidence", "utf8"); const damagedStored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${damagedConversation.issueId}`, originalFilename: "damaged.txt", contentType: "text/plain", body: damagedBody, }); const healthyStored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${healthyConversation.issueId}`, originalFilename: "healthy.txt", contentType: "text/plain", body: healthyBody, }); const damagedAttachment = await issueService(db).createAttachment({ issueId: damagedConversation.issueId, provider: damagedStored.provider, objectKey: damagedStored.objectKey, contentType: damagedStored.contentType, byteSize: damagedStored.byteSize, sha256: damagedStored.sha256, originalFilename: damagedStored.originalFilename, createdByUserId: "owner-user", }); const healthyAttachment = await issueService(db).createAttachment({ issueId: healthyConversation.issueId, provider: healthyStored.provider, objectKey: healthyStored.objectKey, contentType: healthyStored.contentType, byteSize: healthyStored.byteSize, sha256: healthyStored.sha256, originalFilename: healthyStored.originalFilename, createdByUserId: "owner-user", }); const damagedReplacement = Buffer.from(damagedBody); damagedReplacement[0] ^= 0xff; storage.objects.set(damagedStored.objectKey, damagedReplacement); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack provider runtime"); providerRuntime.posts.length = 0; const damagedPublication = await service.publishBoardMessage( endpoint.id, damagedConversation.id, "Damaged attachment send", "board-damaged-file-send", "owner-user", [damagedAttachment.id], ); expect(damagedPublication).toMatchObject({ attempts: 1, state: "retry", nextAttemptAt: expect.any(Date), redactedError: "Chat publication attachment integrity changed after registration", }); expect(providerRuntime.posts).toEqual([ { threadId: damagedChannel.thread.id, text: "Damaged attachment send", }, ]); const healthyPublication = await service.publishBoardMessage( endpoint.id, healthyConversation.id, "Healthy attachment send", "board-healthy-file-send", "owner-user", [healthyAttachment.id], ); expect(healthyPublication).toMatchObject({ attempts: 1, state: "published", }); expect(providerRuntime.posts).toEqual([ { threadId: damagedChannel.thread.id, text: "Damaged attachment send", }, { threadId: healthyChannel.thread.id, text: "Healthy attachment send", }, { threadId: healthyChannel.thread.id, text: "", files: [ expect.objectContaining({ data: healthyBody, filename: "healthy.txt", mimeType: "text/plain", }), ], }, ]); storage.objects.set(damagedStored.objectKey, damagedBody); await db .update(chatPublications) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatPublications.id, damagedPublication.id)); await service.processPendingPublications(); await service.processPendingPublications(); await expect( db .select({ attempts: chatPublications.attempts, state: chatPublications.state, }) .from(chatPublications) .where(eq(chatPublications.id, damagedPublication.id)), ).resolves.toEqual([{ attempts: 2, state: "published" }]); expect( providerRuntime.posts.filter( (post) => post.files?.[0] && post.threadId === damagedChannel.thread.id, ), ).toEqual([ { threadId: damagedChannel.thread.id, text: "", files: [ expect.objectContaining({ data: damagedBody, filename: "damaged.txt", mimeType: "text/plain", }), ], }, ]); } finally { await service.shutdown(); } }); it("keeps missing attachment storage retryable but fails invalid metadata before provider transport", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); try { const channel = makeThread({ channelId: "C-BOARD-MISSING-STORAGE", id: "slack:C-BOARD-MISSING-STORAGE:4400.4", name: "board-missing-storage", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "4400.4", text: "@maya start the missing-storage task", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected board attachment conversation"); const body = Buffer.from("missing storage evidence", "utf8"); const attachment = await issueService(db).createAttachment({ issueId: conversation.issueId, provider: "local_disk", objectKey: "issues/missing-storage-evidence.txt", contentType: "text/plain", byteSize: body.byteLength, sha256: createHash("sha256").update(body).digest("hex"), originalFilename: "missing-storage-evidence.txt", createdByUserId: "owner-user", }); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack provider runtime"); providerRuntime.posts.length = 0; const retrying = await service.publishBoardMessage( endpoint.id, conversation.id, "Missing storage attachment send", "board-missing-storage-send", "owner-user", [attachment.id], ); expect(retrying).toMatchObject({ attempts: 1, state: "retry", nextAttemptAt: expect.any(Date), redactedError: "Attachment storage is unavailable for chat publication", }); expect(providerRuntime.posts).toEqual([ { threadId: channel.thread.id, text: "Missing storage attachment send", }, ]); const [attachmentRow] = await db .select({ assetId: issueAttachments.assetId }) .from(issueAttachments) .where(eq(issueAttachments.id, attachment.id)); if (!attachmentRow) throw new Error("Expected attachment metadata"); await db .update(assets) .set({ contentType: "application/x-executable" }) .where(eq(assets.id, attachmentRow.assetId)); await db .update(chatPublications) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatPublications.id, retrying.id)); await service.processPendingPublications(); await expect( db .select({ attempts: chatPublications.attempts, redactedError: chatPublications.redactedError, state: chatPublications.state, }) .from(chatPublications) .where(eq(chatPublications.id, retrying.id)), ).resolves.toEqual([ { attempts: 2, redactedError: "Chat publication attachment is invalid or outside its authorized task comment", state: "failed", }, ]); expect(providerRuntime.posts.every((post) => !post.files?.length)).toBe( true, ); } finally { await service.shutdown(); } }); it("scopes board-send idempotency to one external conversation", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service } = await configuredSlackEndpoint(fixture); const firstChannel = makeThread({ channelId: "C-BOARD-IDEMPOTENCY-ONE", id: "slack:C-BOARD-IDEMPOTENCY-ONE:4401.1", name: "board-idempotency-one", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: firstChannel.thread, message: makeMessage({ id: "4401.1", text: "@maya start the first board-send task", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: "C-BOARD-IDEMPOTENCY-TWO", label: "board-idempotency-two", availability: "available", enabled: true, }); const secondChannel = makeThread({ channelId: "C-BOARD-IDEMPOTENCY-TWO", id: "slack:C-BOARD-IDEMPOTENCY-TWO:4402.1", name: "board-idempotency-two", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: secondChannel.thread, message: makeMessage({ id: "4402.1", text: "@maya start the second board-send task", mentioned: true, }), trigger: "mention", }); const conversations = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); expect(conversations).toHaveLength(2); const firstConversation = conversations.find( (conversation) => conversation.externalThreadId === firstChannel.thread.id, ); const secondConversation = conversations.find( (conversation) => conversation.externalThreadId === secondChannel.thread.id, ); if (!firstConversation || !secondConversation) { throw new Error("Expected both Slack board-send conversations"); } const first = await service.publishBoardMessage( endpoint.id, firstConversation.id, "First task update", "same-tab-request-1234", "owner-user", ); const second = await service.publishBoardMessage( endpoint.id, secondConversation.id, "Second task update", "same-tab-request-1234", "owner-user", ); expect(second.id).not.toBe(first.id); expect(second.conversationId).toBe(secondConversation.id); await expect( db .select({ body: issueComments.body, issueId: issueComments.issueId }) .from(issueComments) .where(inArray(issueComments.id, [first.commentId!, second.commentId!])) .orderBy(asc(issueComments.body)), ).resolves.toEqual([ { body: "First task update", issueId: firstConversation.issueId }, { body: "Second task update", issueId: secondConversation.issueId }, ]); }); it("publishes an explicitly selected board attachment through Telegram's native file lane", async () => { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture, { storage: storage.storage, }); const chatId = "77119911"; const dm = makeThread({ channelId: `telegram:${chatId}`, id: `telegram:${chatId}`, isDM: true, name: "Telegram board file", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: `${chatId}:1`, text: "Send the selected result here", userId: chatId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, chatId); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Telegram file conversation"); const body = Buffer.from("telegram selected result", "utf8"); const stored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${conversation.issueId}`, originalFilename: "telegram-result.txt", contentType: "text/plain", body, }); const attachment = await issueService(db).createAttachment({ issueId: conversation.issueId, provider: stored.provider, objectKey: stored.objectKey, contentType: stored.contentType, byteSize: stored.byteSize, sha256: stored.sha256, originalFilename: stored.originalFilename, createdByUserId: "owner-user", }); await service.publishBoardMessage( endpoint.id, conversation.id, "Telegram file delivery", "telegram-file-delivery-1", "owner-user", [attachment.id], ); expect(runtime.endpoints.get(endpoint.id)?.posts).toEqual([ { threadId: dm.thread.id, text: "Telegram file delivery" }, { threadId: dm.thread.id, text: "Shared telegram-result.txt.", attachments: [ expect.objectContaining({ name: "telegram-result.txt", mimeType: "text/plain", data: body, type: "file", }), ], }, ]); await expect( db .select({ attempts: chatPublications.attempts, state: chatPublications.state, }) .from(chatPublications) .where( like( chatPublications.idempotencyKey, `explicit-board:${endpoint.id}:${conversation.id}:telegram-file-delivery-1%`, ), ) .orderBy(asc(chatPublications.createdAt), asc(chatPublications.id)), ).resolves.toEqual([ { attempts: 1, state: "published" }, { attempts: 1, state: "published" }, ]); }); it.each([ "ratio_20", "ratio_21", "portrait_21", "dimensions_10000", "dimensions_10001", "jpeg", "gif", "webp", "malformed", "photo_size_limit", "photo_size_over", "ambiguous", ] as const)( "selects the Telegram photo boundary before provider I/O (%s)", async (mode) => { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture, { storage: storage.storage }); let pinned: ReturnType | undefined; let threadSpy: ReturnType | undefined; try { const chatId = "77119914"; const dm = makeThread({ channelId: `telegram:${chatId}`, id: `telegram:${chatId}`, isDM: true, name: "Telegram photo boundaries", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: `${chatId}:1`, text: "Return this exact image", userId: chatId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, chatId); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected photo test conversation"); const [width, height] = mode === "ratio_20" ? [200, 10] : mode === "ratio_21" ? [210, 10] : mode === "portrait_21" ? [10, 210] : mode === "dimensions_10000" ? [9500, 500] : mode === "dimensions_10001" ? [9501, 500] : [16, 16]; const format = mode === "jpeg" || mode === "gif" || mode === "webp" ? mode : "png"; let bytes = await sharp({ create: { width, height, channels: 3, background: "#e08040" }, }) .toFormat(format) .toBuffer(); if (mode === "malformed") bytes = bytes.subarray(0, 20); if (mode === "photo_size_limit" || mode === "photo_size_over") { // A genuine PNG with a legal uncompressed ancillary text chunk; // exercise upload bytes without allocating/decompressing huge pixels. const target = 10_000_000 + Number(mode === "photo_size_over"); const payload = Buffer.alloc(target - bytes.length - 12, 0x78); payload.write("padding\0", 0, "ascii"); const chunk = Buffer.alloc(payload.length + 12); chunk.writeUInt32BE(payload.length); chunk.write("tEXt", 4, "ascii"); payload.copy(chunk, 8); chunk.writeUInt32BE(crc32(chunk.subarray(4, -4)), chunk.length - 4); bytes = Buffer.concat([ bytes.subarray(0, -12), chunk, bytes.subarray(-12), ]); expect(bytes.length).toBe(target); } if (mode !== "malformed") { await expect(sharp(bytes).metadata()).resolves.toMatchObject({ width, height, format, }); } const filename = `original-${mode}.${format}`; const mimeType = `image/${format}`; const stored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${conversation.issueId}`, originalFilename: filename, contentType: mimeType, body: bytes, }); const attachment = await issueService(db).createAttachment({ issueId: conversation.issueId, provider: stored.provider, objectKey: stored.objectKey, contentType: stored.contentType, byteSize: stored.byteSize, sha256: stored.sha256, originalFilename: stored.originalFilename, createdByUserId: "owner-user", }); const expectedMethod = [ "ratio_21", "portrait_21", "dimensions_10001", "gif", "webp", "malformed", "photo_size_over", ].includes(mode) ? "sendDocument" : "sendPhoto"; const methods: string[] = []; const uploadedBytes: Buffer[] = []; vi.stubGlobal( "fetch", vi.fn(async (input: string | URL | Request, init?: RequestInit) => { const url = new URL(String(input)); expect(url.hostname).toBe("api.telegram.org"); const method = url.pathname.split("/").at(-1)!; methods.push(method); if (method === "sendPhoto" || method === "sendDocument") { expect(init?.body).toBeInstanceOf(FormData); const file = (init!.body as FormData).get( method === "sendPhoto" ? "photo" : "document", ) as File; expect(file.name).toBe(filename); expect(file.type).toBe(mimeType); uploadedBytes.push(Buffer.from(await file.arrayBuffer())); if (mode === "ambiguous") throw new TypeError("fetch failed", { cause: Object.assign(new Error("synthetic connection loss"), { code: "UND_ERR_SOCKET", }), }); if (expectedMethod === "sendDocument" && method === "sendPhoto") { return Response.json( { ok: false, error_code: 400, description: "Bad Request: PHOTO_INVALID_DIMENSIONS", }, { status: 400 }, ); } } else expect(method).toBe("sendRichMessage"); return Response.json({ ok: true, result: { message_id: 100 + methods.length, date: 1_788_700_002, chat: { id: Number(chatId), type: "private" }, }, }); }), ); pinned = createChatSdkEndpointRuntime({ companyId: fixture.companyId, endpointId: endpoint.id, callbacks: { onMessage() {} }, logger: "silent", persistence: { async compareAndSet() { return true; }, async deleteIfVersion() { return true; }, async read() { return null; }, }, providerConfig: { provider: "telegram", userName: "photo_fixture_bot", credentials: { botToken: "123:synthetic-photo-fixture", secretToken: "synthetic", }, }, }); const adapter = pinned.getProviderAdapter(); const endpointRuntime = runtime.endpoints.get(endpoint.id)!; const originalThread = endpointRuntime.thread.bind(endpointRuntime); threadSpy = vi .spyOn(endpointRuntime, "thread") .mockImplementation((threadId) => ({ ...originalThread(threadId), post: async (message: unknown) => { const posted = await adapter.postMessage( threadId, message as Parameters[1], ); return { id: posted.id, threadId }; }, })); const result = await service.publishBoardMessage( endpoint.id, conversation.id, "Original image attached", `photo-boundary-${mode}`, "owner-user", [attachment.id], ); expect(methods).toEqual(["sendRichMessage", expectedMethod]); expect(uploadedBytes).toHaveLength(1); // Buffer.equals still compares every byte, without Vitest expanding // a ten-megabyte boundary fixture into a recursive object comparison. expect(uploadedBytes[0]!.equals(bytes)).toBe(true); expect(result.state).toBe( mode === "ambiguous" ? "delivery_unknown" : "published", ); const publications = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), eq(chatPublications.commentId, result.commentId!), ), ) .orderBy(asc(chatPublications.createdAt)); expect( publications.map((row) => ({ state: row.state, attempts: row.attempts, })), ).toEqual([ { state: "published", attempts: 1 }, { state: mode === "ambiguous" ? "delivery_unknown" : "published", attempts: 1, }, ]); await service.processPendingPublications(); expect(methods).toEqual(["sendRichMessage", expectedMethod]); } finally { threadSpy?.mockRestore(); await pinned?.shutdown(); vi.unstubAllGlobals(); await retirePublicationFixture(service, endpoint.id); } }, ); it("maps Telegram image, audio, and video output onto native media lanes", async () => { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture, { storage: storage.storage, }); const chatId = "77119912"; const dm = makeThread({ channelId: `telegram:${chatId}`, id: `telegram:${chatId}`, isDM: true, name: "Telegram native media", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: `${chatId}:1`, text: "Send native media", userId: chatId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, chatId); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Telegram media conversation"); const media = [ { contentType: "image/png", filename: "result.png", type: "image", }, { contentType: "audio/mpeg", filename: "result.mp3", type: "audio", }, { contentType: "audio/mp4", filename: "result.m4a", type: "audio", }, { contentType: "video/mp4", filename: "result.mp4", type: "video", }, ] as const; const attachmentIds: string[] = []; for (const item of media) { const body = item.type === "image" ? await sharp({ create: { width: 16, height: 16, channels: 3, background: "#e08040", }, }) .png() .toBuffer() : Buffer.from(`native-${item.type}`, "utf8"); const stored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${conversation.issueId}`, originalFilename: item.filename, contentType: item.contentType, body, }); const attachment = await issueService(db).createAttachment({ issueId: conversation.issueId, provider: stored.provider, objectKey: stored.objectKey, contentType: stored.contentType, byteSize: stored.byteSize, sha256: stored.sha256, originalFilename: stored.originalFilename, createdByUserId: "owner-user", }); attachmentIds.push(attachment.id); } await service.publishBoardMessage( endpoint.id, conversation.id, "Telegram native media delivery", "telegram-native-media-delivery-1", "owner-user", attachmentIds, ); const posts = runtime.endpoints.get(endpoint.id)?.posts ?? []; expect(posts[0]).toEqual({ threadId: dm.thread.id, text: "Telegram native media delivery", }); expect(posts.slice(1)).toEqual( media.map((item) => ({ threadId: dm.thread.id, text: `Shared ${item.filename}.`, attachments: [ expect.objectContaining({ mimeType: item.contentType, name: item.filename, type: item.type, }), ], })), ); expect(posts.slice(1).every((post) => post.files === undefined)).toBe(true); }); it("sends Telegram audio and video outside native format contracts as exact original documents", async () => { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture, { storage: storage.storage }); try { const chatId = "77119913"; const dm = makeThread({ channelId: `telegram:${chatId}`, id: `telegram:${chatId}`, isDM: true, name: "Telegram exact media files", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: `${chatId}:1`, text: "Return the original files", userId: chatId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, chatId); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); const media = [ { mimeType: "audio/ogg", filename: "voice.ogg" }, { mimeType: "audio/wav", filename: "recording.wav" }, { mimeType: "audio/webm", filename: "recording.webm" }, { mimeType: "video/webm", filename: "clip.webm" }, { mimeType: "video/quicktime", filename: "clip.mov" }, { mimeType: "video/x-m4v", filename: "clip.m4v" }, ].map((item) => ({ ...item, body: Buffer.from(`original:${item.mimeType}:\u0000exact bytes\n`), })); const attachmentIds: string[] = []; for (const item of media) { const stored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${conversation!.issueId}`, originalFilename: item.filename, contentType: item.mimeType, body: item.body, }); const attachment = await issueService(db).createAttachment({ issueId: conversation!.issueId, provider: stored.provider, objectKey: stored.objectKey, contentType: stored.contentType, byteSize: stored.byteSize, sha256: stored.sha256, originalFilename: stored.originalFilename, createdByUserId: "owner-user", }); attachmentIds.push(attachment.id); } const key = "telegram-nonnative-media-exact-originals"; await service.publishBoardMessage( endpoint.id, conversation!.id, "Original media files", key, "owner-user", attachmentIds, ); const posts = runtime.endpoints.get(endpoint.id)!.posts; const filePosts = posts.filter((post) => post.attachments?.length); expect(filePosts).toHaveLength(media.length); expect(filePosts).toEqual( media.map((item) => ({ threadId: dm.thread.id, text: `Shared ${item.filename}.`, attachments: [ expect.objectContaining({ type: "file", data: item.body, mimeType: item.mimeType, name: item.filename, size: item.body.length, }), ], })), ); const publications = await db .select() .from(chatPublications) .where(eq(chatPublications.endpointId, endpoint.id)); const filePublications = publications.filter( (row) => (row.payload.attachmentIds as unknown[] | undefined)?.length, ); expect(filePublications).toHaveLength(media.length); expect( filePublications.every( (row) => row.state === "published" && row.attempts === 1, ), ).toBe(true); const postCount = posts.length; await service.publishBoardMessage( endpoint.id, conversation!.id, "Original media files", key, "owner-user", attachmentIds, ); await service.processPendingPublications(); expect(posts).toHaveLength(postCount); } finally { await retirePublicationFixture(service, endpoint.id); } }); it("publishes an agent's explicitly selected same-run Slack attachment after its response", async () => { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture, { storage: storage.storage, }); const channel = makeThread({ channelId: "C-AGENT-FILE", id: "slack:C-AGENT-FILE:4410.1", name: "agent-file", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "4410.1", text: "@maya create and share a result", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "slack", providerMessageId: "4410.1", }), }); const stored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${conversation.issueId}`, originalFilename: "agent-result.txt", contentType: "text/plain", body: Buffer.from("agent result", "utf8"), }); const attachment = await issueService(db).createAttachment({ issueId: conversation.issueId, provider: stored.provider, objectKey: stored.objectKey, contentType: stored.contentType, byteSize: stored.byteSize, sha256: stored.sha256, originalFilename: stored.originalFilename, createdByAgentId: fixture.assignedAgentId, createdByRunId: runId, }); const authorizationReason = await resolveChatRunPresentationAuthorizationReason(db, { companyId: fixture.companyId, issueId: conversation.issueId, runId, }); expect(authorizationReason).toBe("allow_chat_run_presentation"); const response = await db.transaction((tx) => issueService(tx as unknown as TestDb).addComment( conversation.issueId, "The requested result is attached.", { agentId: fixture.assignedAgentId, runId }, { attachmentIds: [attachment.id], authorType: "agent", authorizationReason, }, ), ); const endpointRuntime = runtime.endpoints.get(endpoint.id); if (!endpointRuntime) throw new Error("Expected Slack endpoint runtime"); endpointRuntime.postResultIds.push("1788.301", "1788.302"); await service.processPendingPublications(); const publications = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, response.id)) .orderBy(asc(chatPublications.createdAt)); expect(publications).toEqual([ expect.objectContaining({ state: "published", payload: { text: "The requested result is attached." }, }), expect.objectContaining({ state: "published", providerMessageId: "1788.302", payload: expect.objectContaining({ attachmentIds: [attachment.id] }), }), ]); expect(runtime.endpoints.get(endpoint.id)?.posts).toEqual([ { threadId: channel.thread.id, text: "The requested result is attached.", }, { threadId: channel.thread.id, text: "", files: [ expect.objectContaining({ filename: "agent-result.txt", data: Buffer.from("agent result", "utf8"), }), ], }, ]); const [fileMessageLink] = await db .select() .from(chatMessageLinks) .where( and( eq(chatMessageLinks.endpointId, endpoint.id), eq(chatMessageLinks.conversationId, conversation.id), eq(chatMessageLinks.direction, "outbound"), eq(chatMessageLinks.providerMessageId, "1788.302"), ), ); expect(fileMessageLink).toMatchObject({ commentId: response.id, providerMessageId: "1788.302", }); if (!callbacks.onReaction) throw new Error("Slack reaction callback was not registered"); const fileMessage = makeMessage({ id: "1788.302", text: "", mentioned: false, }); await callbacks.onReaction({ endpointId: endpoint.id, provider: "slack", event: { adapter: {} as never, added: true, emoji: { name: "thumbs_up", toJSON: () => ":thumbs_up:", toString: () => ":thumbs_up:", }, message: fileMessage, messageId: fileMessage.id, raw: { event_ts: "1788.400" }, rawEmoji: "+1", thread: channel.thread, threadId: channel.thread.id, user: fileMessage.author, }, }); const [reactionDelivery] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.conversationId, conversation.id), eq(chatDeliveries.eventKind, "reaction_added"), ), ); expect(reactionDelivery).toMatchObject({ state: "processed", normalizedEvent: expect.objectContaining({ message: { providerMessageId: "1788.302" }, reaction: expect.objectContaining({ emoji: "thumbs_up" }), }), }); }); it("hands an explicitly bound same-run image to Discord's final native file response", async () => { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, runtime, service } = await configuredDiscordEndpoint(fixture, { storage: storage.storage, }); const rootMessageId = "555555555555555710"; const channel = makeThread({ channelId: "333333333333333710", id: `discord:1457808928258658549:333333333333333710:${rootMessageId}`, name: "agent-image", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "discord", thread: channel.thread, message: makeMessage({ id: rootMessageId, text: "@maya create and show an image", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); if (!conversation) throw new Error("Expected Discord conversation"); const insertRun = async (agentId: string, status = "succeeded") => { const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId, status, contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "discord", providerMessageId: rootMessageId, }), }); return runId; }; const createAgentAttachment = async ( runId: string, agentId: string, filename: string, body: Buffer, contentType = "image/png", ) => { const stored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${conversation.issueId}`, originalFilename: filename, contentType, body, }); return issueService(db).createAttachment({ issueId: conversation.issueId, provider: stored.provider, objectKey: stored.objectKey, contentType: stored.contentType, byteSize: stored.byteSize, sha256: stored.sha256, originalFilename: stored.originalFilename, createdByAgentId: agentId, createdByRunId: runId, }); }; const bindAgentAttachment = ( runId: string, agentId: string, body: string, attachmentId: string, authorizationReason = "allow_self", ) => db.transaction((tx) => issueService(tx as unknown as TestDb).addComment( conversation.issueId, body, { agentId, runId }, { attachmentIds: [attachmentId], authorType: "agent", authorizationReason, }, ), ); const runId = await insertRun(fixture.assignedAgentId, "running"); const imageBody = Buffer.from("generated cat image", "utf8"); const selected = await createAgentAttachment( runId, fixture.assignedAgentId, "orange-tabby.png", imageBody, ); expect(selected.originatingRunId).toBe(runId); const documentBody = Buffer.from("%PDF-1.7 generated report", "utf8"); const selectedDocument = await createAgentAttachment( runId, fixture.assignedAgentId, "cat-notes.pdf", documentBody, "application/pdf", ); const unbound = await createAgentAttachment( runId, fixture.assignedAgentId, "private-draft.png", Buffer.from("private unbound draft", "utf8"), ); const directBody = Buffer.from("directly selected image", "utf8"); const directSelectionComment = await issueService(db).addComment( conversation.issueId, "Recorded a direct upload before final presentation.", { agentId: fixture.assignedAgentId, runId }, { authorType: "agent", authorizationReason: "paperclip_runner_protocol" }, ); const storedDirect = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${conversation.issueId}`, originalFilename: "direct-selection.png", contentType: "image/png", body: directBody, }); const directSelection = await issueService(db).createAttachment({ issueId: conversation.issueId, issueCommentId: directSelectionComment.id, provider: storedDirect.provider, objectKey: storedDirect.objectKey, contentType: storedDirect.contentType, byteSize: storedDirect.byteSize, sha256: storedDirect.sha256, originalFilename: storedDirect.originalFilename, createdByAgentId: fixture.assignedAgentId, createdByRunId: runId, }); expect(directSelection.originatingRunId).toBe(runId); await expect( db .select({ id: chatPublications.id }) .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `attachment:${directSelection.id}:${endpoint.id}`, ), ), ).resolves.toHaveLength(0); const deletedSelectionComment = await issueService(db).addComment( conversation.issueId, "This selection was withdrawn before upload.", { agentId: fixture.assignedAgentId, runId }, { authorType: "agent", authorizationReason: "allow_self" }, ); await db .update(issueComments) .set({ deletedAt: new Date() }) .where(eq(issueComments.id, deletedSelectionComment.id)); await expect( issueService(db).createAttachment({ issueId: conversation.issueId, issueCommentId: deletedSelectionComment.id, provider: "local_disk", objectKey: "issues/deleted-parent.png", contentType: "image/png", byteSize: 14, sha256: "7".repeat(64), originalFilename: "deleted-parent.png", createdByAgentId: fixture.assignedAgentId, createdByRunId: runId, }), ).rejects.toMatchObject({ status: 404 }); await expect( db .select({ id: assets.id }) .from(assets) .where(eq(assets.objectKey, "issues/deleted-parent.png")), ).resolves.toHaveLength(0); const provisionalSelection = await issueService(db).addComment( conversation.issueId, "Selected the requested cat image for chat delivery.", { agentId: fixture.assignedAgentId, runId }, { authorType: "agent", authorizationReason: "internal_agent_write" }, ); const selectionComment = await bindAgentAttachment( runId, fixture.assignedAgentId, "Selected the requested cat image for chat delivery.", selected.id, ); expect(selectionComment.id).toBe(provisionalSelection.id); // A lost HTTP response may make the helper retry the same binding. The // retry must reuse the comment and must not fail or duplicate the file. const selectionRetry = await bindAgentAttachment( runId, fixture.assignedAgentId, "Selected the requested cat image for chat delivery.", selected.id, ); expect(selectionRetry.id).toBe(selectionComment.id); const documentSelectionComment = await bindAgentAttachment( runId, fixture.assignedAgentId, "Selected the requested notes for chat delivery.", selectedDocument.id, ); const otherRunId = await insertRun(fixture.assignedAgentId); const otherRunAttachment = await createAgentAttachment( otherRunId, fixture.assignedAgentId, "other-run.png", Buffer.from("other run", "utf8"), ); const otherRunComment = await bindAgentAttachment( otherRunId, fixture.assignedAgentId, "This belongs to another run.", otherRunAttachment.id, ); await expect( issueService(db).createAttachment({ issueId: conversation.issueId, issueCommentId: otherRunComment.id, provider: "local_disk", objectKey: "issues/wrong-run-parent.png", contentType: "image/png", byteSize: 16, sha256: "9".repeat(64), originalFilename: "wrong-run-parent.png", createdByAgentId: fixture.assignedAgentId, createdByRunId: runId, }), ).rejects.toMatchObject({ status: 422, details: { code: "issue_attachment_parent_run_mismatch" }, }); const remintedAttachment = await createAgentAttachment( otherRunId, fixture.assignedAgentId, "reminted-old-run.png", Buffer.from("old run provenance", "utf8"), ); if (!remintedAttachment.artifactWorkProductId) { throw new Error("Expected upload to create an artifact work product"); } await db .update(issueWorkProducts) .set({ createdByRunId: runId }) .where( eq(issueWorkProducts.id, remintedAttachment.artifactWorkProductId), ); await expect( bindAgentAttachment( runId, fixture.assignedAgentId, "A mutable work-product update cannot change upload provenance.", remintedAttachment.id, ), ).rejects.toMatchObject({ status: 422, details: { code: "issue_attachment_run_origin_mismatch" }, }); const [mutableWorkProduct] = await db .select() .from(issueWorkProducts) .where( eq(issueWorkProducts.id, remintedAttachment.artifactWorkProductId), ); if (!mutableWorkProduct) { throw new Error("Expected reminted attachment work product"); } await db .delete(issueWorkProducts) .where( eq(issueWorkProducts.id, remintedAttachment.artifactWorkProductId), ); await db.insert(issueWorkProducts).values({ ...mutableWorkProduct, id: randomUUID(), createdByRunId: runId, createdAt: new Date(), updatedAt: new Date(), }); await expect( bindAgentAttachment( runId, fixture.assignedAgentId, "Deleting and recreating metadata cannot change upload provenance.", remintedAttachment.id, ), ).rejects.toMatchObject({ status: 422, details: { code: "issue_attachment_run_origin_mismatch" }, }); const legacyAttachment = await createAgentAttachment( runId, fixture.assignedAgentId, "legacy-without-origin.png", Buffer.from("legacy provenance unavailable", "utf8"), ); await db .update(issueAttachments) .set({ originatingRunId: null }) .where(eq(issueAttachments.id, legacyAttachment.id)); await expect( bindAgentAttachment( runId, fixture.assignedAgentId, "Legacy attachment provenance fails closed.", legacyAttachment.id, ), ).rejects.toMatchObject({ status: 422, details: { code: "issue_attachment_run_origin_mismatch" }, }); const otherAgentRunId = await insertRun(fixture.replacementAgentId); const otherAgentAttachment = await createAgentAttachment( otherAgentRunId, fixture.replacementAgentId, "other-agent.png", Buffer.from("other agent", "utf8"), ); await bindAgentAttachment( otherAgentRunId, fixture.replacementAgentId, "This belongs to another agent.", otherAgentAttachment.id, ); const authorizationReason = await resolveChatRunPresentationAuthorizationReason(db, { companyId: fixture.companyId, issueId: conversation.issueId, runId, }); expect(authorizationReason).toBe("allow_chat_run_presentation"); await db .update(heartbeatRuns) .set({ status: "succeeded", finishedAt: new Date() }) .where(eq(heartbeatRuns.id, runId)); const response = await issueService(db).addComment( conversation.issueId, "Here is the requested cat image.", { agentId: fixture.assignedAgentId, runId }, { authorType: "agent", authorizationReason }, ); await service.processPendingPublications(); const publications = await db .select() .from(chatPublications) .where( and( eq(chatPublications.conversationId, conversation.id), inArray(chatPublications.commentId, [ response.id, selectionComment.id, documentSelectionComment.id, directSelectionComment.id, ]), ), ) .orderBy(asc(chatPublications.createdAt)); expect(publications).toEqual([ expect.objectContaining({ commentId: response.id, state: "published", payload: { text: "Here is the requested cat image." }, }), expect.objectContaining({ commentId: directSelectionComment.id, state: "published", payload: expect.objectContaining({ attachmentIds: [directSelection.id], }), }), expect.objectContaining({ commentId: selectionComment.id, state: "published", payload: expect.objectContaining({ attachmentIds: [selected.id] }), }), expect.objectContaining({ commentId: documentSelectionComment.id, state: "published", payload: expect.objectContaining({ attachmentIds: [selectedDocument.id], }), }), ]); expect(JSON.stringify(publications)).not.toContain(unbound.id); expect(JSON.stringify(publications)).not.toContain(otherRunAttachment.id); expect(JSON.stringify(publications)).not.toContain(remintedAttachment.id); expect(JSON.stringify(publications)).not.toContain(legacyAttachment.id); expect(JSON.stringify(publications)).not.toContain(otherAgentAttachment.id); expect(runtime.endpoints.get(endpoint.id)?.posts).toEqual([ { threadId: channel.thread.id, text: "Here is the requested cat image.", }, { threadId: channel.thread.id, text: "Shared direct-selection.png.", files: [ expect.objectContaining({ filename: "direct-selection.png", mimeType: "image/png", data: directBody, }), ], }, { threadId: channel.thread.id, text: "Shared orange-tabby.png.", files: [ expect.objectContaining({ filename: "orange-tabby.png", mimeType: "image/png", data: imageBody, }), ], }, { threadId: channel.thread.id, text: "Shared cat-notes.pdf.", files: [ expect.objectContaining({ filename: "cat-notes.pdf", mimeType: "application/pdf", data: documentBody, }), ], }, ]); await expect( db .select({ issueCommentId: issueAttachments.issueCommentId }) .from(issueAttachments) .where(eq(issueAttachments.id, unbound.id)), ).resolves.toEqual([{ issueCommentId: null }]); }); it("serializes and caps an agent run's selected Discord files before final handoff", async () => { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, service } = await configuredDiscordEndpoint( fixture, { storage: storage.storage }, ); const rootMessageId = "555555555555555711"; const channel = makeThread({ channelId: "333333333333333711", id: `discord:1457808928258658549:333333333333333711:${rootMessageId}`, name: "agent-file-cap", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "discord", thread: channel.thread, message: makeMessage({ id: rootMessageId, text: "@maya prepare the complete file set", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); if (!conversation) throw new Error("Expected Discord conversation"); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "discord", providerMessageId: rootMessageId, }), }); const attachments: Array<{ id: string }> = []; for (let index = 0; index < 21; index += 1) { const body = Buffer.from(`selected file ${index + 1}`, "utf8"); const stored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${conversation.issueId}`, originalFilename: `selected-${String(index + 1).padStart(2, "0")}.txt`, contentType: "text/plain", body, }); attachments.push( await issueService(db).createAttachment({ issueId: conversation.issueId, provider: stored.provider, objectKey: stored.objectKey, contentType: stored.contentType, byteSize: stored.byteSize, sha256: stored.sha256, originalFilename: stored.originalFilename, createdByAgentId: fixture.assignedAgentId, createdByRunId: runId, }), ); } const issuesSvc = issueService(db); const initialSelection = await db.transaction((tx) => issuesSvc.addComment( conversation.issueId, "Prepared the selected file batch.", { agentId: fixture.assignedAgentId, runId }, { attachmentIds: attachments.slice(0, 19).map((item) => item.id), authorType: "agent", authorizationReason: "allow_self", }, tx, ), ); const contenderBodies = [ "Prepared candidate file twenty.", "Prepared candidate file twenty-one.", ]; const contenderResults = await Promise.allSettled( attachments.slice(19).map((attachment, index) => db.transaction((tx) => issuesSvc.addComment( conversation.issueId, contenderBodies[index]!, { agentId: fixture.assignedAgentId, runId }, { attachmentIds: [attachment.id], authorType: "agent", authorizationReason: "allow_self", }, tx, ), ), ), ); const winnerIndex = contenderResults.findIndex( (result) => result.status === "fulfilled", ); const loserIndex = contenderResults.findIndex( (result) => result.status === "rejected", ); expect(winnerIndex).toBeGreaterThanOrEqual(0); expect(loserIndex).toBeGreaterThanOrEqual(0); if (winnerIndex < 0 || loserIndex < 0) { throw new Error("Expected exactly one capped attachment contender"); } const rejected = contenderResults[loserIndex]; if (rejected?.status !== "rejected") { throw new Error("Expected the twenty-first attachment to be rejected"); } expect(rejected.reason).toMatchObject({ status: 422, details: { code: "chat_attachment_selection_limit_exceeded", limit: 20, selectedCount: 21, }, }); const winningAttachment = attachments[19 + winnerIndex]!; const rejectedAttachment = attachments[19 + loserIndex]!; await expect( db .select({ id: issueAttachments.id, issueCommentId: issueAttachments.issueCommentId, }) .from(issueAttachments) .where( inArray( issueAttachments.id, attachments.map((item) => item.id), ), ) .then((rows) => ({ bound: rows.filter((row) => row.issueCommentId !== null).length, rejected: rows.find((row) => row.id === rejectedAttachment.id), })), ).resolves.toEqual({ bound: 20, rejected: { id: rejectedAttachment.id, issueCommentId: null, }, }); await expect( db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.body, contenderBodies[loserIndex]!)), ).resolves.toHaveLength(0); await expect( issuesSvc.createAttachment({ issueId: conversation.issueId, issueCommentId: initialSelection.id, provider: "local_disk", objectKey: "issues/direct-selection-overflow.txt", contentType: "text/plain", byteSize: 25, sha256: "8".repeat(64), originalFilename: "direct-selection-overflow.txt", createdByAgentId: fixture.assignedAgentId, createdByRunId: runId, }), ).rejects.toMatchObject({ status: 422, details: { code: "chat_attachment_selection_limit_exceeded", limit: 20, selectedCount: 21, }, }); await expect( db .select({ id: assets.id }) .from(assets) .where(eq(assets.objectKey, "issues/direct-selection-overflow.txt")), ).resolves.toHaveLength(0); const authorizationReason = await resolveChatRunPresentationAuthorizationReason(db, { companyId: fixture.companyId, issueId: conversation.issueId, runId, }); expect(authorizationReason).toBe("allow_chat_run_presentation"); // Reusing the first selected attachment on the final comment makes that // file appear in both the direct and carried sets. Dedupe must happen // before the provider cap is applied, or the later winning file is lost. const response = await db.transaction((tx) => issuesSvc.addComment( conversation.issueId, "Prepared the selected file batch.", { agentId: fixture.assignedAgentId, runId }, { attachmentIds: [attachments[0]!.id], authorType: "agent", authorizationReason, }, tx, ), ); expect(response.id).toBe(initialSelection.id); await service.processPendingPublications(1_000); const publications = await db .select({ payload: chatPublications.payload, state: chatPublications.state, }) .from(chatPublications) .where(eq(chatPublications.conversationId, conversation.id)); const publishedAttachmentIds = publications.flatMap((publication) => { const attachmentIds = publication.payload && typeof publication.payload === "object" && "attachmentIds" in publication.payload && Array.isArray(publication.payload.attachmentIds) ? publication.payload.attachmentIds : []; return attachmentIds.filter( (attachmentId): attachmentId is string => typeof attachmentId === "string", ); }); expect( publications.every((publication) => publication.state === "published"), ).toBe(true); expect(publishedAttachmentIds).toHaveLength(20); expect(new Set(publishedAttachmentIds).size).toBe(20); expect(publishedAttachmentIds).toEqual( expect.arrayContaining([ ...attachments.slice(0, 19).map((item) => item.id), winningAttachment.id, ]), ); expect(publishedAttachmentIds).not.toContain(rejectedAttachment.id); const nonChatRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: nonChatRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: { issueId: conversation.issueId, source: "issue.comment", }, }); const nonChatAttachments: Array<{ id: string }> = []; for (let index = 0; index < 21; index += 1) { nonChatAttachments.push( await issuesSvc.createAttachment({ issueId: conversation.issueId, provider: "local_disk", objectKey: `issues/non-chat-${index + 1}.txt`, contentType: "text/plain", byteSize: 1, sha256: String(index).padStart(64, "0"), originalFilename: `non-chat-${index + 1}.txt`, createdByAgentId: fixture.assignedAgentId, createdByRunId: nonChatRunId, }), ); } const firstInternalBatch = await db.transaction((tx) => issuesSvc.addComment( conversation.issueId, "Internal task files one through twenty.", { agentId: fixture.assignedAgentId, runId: nonChatRunId }, { attachmentIds: nonChatAttachments.slice(0, 20).map((item) => item.id), authorType: "agent", authorizationReason: "allow_self", }, tx, ), ); const secondInternalBatch = await db.transaction((tx) => issuesSvc.addComment( conversation.issueId, "Internal task file twenty-one.", { agentId: fixture.assignedAgentId, runId: nonChatRunId }, { attachmentIds: [nonChatAttachments[20]!.id], authorType: "agent", authorizationReason: "allow_self", }, tx, ), ); expect(firstInternalBatch.id).not.toBe(secondInternalBatch.id); await expect( db .select({ issueCommentId: issueAttachments.issueCommentId }) .from(issueAttachments) .where( inArray( issueAttachments.id, nonChatAttachments.map((item) => item.id), ), ) .then((rows) => rows.filter((attachment) => attachment.issueCommentId !== null), ), ).resolves.toHaveLength(21); }); it("publishes a serialized Telegram run response after the preceding run completes the conversation", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture); const dm = makeThread({ channelId: "77119922", id: "telegram:77119922", isDM: true, name: "Telegram completion race", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: "77119922:1", text: "Start the setup conversation", userId: "77119922", }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, "77119922"); await service.test(endpoint.id, "owner-user"); for (const [id, text] of [ ["77119922:2", "First queued question"], ["77119922:3", "Second queued question"], ] as const) { await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id, text, userId: "77119922" }), trigger: "direct_message", }); } const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const inboundLinks = await db .select() .from(chatMessageLinks) .where( and( eq(chatMessageLinks.endpointId, endpoint.id), eq(chatMessageLinks.direction, "inbound"), inArray(chatMessageLinks.providerMessageId, [ "77119922:2", "77119922:3", ]), ), ); const firstWakeCommentId = inboundLinks.find( (link) => link.providerMessageId === "77119922:2", )?.commentId; const secondWakeCommentId = inboundLinks.find( (link) => link.providerMessageId === "77119922:3", )?.commentId; if (!firstWakeCommentId || !secondWakeCommentId) { throw new Error("Expected both inbound Telegram comments to be linked"); } const firstRunId = randomUUID(); const secondRunId = randomUUID(); const internalRecoveryRunId = randomUUID(); await db.insert(heartbeatRuns).values([ { id: firstRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: { issueId: conversation.issueId, source: "chat:telegram", wakeCommentId: firstWakeCommentId, wakeCommentIds: [firstWakeCommentId], }, }, { id: secondRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: { issueId: conversation.issueId, source: "chat:telegram", wakeCommentId: secondWakeCommentId, wakeCommentIds: [secondWakeCommentId], }, }, { id: internalRecoveryRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: { issueId: conversation.issueId, source: "issue.comment", wakeReason: "finish_successful_run_handoff", wakeSource: "automation", }, }, ]); const firstResponse = await addSelectedChatFinal({ agentId: fixture.assignedAgentId, body: "First queued answer", companyId: fixture.companyId, issueId: conversation.issueId, runId: firstRunId, }); await service.processPendingPublications(); const activeInternalComment = await issueService(db).addComment( conversation.issueId, "Active recovery note that must stay internal", { agentId: fixture.assignedAgentId, runId: internalRecoveryRunId, }, { authorType: "agent" }, ); const internalAttachment = await issueService(db).createAttachment({ issueId: conversation.issueId, issueCommentId: activeInternalComment.id, provider: "local_disk", objectKey: "issues/internal-recovery.txt", contentType: "text/plain", byteSize: 18, sha256: "a".repeat(64), originalFilename: "internal-recovery.txt", createdByAgentId: fixture.assignedAgentId, createdByRunId: internalRecoveryRunId, }); await db .update(issues) .set({ status: "done" }) .where(eq(issues.id, conversation.issueId)); await db .update(chatConversations) .set({ state: "completed" }) .where(eq(chatConversations.id, conversation.id)); const secondResponse = await addSelectedChatFinal({ agentId: fixture.assignedAgentId, body: "Second queued answer", companyId: fixture.companyId, issueId: conversation.issueId, runId: secondRunId, }); const lateInternalComment = await issueService(db).addComment( conversation.issueId, "Later internal-only note", { agentId: fixture.assignedAgentId }, { authorType: "agent" }, ); const responsePublications = await db .select() .from(chatPublications) .where( inArray(chatPublications.commentId, [ firstResponse.id, secondResponse.id, activeInternalComment.id, lateInternalComment.id, ]), ); expect(responsePublications).toEqual( expect.arrayContaining([ expect.objectContaining({ commentId: firstResponse.id, conversationId: conversation.id, state: "published", }), expect.objectContaining({ commentId: secondResponse.id, conversationId: conversation.id, state: "pending", }), ]), ); for (const internalCommentId of [ activeInternalComment.id, lateInternalComment.id, ]) { expect( responsePublications.find( (publication) => publication.commentId === internalCommentId, ), ).toBeUndefined(); } expect( await db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `attachment:${internalAttachment.id}:${endpoint.id}`, ), ), ).toHaveLength(0); await service.processPendingPublications(1_000); expect( runtime.endpoints.get(endpoint.id)?.posts.map((post) => post.text), ).toEqual(["First queued answer", "Second queued answer"]); const chatAttachment = await issueService(db).createAttachment({ issueId: conversation.issueId, issueCommentId: secondResponse.id, provider: "local_disk", objectKey: "issues/chat-result.txt", contentType: "text/plain", byteSize: 11, sha256: "b".repeat(64), originalFilename: "chat-result.txt", createdByAgentId: fixture.assignedAgentId, createdByRunId: secondRunId, }); await expect( db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `attachment:${chatAttachment.id}:${endpoint.id}`, ), ), ).resolves.toEqual([ expect.objectContaining({ commentId: secondResponse.id, conversationId: conversation.id, state: "pending", }), ]); // This projection-only fixture intentionally has no object-store bytes. // Retire its staged work so another test's global worker cannot retry it. await db .update(chatPublications) .set({ state: "cancelled", nextAttemptAt: null }) .where( and( eq(chatPublications.endpointId, endpoint.id), eq( chatPublications.idempotencyKey, `attachment:${chatAttachment.id}:${endpoint.id}`, ), ), ); await service.shutdown(); }); it("holds ambiguous provider sends for an audited duplicate-risk resolution without reordering", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const channel = makeThread({ channelId: "C-UNKNOWN", id: "slack:C-UNKNOWN:4500.1", name: "unknown-delivery", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "4500.1", text: "@maya test an ambiguous response", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const firstComment = await issueService(db).addComment( conversation.issueId, "First safe response", { userId: "owner-user" }, { authorType: "user" }, ); const secondComment = await issueService(db).addComment( conversation.issueId, "Second safe response", { userId: "owner-user" }, { authorType: "user" }, ); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected provider runtime"); providerRuntime.postError = new Error("socket reset after write"); await service.publishComment(endpoint.id, conversation.id, firstComment.id); const firstPublication = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, firstComment.id)) .then((rows) => rows[0]); expect(firstPublication).toMatchObject({ state: "delivery_unknown", attempts: 1, providerMessageId: null, }); await service.publishComment( endpoint.id, conversation.id, secondComment.id, ); const secondPublication = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, secondComment.id)) .then((rows) => rows[0]); expect(secondPublication.state).toBe("pending"); const activity = await service.listActivity(endpoint.id); expect( activity.find((item) => item.id === firstPublication.id), ).toMatchObject({ kind: "publication", status: "delivery_unknown", replayable: false, resolutionActions: ["mark_delivered", "retry_anyway", "cancel"], }); await expect( service.replayPublication(endpoint.id, firstPublication.id), ).rejects.toMatchObject({ status: 409, details: { code: "chat_publication_resolution_required" }, }); providerRuntime.postError = null; await service.resolvePublication( endpoint.id, firstPublication.id, "retry_anyway", "owner-user", ); await service.processPendingPublications(); const replayed = await db .select() .from(chatPublications) .where(eq(chatPublications.id, firstPublication.id)) .then((rows) => rows[0]); const releasedSecond = await db .select() .from(chatPublications) .where(eq(chatPublications.id, secondPublication.id)) .then((rows) => rows[0]); expect(replayed.state).toBe("published"); expect(releasedSecond.state).toBe("published"); expect(providerRuntime.posts.map((post) => post.text)).toEqual([ "First safe response", "Second safe response", ]); const [resolutionActivity] = await db .select() .from(activityLog) .where(eq(activityLog.entityId, firstPublication.id)); expect(resolutionActivity).toMatchObject({ actorType: "user", actorId: "owner-user", action: "chat.publication_retry_anyway", }); }); it("does not wake twice when an operator replays an already admitted failed delivery", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service, wakeup } = await configuredSlackEndpoint(fixture); const channel = makeThread({ channelId: "C-REPLAY", id: "slack:C-REPLAY:5000.1", name: "replay", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "5000.1", text: "@maya retry this", mentioned: true, }), trigger: "mention", }); const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); await db .update(chatDeliveries) .set({ state: "failed", redactedError: "temporary failure" }) .where(eq(chatDeliveries.id, delivery.id)); await service.replayDelivery(endpoint.id, delivery.id); expect(wakeup).toHaveBeenCalledTimes(1); expect(wakeup.mock.calls[0]?.[1]).toMatchObject({ reason: "External chat message received", payload: { mutation: "chat_message_received", wakeCommentId: expect.any(String), }, }); const [replayed] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, delivery.id)); expect(replayed).toMatchObject({ state: "processed", attempts: 2, redactedError: null, }); }); describe("Discord native commands with durable service authority", () => { const guildId = "1457808928258658549"; const channelId = "333333333333333333"; const externalUserId = "444444444444444419"; const rootMessageId = "555555555555555619"; const registeredCommandId = "888888888888888819"; let interactionSequence = 0n; async function commandFixture() { const fixture = await seedCompany(); const applicationId = uniqueDiscordApplicationId(); let holdWork = false; const queued: Array<() => void> = []; const remoteCommands: Array> = []; const registrationCalls: string[] = []; const providerFetch: typeof globalThis.fetch = async (input, init) => { if ( [ `/api/v10/applications/${applicationId}/commands`, `/api/v10/applications/${applicationId}/commands/${registeredCommandId}`, ].includes(new URL(String(input)).pathname) ) { const method = init?.method ?? "GET"; registrationCalls.push(method); if (method === "GET") return Response.json(remoteCommands); if (method !== "POST" && method !== "PATCH") throw new Error("Unexpected fixture command request"); const definition = JSON.parse(String(init?.body)); const command = { ...definition, id: registeredCommandId, application_id: applicationId, version: "999999999999999919", }; if (method === "PATCH") remoteCommands.splice(0, remoteCommands.length, command); else remoteCommands.push(command); return Response.json(command); } return fakeDiscordFetch(applicationId)(input); }; const context = createService(new FakeChatSdkRuntime(), providerFetch, { scheduleDeferredWork(task) { if (holdWork) queued.push(task); else setImmediate(task); }, }); const endpoint = await context.service.create( fixture.companyId, { provider: "discord", assignedAgentId: fixture.assignedAgentId, name: "Discord command fixture", }, "owner-user", ); const identityAdapter = createDiscordAdapter({ applicationId, botToken: "synthetic-identity-only", webhookVerifier: async () => false, }); context.runtime.initializeHook = async (endpointId) => { if (endpointId !== endpoint.id) return; const current = context.runtime.get(endpointId)!; const original = current.thread.bind(current); // Real pinned source IDs, with data transport only still mocked. current.thread = (threadId) => ({ ...original(threadId), channelId: identityAdapter.channelIdFromThreadId(threadId), isDM: threadId.startsWith("discord:@me:"), }); }; await context.service.configure( endpoint.id, { action: "configure", credentials: { applicationId, botToken: "discord-secret", guildId }, }, "owner-user", ); const initialCallbacks = context.runtime.configurations.get( endpoint.id, )!.callbacks; if (!initialCallbacks.onDiscordRootMentionAdmission) throw new Error("Discord root fixture callback unavailable"); const threadId = `discord:${guildId}:${channelId}:${rootMessageId}`; await initialCallbacks.onDiscordRootMentionAdmission({ endpointId: endpoint.id, guildId, channelId, messageId: rootMessageId, message: { ...makeMessage({ id: rootMessageId, text: "@maya investigate the command fixture", mentioned: true, userId: externalUserId, }), threadId, } as Message, threadId, userId: externalUserId, }); const delivery = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.providerEventId, `${threadId}:${rootMessageId}`), ), ) .then((rows) => rows[0]); if (!delivery) throw new Error("Discord command setup delivery absent"); await context.service.processPendingDeliveries(25, delivery.id); await qualifySetupRoundTrip(context.service, endpoint.id, externalUserId); await context.service.test(endpoint.id, "owner-user"); await context.service.reconcileProviderRuntimes(); const callbacks = context.runtime.configurations.get( endpoint.id, )!.callbacks; if (!callbacks.onSlashCommand) throw new Error("Registered Discord command callback unavailable"); const [conversation] = await context.service.listConversations( endpoint.id, ); if (!conversation) throw new Error("Discord command conversation absent"); const principal = await db .select() .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, fixture.companyId), eq(chatExternalPrincipals.provider, "discord"), eq(chatExternalPrincipals.providerAccountId, guildId), eq(chatExternalPrincipals.externalId, externalUserId), ), ) .then((rows) => rows[0]); if (!principal) throw new Error("Discord command principal absent"); const intent = await context.service.createLinkIntent( endpoint.id, principal.id, 1800, ); await context.service.confirmIdentityLink( new URL(intent.confirmationUrl).searchParams.get("token")!, "owner-user", ); const scope = { companyId: fixture.companyId, endpointId: endpoint.id, applicationId, guildId, }; expect( registrationCalls.filter((method) => method === "POST"), ).toHaveLength(1); holdWork = true; const pinnedRuntimes: Array< ReturnType > = []; const coldContexts: Array> = []; async function parser( configuration = context.runtime.configurations.get(endpoint.id)!, ) { const pinned = createChatSdkEndpointRuntime({ ...configuration, callbacks: { onMessage() {}, onSlashCommand: configuration.callbacks.onSlashCommand, }, enableDiscordGateway: false, logger: "silent", }); pinnedRuntimes.push(pinned); await pinned.initialize(); return pinned.getProviderAdapter() as unknown as { handleGatewayInteraction(input: unknown): Promise; }; } const adapter = await parser(); const interaction = ( command: "status" | "new" | "close", overrides: Record = {}, ) => ({ id: ( ((BigInt(Date.now()) - 1420070400000n) << 22n) + interactionSequence++ ).toString(), applicationId, commandId: registeredCommandId, commandName: "paperclip", commandType: 1, type: 2, version: 1, context: 0, guildId, channelId: rootMessageId, channel: { id: rootMessageId, parentId: channelId, type: 11 }, authorizingIntegrationOwners: { guildId, userId: null }, user: { id: externalUserId, username: "operator", globalName: "Operator", bot: false, discriminator: "0", }, options: { data: [{ name: command, type: 1 }] }, createdTimestamp: Date.now(), token: "synthetic-command-interaction-token", isChatInputCommand: () => true, isModalSubmit: () => false, isMessageComponent: () => false, deferReply: vi.fn(async () => undefined), editReply: vi.fn(async () => undefined), deferred: false, replied: false, ...overrides, }); const actions = () => db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "discord_native_command"), ), ) .orderBy(asc(chatActions.createdAt)); const publications = () => db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, endpoint.id), like(chatPublications.idempotencyKey, "control:%"), ), ); return { ...context, fixture, endpoint, callbacks, conversation, applicationId, principal, scope, adapter, parser, interaction, actions, publications, queued, registrationCalls, async coldUnavailableParser(legacyCopy = false) { await context.service.shutdown(); await db .update(chatEndpoints) .set({ capabilities: { ...endpoint.capabilities, slashCommands: false }, }) .where(eq(chatEndpoints.id, endpoint.id)); const registration = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "discord_command_registration"), ), ) .then((rows) => rows[0]!); let priorRegistration = registration.payload.registration; if (legacyCopy) { const prior = structuredClone(remoteCommands[0]!); (prior.options as Array<{ description: string }>)[2]!.description = "Close the current Paperclip task"; remoteCommands[0] = prior; const { id: _id, application_id: _application, version: _version, ...definition } = prior; const stored = priorRegistration as { receipt: Record; }; priorRegistration = { ...stored, receipt: { ...stored.receipt, definitionDigest: createHash("sha256") .update(JSON.stringify(definition)) .digest("hex"), }, }; } await db .update(chatActions) .set({ payload: { registration: priorRegistration }, result: { schema: "paperclip.discord.command-registration-result.v1", outcome: legacyCopy ? "registered" : "unavailable", retryAt: legacyCopy ? new Date(0).toISOString() : new Date(Date.now() + 60_000).toISOString(), }, }) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "discord_command_registration"), ), ); const cold = createService(new FakeChatSdkRuntime(), providerFetch, { scheduleDeferredWork(task) { queued.push(task); }, }); coldContexts.push(cold); await cold.service.reconcileProviderRuntimes(); return { ...cold, adapter: await parser( cold.runtime.configurations.get(endpoint.id)!, ), }; }, async close() { try { await Promise.all( pinnedRuntimes.map((pinned) => pinned.shutdown()), ); } finally { try { await Promise.all( coldContexts.map((cold) => cold.service.shutdown()), ); } finally { await retirePublicationFixture(context.service, endpoint.id); } } }, }; } it.each(["working", "final", "unknown_final", "card"] as const)( "qualifies pinned Discord close-owned progress retirement (%s)", async (mode) => { const f = await commandFixture(); try { // The setup root can still have a retryable wake after a concurrent // fixture drain. Start this run from a fresh, actually accepted // request and prove its receipt exists before testing its removal. const sourceMessageId = ( ((BigInt(Date.now()) - 1420070400000n) << 22n) + 1n ).toString(); const sourceThread = makeThread({ id: f.conversation.externalThreadId, channelId: f.conversation.externalConversationId, name: "Close-owned progress qualification", }); await deliverMessage({ callbacks: f.callbacks, endpointId: f.endpoint.id, provider: "discord", thread: sourceThread.thread, message: makeMessage({ id: sourceMessageId, text: "Work on this request while I test closing the conversation", userId: externalUserId, }), trigger: "subscribed_message", }); const [sourceDelivery] = await db .select({ id: chatDeliveries.id, state: chatDeliveries.state }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, f.endpoint.id), sql`${chatDeliveries.normalizedEvent}->'message'->>'providerMessageId' = ${sourceMessageId}`, ), ); expect(sourceDelivery?.state).toBe("processed"); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where( and( eq(chatActions.endpointId, f.endpoint.id), eq(chatActions.deliveryId, sourceDelivery!.id), eq(chatActions.kind, "inbound_wakeup"), ), ), ).resolves.toEqual([{ status: "processed" }]); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where( and( eq(chatActions.endpointId, f.endpoint.id), eq( chatActions.providerActionId, `receipt_reaction:${sourceDelivery!.id}`, ), ), ), ).resolves.toEqual([{ status: "processed" }]); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: f.fixture.companyId, agentId: f.fixture.assignedAgentId, status: "running", contextSnapshot: await chatWakeContext({ endpointId: f.endpoint.id, issueId: f.conversation.issueId, provider: "discord", providerMessageId: sourceMessageId, }), }); const [progress] = await db .insert(chatPublications) .values({ companyId: f.fixture.companyId, endpointId: f.endpoint.id, conversationId: f.conversation.id, issueId: f.conversation.issueId, idempotencyKey: `run:${runId}:working:${f.endpoint.id}`, payload: { text: "Maya is working…", progressState: "working" }, state: "pending", }) .returning(); await f.service.processPendingPublications(); const runtime = f.runtime.endpoints.get(f.endpoint.id)!; const progressMessageId = await db .select({ id: chatPublications.providerMessageId }) .from(chatPublications) .where(eq(chatPublications.id, progress.id)) .then((rows) => rows[0]!.id!); if (mode === "final") { await addSelectedChatFinal({ agentId: f.fixture.assignedAgentId, body: "The Discord final must remain", companyId: f.fixture.companyId, issueId: f.conversation.issueId, runId, }); await f.service.processPendingPublications(); } if (mode === "card") await db .update(chatPublications) .set({ payload: { ...progress.payload, card: { title: "Not a plain progress lane" }, }, }) .where(eq(chatPublications.id, progress.id)); const command = f.interaction("close"); await f.adapter.handleGatewayInteraction(command); const close = (await f.publications()).find((publication) => publication.idempotencyKey.startsWith("control:close:"), )!; expect(close).toBeDefined(); if (mode === "unknown_final") { const final = await addSelectedChatFinal({ agentId: f.fixture.assignedAgentId, body: "Possibly delivered Discord final", companyId: f.fixture.companyId, issueId: f.conversation.issueId, runId, }); await db .update(chatPublications) .set({ state: "delivery_unknown", createdAt: new Date(close.createdAt.getTime() + 1000), }) .where(eq(chatPublications.commentId, final.id)); } const editCount = runtime.edits.length, postCount = runtime.posts.length; await f.service.processPendingPublications(); await f.service.processPendingReceiptReactions(); expect(runtime.edits).toHaveLength( editCount + (mode === "working" ? 1 : 0), ); expect(runtime.posts).toHaveLength( postCount + (mode === "working" ? 0 : 1), ); if (mode === "working") { expect(runtime.edits.at(-1)).toMatchObject({ messageId: progressMessageId, text: close.payload.text, }); await waitForProcessedReceiptRemoval(f.endpoint.id, { threadId: f.conversation.externalThreadId, messageId: sourceMessageId, emoji: "eyes", }); expect(runtime.removedReactions).toContainEqual({ threadId: f.conversation.externalThreadId, messageId: sourceMessageId, emoji: "eyes", }); } await expect( db .select({ status: heartbeatRuns.status }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, runId)), ).resolves.toEqual([{ status: "running" }]); await f.adapter.handleGatewayInteraction(command); await f.service.processPendingPublications(); expect(runtime.edits).toHaveLength( editCount + (mode === "working" ? 1 : 0), ); } finally { await f.close(); } }, ); it("answers a cold unavailable Discord command privately without leaving the interaction thinking", async () => { const f = await commandFixture(); try { const cold = await f.coldUnavailableParser(); const beforeActions = await f.actions(); const beforePublications = await f.publications(); const command = f.interaction("close"); await cold.adapter.handleGatewayInteraction(command); expect .soft(command.deferReply) .toHaveBeenCalledExactlyOnceWith({ flags: 64 }); expect(command.editReply).toHaveBeenCalledExactlyOnceWith({ content: expect.stringContaining( "This command is not available here", ), allowedMentions: { parse: [] }, }); expect(await f.actions()).toEqual(beforeActions); expect(await f.publications()).toEqual(beforePublications); expect(cold.wakeup).not.toHaveBeenCalled(); expect( ( await db .select() .from(chatConversations) .where(eq(chatConversations.id, f.conversation.id)) )[0]?.state, ).toBe("active"); } finally { await f.close(); } }); it("upgrades a retained prior-copy registration after service restart before admitting close", async () => { const f = await commandFixture(); try { const cold = await f.coldUnavailableParser(true); expect( f.registrationCalls.filter((method) => method === "POST"), ).toHaveLength(1); expect( f.registrationCalls.filter((method) => method === "PATCH"), ).toHaveLength(1); const command = f.interaction("close"); await cold.adapter.handleGatewayInteraction(command); expect(command.editReply).toHaveBeenCalledExactlyOnceWith({ content: expect.stringContaining("request recorded"), allowedMentions: { parse: [] }, }); expect(await f.actions()).toHaveLength(1); expect(await f.publications()).toHaveLength(1); expect(cold.wakeup).not.toHaveBeenCalled(); await cold.service.processPendingPublications(100); expect((await f.publications())[0]?.state).toBe("published"); expect( ( await db .select() .from(chatConversations) .where(eq(chatConversations.id, f.conversation.id)) )[0]?.state, ).toBe("completed"); } finally { await f.close(); } }); it("returns private status, keeps guild new as guidance, and closes only after a real public control receipt", async () => { const f = await commandFixture(); try { const wakeCount = f.wakeup.mock.calls.length; const status = f.interaction("status"); await f.adapter.handleGatewayInteraction(status); expect(status.deferReply).toHaveBeenCalledWith({ flags: 64 }); expect(status.editReply).toHaveBeenCalledWith( expect.objectContaining({ content: expect.stringContaining(f.conversation.issueIdentifier!), allowedMentions: { parse: [] }, }), ); expect(await f.publications()).toHaveLength(0); const newCommand = f.interaction("new"); await f.adapter.handleGatewayInteraction(newCommand); expect(newCommand.editReply).toHaveBeenCalledWith( expect.objectContaining({ content: expect.stringContaining("new Discord thread"), }), ); expect(await f.publications()).toHaveLength(0); const close = f.interaction("close"); await f.adapter.handleGatewayInteraction(close); expect(close.editReply).toHaveBeenCalledWith( expect.objectContaining({ content: expect.stringContaining("recorded"), }), ); const [publication] = await f.publications(); expect(publication).toMatchObject({ state: "pending", conversationId: f.conversation.id, issueId: f.conversation.issueId, }); expect(publication.idempotencyKey).toMatch(/^control:close:/); expect( ( await db .select() .from(chatConversations) .where(eq(chatConversations.id, f.conversation.id)) )[0]?.state, ).toBe("active"); expect(await f.actions()).toHaveLength(3); expect(f.wakeup).toHaveBeenCalledTimes(wakeCount); await f.service.processPendingPublications(100); expect((await f.publications())[0]).toMatchObject({ state: "published", }); expect( ( await db .select() .from(chatConversations) .where(eq(chatConversations.id, f.conversation.id)) )[0]?.state, ).toBe("completed"); // Fresh adapter removes only process-local ACK suppression. The real // service must replay the exact durable action, not stage another close // or infer a replacement conversation from its now-completed target. const replayAdapter = await f.parser(); const replay = f.interaction("close", { id: close.id }); await replayAdapter.handleGatewayInteraction(replay); expect(replay.editReply).toHaveBeenCalledWith( expect.objectContaining({ content: expect.stringContaining("recorded"), }), ); expect(await f.actions()).toHaveLength(3); expect(await f.publications()).toHaveLength(1); expect(await f.service.listConversations(f.endpoint.id)).toHaveLength( 1, ); expect(JSON.stringify(await f.actions())).not.toContain( "synthetic-command-interaction-token", ); } finally { await f.close(); } }); it.each([ "missing owner", "unconfirmed registration", "disabled capability", "altered origin fence", "viewer", "disabled reach", "disabled connection", "stale generation", "replaced Gateway lease", ] as const)( "denies a previously accepted command after %s without another action or public receipt", async (reason) => { const f = await commandFixture(); try { const first = f.interaction("status"); await f.adapter.handleGatewayInteraction(first); expect(first.editReply).toHaveBeenCalledWith( expect.objectContaining({ content: expect.stringContaining(f.conversation.issueIdentifier!), }), ); const before = await f.actions(); expect(before).toHaveLength(1); const current = await db .select() .from(chatEndpoints) .where(eq(chatEndpoints.id, f.endpoint.id)) .then((rows) => rows[0]!); if (reason === "missing owner") await db .delete(chatDiscordCommandOwners) .where( eq(chatDiscordCommandOwners.applicationId, f.applicationId), ); if (reason === "unconfirmed registration") await db .update(chatActions) .set({ result: { outcome: "unknown" } }) .where( and( eq(chatActions.endpointId, f.endpoint.id), eq(chatActions.kind, "discord_command_registration"), ), ); if (reason === "disabled capability") await db .update(chatEndpoints) .set({ capabilities: { ...current.capabilities, slashCommands: false }, }) .where(eq(chatEndpoints.id, current.id)); if (reason === "altered origin fence") await db .update(chatActions) .set({ payload: { ...before[0]!.payload, runtimeFence: { ...(before[0]!.payload.runtimeFence as object), generation: 999999, }, }, }) .where(eq(chatActions.id, before[0]!.id)); if (reason === "viewer") await db .update(companyMemberships) .set({ membershipRole: "viewer" }) .where( and( eq(companyMemberships.companyId, f.fixture.companyId), eq(companyMemberships.principalId, "owner-user"), ), ); if (reason === "disabled reach") await db .update(chatEndpointResources) .set({ enabled: false }) .where( and( eq(chatEndpointResources.endpointId, f.endpoint.id), eq(chatEndpointResources.providerResourceId, channelId), ), ); if (reason === "disabled connection") await db .update(toolConnections) .set({ enabled: false }) .where(eq(toolConnections.id, current.connectionId)); if (reason === "stale generation") await db .update(chatEndpoints) .set({ setup: { ...current.setup, runtimeGeneration: Number(current.setup.runtimeGeneration ?? 0) + 1, }, }) .where(eq(chatEndpoints.id, current.id)); if (reason === "replaced Gateway lease") await db .update(chatEndpointLeases) .set({ token: randomUUID() }) .where( and( eq(chatEndpointLeases.endpointId, f.endpoint.id), eq(chatEndpointLeases.leaseKey, "discord_gateway_runtime"), ), ); const afterRevocation = await f.actions(); const replay = f.interaction("status", { id: first.id }); await (await f.parser()).handleGatewayInteraction(replay); expect(replay.deferReply).toHaveBeenCalledExactlyOnceWith({ flags: 64, }); expect(replay.editReply).toHaveBeenCalledExactlyOnceWith( expect.objectContaining({ content: expect.stringContaining("not available"), }), ); expect(await f.actions()).toEqual(afterRevocation); expect(await f.publications()).toHaveLength(0); } finally { await f.close(); } }, ); it("serializes simultaneous exact close deliveries and denies a changed command or registered ID", async () => { const f = await commandFixture(); try { const first = f.interaction("close"); const duplicate = f.interaction("close", { id: first.id }); const other = await f.parser(); await Promise.all([ f.adapter.handleGatewayInteraction(first), other.handleGatewayInteraction(duplicate), ]); for (const command of [first, duplicate]) expect(command.editReply).toHaveBeenCalledWith( expect.objectContaining({ content: expect.stringContaining("recorded"), }), ); const before = await f.actions(); expect(before).toHaveLength(1); expect(await f.publications()).toHaveLength(1); const changed = f.interaction("new", { id: first.id }); await (await f.parser()).handleGatewayInteraction(changed); expect(changed.editReply).toHaveBeenCalledWith( expect.objectContaining({ content: expect.stringContaining("not available"), }), ); const changedActor = f.interaction("close", { id: first.id, user: { id: "444444444444444420", username: "other", bot: false, discriminator: "0", }, }); await (await f.parser()).handleGatewayInteraction(changedActor); expect(changedActor.editReply).toHaveBeenCalledWith( expect.objectContaining({ content: expect.stringContaining("not available"), }), ); const unregistered = f.interaction("close", { commandId: "888888888888888820", }); await (await f.parser()).handleGatewayInteraction(unregistered); expect(unregistered.editReply).toHaveBeenCalledWith( expect.objectContaining({ content: expect.stringContaining("not available"), }), ); expect(await f.actions()).toEqual(before); expect(await f.publications()).toHaveLength(1); } finally { await f.close(); } }); it("records DM new once and never applies its old replay to a replacement conversation", async () => { const f = await commandFixture(); try { const dmId = "666666666666666619"; const dmThreadId = `discord:@me:${dmId}`; const thread = f.runtime .get(f.endpoint.id)! .thread(dmThreadId) as unknown as Thread; const message = (id: string, text: string) => ({ ...makeMessage({ id, text, userId: externalUserId }), threadId: dmThreadId, }) as Message; const receive = async (id: string, text: string) => { await f.callbacks.onMessage({ endpointId: f.endpoint.id, provider: "discord", trigger: "direct_message", thread, message: message(id, text), }); const delivery = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, f.endpoint.id), eq(chatDeliveries.providerEventId, `${dmThreadId}:${id}`), ), ) .then((rows) => rows[0]); if (!delivery) throw new Error("DM fixture delivery absent"); await f.service.processPendingDeliveries(25, delivery.id); expect( await db .select({ state: chatDeliveries.state, error: chatDeliveries.redactedError, }) .from(chatDeliveries) .where(eq(chatDeliveries.id, delivery.id)), ).toEqual([expect.objectContaining({ state: "processed" })]); }; await receive("555555555555555620", "Investigate the original DM task"); const old = await db .select() .from(chatConversations) .where( and( eq(chatConversations.endpointId, f.endpoint.id), eq(chatConversations.externalThreadId, dmThreadId), ), ) .then((rows) => rows[0]!); expect(old).toMatchObject({ state: "active", isDirectMessage: true, sessionGeneration: 1, }); const dm = { context: 1, guildId: null, channelId: dmId, channel: { id: dmId, type: 1 }, authorizingIntegrationOwners: { guildId: "0", userId: null }, }; const next = f.interaction("new", dm); await f.adapter.handleGatewayInteraction(next); expect(next.editReply).toHaveBeenCalledWith( expect.objectContaining({ content: expect.stringContaining("recorded"), }), ); expect((await f.publications())[0]).toMatchObject({ state: "pending", conversationId: old.id, }); await f.service.processPendingPublications(100); expect((await f.publications())[0]).toMatchObject({ state: "published", }); await receive( (((BigInt(Date.now()) - 1420070400000n) << 22n) + 1000n).toString(), "Start the replacement DM task", ); const conversations = await db .select() .from(chatConversations) .where( and( eq(chatConversations.endpointId, f.endpoint.id), eq(chatConversations.externalThreadId, dmThreadId), ), ) .orderBy(asc(chatConversations.sessionGeneration)); expect(conversations).toHaveLength(2); expect(conversations[0]).toMatchObject({ id: old.id, state: "completed", sessionGeneration: 1, }); expect(conversations[1]).toMatchObject({ state: "active", sessionGeneration: 2, }); expect(conversations[1]!.issueId).not.toBe(old.issueId); const wakeCount = f.wakeup.mock.calls.length; const replay = f.interaction("new", { ...dm, id: next.id }); await (await f.parser()).handleGatewayInteraction(replay); expect(replay.editReply).toHaveBeenCalledWith( expect.objectContaining({ content: expect.stringContaining("recorded"), }), ); expect(await f.actions()).toHaveLength(1); expect(await f.publications()).toHaveLength(1); expect( ( await db .select() .from(chatConversations) .where(eq(chatConversations.id, conversations[1]!.id)) )[0]?.state, ).toBe("active"); expect(f.wakeup).toHaveBeenCalledTimes(wakeCount); for (const [offset, command] of [ [1n, "new"], [2n, "close"], ] as const) { const stale = f.interaction(command, { ...dm, id: (BigInt(next.id) - offset).toString(), }); await f.adapter.handleGatewayInteraction(stale); expect(stale.editReply).toHaveBeenCalledWith( expect.objectContaining({ content: expect.stringContaining("predates"), }), ); expect(await f.publications()).toHaveLength(1); expect( ( await db .select() .from(chatConversations) .where(eq(chatConversations.id, conversations[1]!.id)) )[0]?.state, ).toBe("active"); expect(f.wakeup).toHaveBeenCalledTimes(wakeCount); } } finally { await f.close(); } }); it.each(["retired runtime", "replaced lease"] as const)( "acknowledges privately before a blocked DB wait but denies %s before commit", async (loss) => { const f = await commandFixture(); let release = () => {}; let blocker: Promise | undefined; let delivery: Promise | undefined; try { const current = await db .select() .from(chatEndpoints) .where(eq(chatEndpoints.id, f.endpoint.id)) .then((rows) => rows[0]!); let locked = () => {}; const acquired = new Promise((resolve) => { locked = resolve; }); const held = new Promise((resolve) => { release = resolve; }); blocker = db.transaction(async (tx) => { await tx .select() .from(toolConnections) .where(eq(toolConnections.id, current.connectionId)) .for("no key update"); locked(); await held; }); await acquired; const command = f.interaction("close"); delivery = f.adapter.handleGatewayInteraction(command); await expect.poll(() => command.deferReply.mock.calls.length).toBe(1); await expect .poll(async () => { try { await db.transaction(async (tx) => { await tx .select() .from(chatEndpoints) .where(eq(chatEndpoints.id, f.endpoint.id)) .for("no key update", { noWait: true }); }); return false; } catch (error) { const code = (error as { code?: unknown; cause?: { code?: unknown } }) .cause?.code ?? (error as { code?: unknown }).code; if (code !== "55P03") throw error; return true; } }) .toBe(true); expect(command.editReply).not.toHaveBeenCalled(); if (loss === "retired runtime") await f.runtime.removeEndpoint(f.endpoint.id); else await db .update(chatEndpointLeases) .set({ token: randomUUID() }) .where( and( eq(chatEndpointLeases.endpointId, f.endpoint.id), eq(chatEndpointLeases.leaseKey, "discord_gateway_runtime"), ), ); release(); await blocker; await delivery; expect(command.editReply).toHaveBeenCalledExactlyOnceWith( expect.objectContaining({ content: expect.stringContaining("not available"), }), ); expect(await f.actions()).toHaveLength(0); expect(await f.publications()).toHaveLength(0); expect( ( await db .select() .from(chatConversations) .where(eq(chatConversations.id, f.conversation.id)) )[0]?.state, ).toBe("active"); } finally { release(); try { await blocker; await delivery; } finally { await f.close(); } } }, ); describe("fresh-after-close publication bindings", () => { let lastFreshDiscordMessageId = 0n; const freshDiscordMessageId = () => { const current = ((BigInt(Date.now()) - 1420070400000n) << 22n) + 1000n; lastFreshDiscordMessageId = current > lastFreshDiscordMessageId ? current : lastFreshDiscordMessageId + 1n; return lastFreshDiscordMessageId.toString(); }; async function reopenedFixture(includeDuringConfirmationSource = false) { const f = await commandFixture(); const threadId = `discord:${guildId}:${channelId}:${rootMessageId}`; const admitMessage = async (message: ReturnType) => { await deliverMessage({ callbacks: f.callbacks, endpointId: f.endpoint.id, provider: "discord", thread: makeThread({ id: threadId, channelId: f.conversation.externalConversationId, name: "reopened", }).thread, message, trigger: "mention", }); const [delivery] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, f.endpoint.id), sql`${chatDeliveries.normalizedEvent}->'message'->>'providerMessageId' = ${message.id}`, ), ); if (!delivery) throw new Error("Expected exact source delivery"); await f.service.processPendingDeliveries(25, delivery.id); return delivery; }; const bindRun = async (messageId: string, admitted = true) => { const context = await chatWakeContext({ endpointId: f.endpoint.id, issueId: f.conversation.issueId, provider: "discord", providerMessageId: messageId, }); const [action] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, f.endpoint.id), eq(chatActions.kind, "inbound_wakeup"), sql`${chatActions.payload}->>'commentId' = ${context.wakeCommentId}`, ), ); if (!action) throw new Error("Expected admitted source action"); const runId = randomUUID(); // The fixture heartbeat does not execute a model. Persist the exact // scheduler linkage for the already admitted source, then exercise // real binding/milestone/final publication code, not an auth mock. await db.insert(heartbeatRuns).values({ id: runId, companyId: f.fixture.companyId, agentId: f.fixture.assignedAgentId, runtimeMode: "native", nativeIssueId: f.conversation.issueId, status: "running", wakeupRequestId: admitted ? action.id : null, startedAt: new Date(), contextSnapshot: { ...context, endpointId: f.endpoint.id, paperclipHarnessCheckedOut: true, }, }); if (admitted) { const receipt = await db .update(agentWakeupRequests) .set({ runId, status: "claimed" }) .where(eq(agentWakeupRequests.id, action.id)) .returning({ id: agentWakeupRequests.id }); expect(receipt).toHaveLength(1); } return { runId, action, context }; }; const historical = await bindRun(rootMessageId, false); const oldMessage = makeMessage({ id: "555555555555555620", text: "@maya old request before close", mentioned: true, userId: externalUserId, }); await admitMessage(oldMessage); const old = await bindRun(oldMessage.id); await db .update(heartbeatRuns) .set({ status: "succeeded", finishedAt: new Date() }) .where(inArray(heartbeatRuns.id, [old.runId, historical.runId])); const controlInvocation = f.interaction("close"); await f.adapter.handleGatewayInteraction(controlInvocation); let duringConfirmation: { runId: string; receivedAt: Date } | null = null; if (includeDuringConfirmationSource) { const messageId = freshDiscordMessageId(); const message = makeMessage({ id: messageId, text: "@maya this source was already admitted before close committed", mentioned: true, userId: externalUserId, }); message.metadata.dateSent = new Date( Number((BigInt(messageId) >> 22n) + 1420070400000n), ); const delivery = await admitMessage(message); const admitted = await bindRun(messageId); duringConfirmation = { runId: admitted.runId, receivedAt: delivery.receivedAt, }; } await f.service.processPendingPublications(); const [control] = await f.publications(); expect(control).toMatchObject({ state: "published", conversationId: f.conversation.id, }); expect((await f.service.listConversations(f.endpoint.id))[0]?.state).toBe( "completed", ); const messageId = freshDiscordMessageId(); const freshMessage = makeMessage({ id: messageId, text: "@maya reply to this fresh request", mentioned: true, userId: externalUserId, }); const delivery = await admitMessage(freshMessage); const fresh = await bindRun(messageId); expect((await f.service.listConversations(f.endpoint.id))[0]).toMatchObject( { id: f.conversation.id, state: "active", sessionGeneration: f.conversation.sessionGeneration, }, ); return { ...f, old, historical, fresh, delivery, control, controlInvocation, duringConfirmation, admitMessage, bindRun, }; } it("publishes working and the exact fresh final after pinned close without resurrecting the old source", async () => { const f = await reopenedFixture(); try { const scope = { companyId: f.fixture.companyId, issueId: f.conversation.issueId, }; await expect( resolveChatRunPresentationAuthorizationReason(db, { ...scope, runId: f.old.runId, }), ).resolves.toBe("internal_agent_write"); await expect( resolveChatRunPresentationAuthorizationReason(db, { ...scope, runId: f.historical.runId, }), ).resolves.toBe("internal_agent_write"); await expect( resolveChatRunPresentationAuthorizationReason(db, { ...scope, runId: f.fresh.runId, }), ).resolves.toBe("allow_chat_run_presentation"); await enqueueChatRunMilestones(db); await f.service.processPendingPublications(); expect( f.runtime.get(f.endpoint.id)?.posts.map((post) => post.text), ).toContain("Maya is working…"); await db .update(heartbeatRuns) .set({ status: "succeeded", finishedAt: new Date() }) .where(eq(heartbeatRuns.id, f.fresh.runId)); const comment = await addSelectedChatFinal({ ...scope, agentId: f.fixture.assignedAgentId, runId: f.fresh.runId, body: "DISCORD-FRESH-AFTER-CLOSE", }); await f.service.processPendingPublications(); const publications = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, comment.id)); expect(publications).toEqual([ expect.objectContaining({ conversationId: f.conversation.id, state: "published", attempts: 1, payload: { text: "DISCORD-FRESH-AFTER-CLOSE" }, }), ]); expect((await f.publications())[0]).toEqual(f.control); } finally { await f.close(); } }); it("allows an admitted source sent after close command but before close confirmation to present", async () => { const f = await reopenedFixture(); try { const commandTime = Number( (BigInt(f.controlInvocation.id) >> 22n) + 1420070400000n, ); const sourceTime = commandTime + 1; expect(sourceTime).toBeLessThan(f.control.publishedAt!.getTime()); const message = makeMessage({ id: (((BigInt(sourceTime) - 1420070400000n) << 22n) + 1001n).toString(), text: "@maya answer this genuinely post-command request", mentioned: true, userId: externalUserId, }); message.metadata.dateSent = new Date(sourceTime); const delivery = await f.admitMessage(message); expect(delivery).toMatchObject({ state: "processed", conversationId: f.conversation.id, }); expect(delivery.receivedAt.getTime()).toBeGreaterThan( f.control.publishedAt!.getTime(), ); const admitted = await f.bindRun(message.id); expect(admitted.action.status).toBe("processed"); await expect( resolveChatRunPresentationAuthorizationReason(db, { companyId: f.fixture.companyId, issueId: f.conversation.issueId, runId: admitted.runId, }), ).resolves.toBe("allow_chat_run_presentation"); } finally { await f.close(); } }); it("keeps a source admitted before close confirmation internal after a fresh source reopens", async () => { const f = await reopenedFixture(true); try { expect(f.duringConfirmation).not.toBeNull(); expect(f.duringConfirmation!.receivedAt.getTime()).toBeLessThan( f.control.publishedAt!.getTime(), ); await expect( resolveChatRunPresentationAuthorizationReason(db, { companyId: f.fixture.companyId, issueId: f.conversation.issueId, runId: f.duringConfirmation!.runId, }), ).resolves.toBe("internal_agent_write"); await expect( resolveChatRunPresentationAuthorizationReason(db, { companyId: f.fixture.companyId, issueId: f.conversation.issueId, runId: f.fresh.runId, }), ).resolves.toBe("allow_chat_run_presentation"); } finally { await f.close(); } }); it.each(["only_unproven", "newer_unproven", "newer_issue_mismatch"] as const)( "does not grant presentation across a published control with %s proof", async (mode) => { const f = await reopenedFixture(); try { let control = f.control; let source = f.fresh; if (mode !== "only_unproven") { await f.adapter.handleGatewayInteraction(f.interaction("close")); await f.service.processPendingPublications(); control = (await f.publications()).find( (row) => row.id !== f.control.id, )!; expect(control.state).toBe("published"); const messageId = freshDiscordMessageId(); const message = makeMessage({ id: messageId, text: "@maya a genuinely fresh source still needs every control proof", mentioned: true, userId: externalUserId, }); message.metadata.dateSent = new Date( Number((BigInt(messageId) >> 22n) + 1420070400000n), ); expect((await f.admitMessage(message)).state).toBe("processed"); source = await f.bindRun(messageId); await expect( resolveChatRunPresentationAuthorizationReason(db, { companyId: f.fixture.companyId, issueId: f.conversation.issueId, runId: source.runId, }), ).resolves.toBe("allow_chat_run_presentation"); } if (mode === "newer_issue_mismatch") { const unrelated = await issueService(db).create(f.fixture.companyId, { title: "Independent task must not supply control proof", createdByUserId: "owner-user", }); await db .update(chatPublications) .set({ issueId: unrelated.id }) .where(eq(chatPublications.id, control.id)); } else { await db .update(chatActions) .set({ status: "issued" }) .where( eq( chatActions.providerActionId, `task-control-authorization:${control.id}`, ), ); } if (mode !== "only_unproven") { expect( ( await db .select() .from(chatActions) .where( eq( chatActions.providerActionId, `task-control-authorization:${f.control.id}`, ), ) )[0]?.status, ).toBe("processed"); } await expect( resolveChatRunPresentationAuthorizationReason(db, { companyId: f.fixture.companyId, issueId: f.conversation.issueId, runId: source.runId, }), ).resolves.toBe("internal_agent_write"); } finally { await f.close(); } }, ); it.each(["old_source", "fresh_source"] as const)( "does not let an automatic child copy %s presentation authority", async (source) => { const f = await reopenedFixture(); try { const parent = source === "old_source" ? f.old : f.fresh; const runId = randomUUID(), wakeId = randomUUID(); await db.insert(agentWakeupRequests).values({ id: wakeId, companyId: f.fixture.companyId, agentId: f.fixture.assignedAgentId, source: "automation", triggerDetail: "system", reason: "issue_continuation_needed", requestedByActorType: "system", status: "queued", payload: { issueId: f.conversation.issueId, retryOfRunId: parent.runId, }, }); await db.insert(heartbeatRuns).values({ id: runId, companyId: f.fixture.companyId, agentId: f.fixture.assignedAgentId, nativeIssueId: f.conversation.issueId, runtimeMode: "native", status: "running", startedAt: new Date(), wakeupRequestId: wakeId, retryOfRunId: parent.runId, contextSnapshot: { ...parent.context, endpointId: f.endpoint.id, paperclipHarnessCheckedOut: true, }, }); await db .update(agentWakeupRequests) .set({ runId, status: "claimed" }) .where(eq(agentWakeupRequests.id, wakeId)); await expect( resolveChatRunPresentationAuthorizationReason(db, { companyId: f.fixture.companyId, issueId: f.conversation.issueId, runId, }), ).resolves.toBe("internal_agent_write"); } finally { await f.close(); } }, ); it("does not drop an older coalesced source while selecting the fresh batch", async () => { const f = await reopenedFixture(); try { await db .update(agentWakeupRequests) .set({ status: "coalesced", runId: f.fresh.runId, payload: sql`${agentWakeupRequests.payload} || ${JSON.stringify({ coalescedIntoWakeupRequestId: f.fresh.action.id })}::jsonb`, }) .where(eq(agentWakeupRequests.id, f.old.action.id)); // The submitted context omits the old sibling. The receipt set, not // this mutable subset, must determine whether the batch is fresh. await expect( resolveChatRunPresentationAuthorizationReason(db, { companyId: f.fixture.companyId, issueId: f.conversation.issueId, runId: f.fresh.runId, }), ).resolves.toBe("internal_agent_write"); await db .update(heartbeatRuns) .set({ contextSnapshot: { ...f.fresh.context, endpointId: f.endpoint.id, paperclipHarnessCheckedOut: true, wakeCommentIds: [ f.old.context.wakeCommentId, f.fresh.context.wakeCommentId, ], }, }) .where(eq(heartbeatRuns.id, f.fresh.runId)); await expect( resolveChatRunPresentationAuthorizationReason(db, { companyId: f.fixture.companyId, issueId: f.conversation.issueId, runId: f.fresh.runId, }), ).resolves.toBe("internal_agent_write"); } finally { await f.close(); } }); it("allows a complete freshly admitted coalesced batch", async () => { const f = await reopenedFixture(); try { const secondMessage = makeMessage({ id: freshDiscordMessageId(), text: "@maya include this fresh detail", mentioned: true, userId: externalUserId, }); await f.admitMessage(secondMessage); const second = await chatWakeContext({ endpointId: f.endpoint.id, issueId: f.conversation.issueId, provider: "discord", providerMessageId: secondMessage.id, }); const [action] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, f.endpoint.id), eq(chatActions.kind, "inbound_wakeup"), sql`${chatActions.payload}->>'commentId' = ${second.wakeCommentId}`, ), ); const receipt = await db .update(agentWakeupRequests) .set({ status: "coalesced", runId: f.fresh.runId, payload: sql`${agentWakeupRequests.payload} || ${JSON.stringify({ coalescedIntoWakeupRequestId: f.fresh.action.id })}::jsonb`, }) .where(eq(agentWakeupRequests.id, action.id)) .returning({ id: agentWakeupRequests.id }); expect(receipt).toHaveLength(1); // Model the scheduler's queued-batch claim only after both actual // admissions. Coalescing execution itself has its own heartbeat tests. await db .update(heartbeatRuns) .set({ startedAt: new Date(), contextSnapshot: { ...f.fresh.context, endpointId: f.endpoint.id, paperclipHarnessCheckedOut: true, wakeCommentIds: [ f.fresh.context.wakeCommentId, second.wakeCommentId, ], }, }) .where(eq(heartbeatRuns.id, f.fresh.runId)); await expect( resolveChatRunPresentationAuthorizationReason(db, { companyId: f.fixture.companyId, issueId: f.conversation.issueId, runId: f.fresh.runId, }), ).resolves.toBe("allow_chat_run_presentation"); } finally { await f.close(); } }); it("keeps a newly closed source internal after a still newer source reopens the thread", async () => { const f = await reopenedFixture(); try { await f.adapter.handleGatewayInteraction(f.interaction("close")); await f.service.processPendingPublications(); const controls = await f.publications(); expect(controls).toHaveLength(2); expect(controls.every((control) => control.state === "published")).toBe( true, ); const nextMessage = makeMessage({ id: freshDiscordMessageId(), text: "@maya genuinely new request after second close", mentioned: true, userId: externalUserId, }); await f.admitMessage(nextMessage); const next = await f.bindRun(nextMessage.id); expect((await f.service.listConversations(f.endpoint.id))[0]?.state).toBe( "active", ); const scope = { companyId: f.fixture.companyId, issueId: f.conversation.issueId, }; await expect( resolveChatRunPresentationAuthorizationReason(db, { ...scope, runId: f.fresh.runId, }), ).resolves.toBe("internal_agent_write"); await expect( resolveChatRunPresentationAuthorizationReason(db, { ...scope, runId: next.runId, }), ).resolves.toBe("allow_chat_run_presentation"); } finally { await f.close(); } }); it.each(["delayed", "missing"] as const)( "refuses %s provider chronology at intake when first ingested after close", async (mode) => { const f = await reopenedFixture(); try { const before = await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, f.conversation.issueId)) .orderBy(asc(issueComments.id)); const message = makeMessage({ id: mode === "delayed" ? (BigInt(f.controlInvocation.id) - (5000n << 22n)).toString() : "missing-provider-snowflake", text: "@maya this unseen source must not become fresh work", mentioned: true, userId: externalUserId, }); const delivery = await f.admitMessage(message); expect(delivery.receivedAt.getTime()).toBeGreaterThan( f.control.publishedAt!.getTime(), ); expect(delivery).toMatchObject({ state: "filtered", conversationId: null, }); expect( await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, f.conversation.issueId)) .orderBy(asc(issueComments.id)), ).toEqual(before); expect( await db .select() .from(chatActions) .where( and( eq(chatActions.deliveryId, delivery.id), eq(chatActions.kind, "inbound_wakeup"), ), ), ).toEqual([]); } finally { await f.close(); } }, ); it("orders same-millisecond Discord sources only within the same worker and process", async () => { const f = await reopenedFixture(); try { for (const offset of [-1n, 1n << 12n, 1n << 17n, 1n]) { const message = makeMessage({ id: (BigInt(f.controlInvocation.id) + offset).toString(), text: "@maya same provider millisecond", mentioned: true, userId: externalUserId, }); const delivery = await f.admitMessage(message); expect(delivery.state).toBe(offset === 1n ? "processed" : "filtered"); } } finally { await f.close(); } }); it.each(["close", "new"] as const)( "does not let an unseen stale native Discord %s act on reopened work", async (command) => { const f = await reopenedFixture(); try { const before = await f.publications(); const stale = f.interaction(command, { id: (BigInt(f.controlInvocation.id) - 1n).toString(), }); await f.adapter.handleGatewayInteraction(stale); expect(await f.publications()).toEqual(before); expect( (await f.service.listConversations(f.endpoint.id))[0]?.state, ).toBe("active"); if (command === "close") expect(stale.editReply).toHaveBeenCalledWith( expect.objectContaining({ content: expect.stringContaining("predates"), }), ); // Guild-thread /new remains guidance-only regardless of age. } finally { await f.close(); } }, ); it.each([ "edited", "deleted", "pending_lifecycle", "generation", "runtime", "credentials", "revoked_access", "reach", "thread", "unadmitted", "reassigned", ] as const)( "does not cross a historical close after current %s authority changes", async (mode) => { const f = await reopenedFixture(); try { if (mode === "edited") await db .update(issueComments) .set({ body: "Edited source", updatedAt: new Date() }) .where(eq(issueComments.id, f.fresh.context.wakeCommentId)); if (mode === "deleted") await db .update(issueComments) .set({ deletedAt: new Date() }) .where(eq(issueComments.id, f.fresh.context.wakeCommentId)); if (mode === "pending_lifecycle") await db.insert(chatDeliveries).values({ companyId: f.fixture.companyId, endpointId: f.endpoint.id, conversationId: f.conversation.id, providerEventId: `edit:${randomUUID()}`, deduplicationKey: randomUUID(), eventKind: "message_updated", state: "received", normalizedEvent: { runtimeContext: f.delivery.normalizedEvent.runtimeContext, message: { targetProviderEventId: f.delivery.providerEventId, }, }, }); if (mode === "generation") await db .update(chatConversations) .set({ sessionGeneration: f.conversation.sessionGeneration + 1, }) .where(eq(chatConversations.id, f.conversation.id)); if (mode === "runtime") await db .update(chatEndpoints) .set({ setup: sql`jsonb_set(${chatEndpoints.setup}, '{runtimeGeneration}', to_jsonb(coalesce((${chatEndpoints.setup}->>'runtimeGeneration')::integer, 0) + 1))`, }) .where(eq(chatEndpoints.id, f.endpoint.id)); if (mode === "credentials") await db .update(toolConnections) .set({ credentialSecretRefs: [] }) .where(eq(toolConnections.id, f.endpoint.connectionId)); if (mode === "revoked_access") { await db .update(chatEndpoints) .set({ allowUnlinkedPeople: false }) .where(eq(chatEndpoints.id, f.endpoint.id)); await db .update(chatIdentityLinks) .set({ status: "revoked" }) .where( and( eq(chatIdentityLinks.endpointId, f.endpoint.id), eq(chatIdentityLinks.principalId, f.principal.id), ), ); } if (mode === "reach") await db .update(chatEndpointResources) .set({ enabled: false }) .where(eq(chatEndpointResources.endpointId, f.endpoint.id)); if (mode === "thread") await db .update(chatDeliveries) .set({ normalizedEvent: { ...f.delivery.normalizedEvent, conversation: { ...(f.delivery.normalizedEvent.conversation as Record< string, unknown >), externalThreadId: "discord:foreign:thread", }, }, }) .where(eq(chatDeliveries.id, f.delivery.id)); if (mode === "unadmitted") await db .update(agentWakeupRequests) .set({ status: "skipped" }) .where(eq(agentWakeupRequests.id, f.fresh.action.id)); if (mode === "reassigned") await db .update(issues) .set({ assigneeAgentId: f.fixture.replacementAgentId }) .where(eq(issues.id, f.conversation.issueId)); await expect( resolveChatRunPresentationAuthorizationReason(db, { companyId: f.fixture.companyId, issueId: f.conversation.issueId, runId: f.fresh.runId, }), ).resolves.toBe("internal_agent_write"); await enqueueChatRunMilestones(db); expect( await db .select() .from(chatPublications) .where( like(chatPublications.idempotencyKey, `run:${f.fresh.runId}:%`), ), ).toEqual([]); } finally { await f.close(); } }, ); }); it("rolls back staged public control when the durable command receipt fails", async () => { const f = await commandFixture(); const functionName = `discord_command_failure_${randomUUID().replaceAll("-", "")}`; const triggerName = `${functionName}_trigger`; try { // Synthetic PostgreSQL fault after the outbox insert, scoped only to // this endpoint. Provider error text must never become private content. await db.execute( sql.raw( `CREATE FUNCTION ${functionName}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.endpoint_id = '${f.endpoint.id}'::uuid AND NEW.kind = 'discord_native_command' THEN RAISE EXCEPTION 'PRIVATE-COMMAND-PERSISTENCE-DETAIL'; END IF; RETURN NEW; END $$`, ), ); await db.execute( sql.raw( `CREATE TRIGGER ${triggerName} BEFORE INSERT ON chat_actions FOR EACH ROW EXECUTE FUNCTION ${functionName}()`, ), ); const close = f.interaction("close"); await f.adapter.handleGatewayInteraction(close); expect(close.deferReply).toHaveBeenCalledExactlyOnceWith({ flags: 64 }); expect(close.editReply).toHaveBeenCalledExactlyOnceWith( expect.objectContaining({ content: expect.stringContaining("could not be confirmed"), }), ); expect(JSON.stringify(close.editReply.mock.calls)).not.toContain( "PRIVATE-COMMAND", ); expect(await f.actions()).toHaveLength(0); expect(await f.publications()).toHaveLength(0); expect( ( await db .select() .from(chatConversations) .where(eq(chatConversations.id, f.conversation.id)) )[0]?.state, ).toBe("active"); } finally { try { await db.execute( sql.raw(`DROP TRIGGER IF EXISTS ${triggerName} ON chat_actions`), ); await db.execute( sql.raw(`DROP FUNCTION IF EXISTS ${functionName}()`), ); } finally { await f.close(); } } }); }); it("turns a Slack slash command into a new native thread and one Paperclip task", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const command = endpoint.setup.command; if (!command) throw new Error("Slack endpoint did not expose its command"); await db .update(agents) .set({ name: "Maya Renamed After Slack Registration" }) .where(eq(agents.id, fixture.assignedAgentId)); expect((await service.get(endpoint.id)).setup.command).toBe(command); if (!callbacks.onSlashCommand) throw new Error("Slack slash command callback was not registered"); await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: "C-COMMANDS", label: "commands", availability: "available", enabled: true, }); const starterThreadId = "slack:C-COMMANDS:outbound-1"; const post = vi.fn(async () => ({ id: "6000.1", // The real Chat SDK channel wrapper returns its channel id here. The // Slack message timestamp above is the native thread root. threadId: "slack:C-COMMANDS", })); const postEphemeral = vi.fn(async () => ({ id: "ephemeral-6000", threadId: starterThreadId, })); await callbacks.onSlashCommand({ endpointId: endpoint.id, provider: "slack", event: { channel: { id: "C-COMMANDS", name: "commands", isDM: false, post, postEphemeral, } as never, command, text: "investigate the command path", triggerId: "trigger-6000", user: { userId: "U-COMMANDER", userName: "commander", fullName: "Command User", isBot: false, isMe: false, isSystem: false, }, raw: { trigger_id: "trigger-6000" }, adapter: {} as never, openModal: async () => undefined, }, }); await vi.waitFor(() => expect(runtime.endpoints.get(endpoint.id)?.posts).toContainEqual({ threadId: "slack:C-COMMANDS:", text: "Starting a task…", }), ); expect(post).not.toHaveBeenCalled(); await vi.waitFor(async () => expect(await service.listConversations(endpoint.id)).toHaveLength(1), ); const [conversation] = await service.listConversations(endpoint.id); expect(conversation).toMatchObject({ externalThreadId: starterThreadId, state: "active", }); const comments = await db .select() .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)); expect(comments.map((comment) => comment.body)).toEqual([ "investigate the command path", ]); expect(runtime.endpoints.get(endpoint.id)?.reactions).toEqual([]); await vi.waitFor(async () => { expect( await db .select({ state: chatDeliveries.state, redactedError: chatDeliveries.redactedError, normalizedEvent: chatDeliveries.normalizedEvent, }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)), ).toEqual([ { state: "processed", redactedError: null, normalizedEvent: expect.objectContaining({ acknowledgement: { receiptReactionSupported: false }, message: expect.objectContaining({ text: "investigate the command path", }), }), }, ]); }); }); it("admits concurrent and retried Slack slash commands only once", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredSlackEndpoint(fixture); const command = endpoint.setup.command; if (!command) throw new Error("Slack endpoint did not expose its command"); if (!callbacks.onSlashCommand) throw new Error("Slack slash command callback was not registered"); await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: "C-COMMAND-RETRY", label: "command-retry", availability: "available", enabled: true, }); const starterThreadId = "slack:C-COMMAND-RETRY:outbound-1"; let releasePost!: () => void; const post = vi.fn(); const postEphemeral = vi.fn(async () => ({ id: "ephemeral-6001", threadId: starterThreadId, })); const slashEvent = { endpointId: endpoint.id, provider: "slack" as const, event: { channel: { id: "C-COMMAND-RETRY", name: "command-retry", isDM: false, post, postEphemeral, } as never, command, text: "investigate one retried command", triggerId: "trigger-6001", user: { userId: "U-COMMAND-RETRY", userName: "command-retry", fullName: "Command Retry User", isBot: false, isMe: false, isSystem: false, }, raw: { trigger_id: "trigger-6001" }, adapter: {} as never, openModal: async () => undefined, }, }; const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack provider runtime"); const postEntered = vi.fn(); providerRuntime.postHook = async () => { postEntered(); await new Promise((resolve) => { releasePost = resolve; }); }; const first = callbacks.onSlashCommand(slashEvent); await expect( Promise.race([ first.then(() => "acknowledged" as const), new Promise<"timed_out">((resolve) => setTimeout(() => resolve("timed_out"), 250), ), ]), ).resolves.toBe("acknowledged"); await vi.waitFor(() => expect(postEntered).toHaveBeenCalledTimes(1)); const concurrentRetry = callbacks.onSlashCommand(slashEvent); releasePost(); await Promise.all([first, concurrentRetry]); providerRuntime.postHook = undefined; // A later provider retry resumes from the durable root binding. The // synthetic inbound ledger then proves that the task mutation already ran. await callbacks.onSlashCommand(slashEvent); expect(post).not.toHaveBeenCalled(); expect(providerRuntime.posts).toEqual([ { threadId: "slack:C-COMMAND-RETRY:", text: "Starting a task…", }, ]); await vi.waitFor(() => expect(wakeup).toHaveBeenCalledTimes(1)); const conversations = await service.listConversations(endpoint.id); expect(conversations).toEqual([ expect.objectContaining({ externalThreadId: starterThreadId, state: "active", }), ]); await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, conversations[0]!.issueId)), ).resolves.toEqual([{ body: "investigate one retried command" }]); // The wake callback runs before slash admission commits its terminal // receipt. Observe that exact durable receipt before simulating an // operator resolution, rather than racing the final admission write. const [action] = await vi.waitFor(async () => { const actions = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slash_task_start"), ), ); expect(actions).toEqual([ expect.objectContaining({ kind: "slash_task_start", status: "processed", result: expect.objectContaining({ threadId: starterThreadId, providerMessageId: "outbound-1", }), }), ]); return actions; }); await db .update(chatActions) .set({ status: "delivery_unknown", result: { code: "slash_task_delivery_unknown" }, }) .where(eq(chatActions.id, action!.id)); expect(await service.listActivity(endpoint.id)).toEqual( expect.arrayContaining([ expect.objectContaining({ id: action!.id, kind: "action", status: "delivery_unknown", summary: "Slack slash-command task start delivery unknown", detail: expect.stringContaining("will not replay it automatically"), replayable: false, resolutionActions: ["retry_anyway", "cancel"], }), ]), ); }); it("releases authorization row locks before a Slack slash starter waits on provider I/O", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture, { scheduleDeferredWork: () => undefined, }); if (!callbacks.onSlashCommand || !endpoint.setup.command) { throw new Error("Slack slash command setup was incomplete"); } const [resource] = await db .insert(chatEndpointResources) .values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: "C-COMMAND-LOCK-PROBE", label: "command-lock-probe", availability: "available", enabled: true, }) .returning(); await callbacks.onSlashCommand({ endpointId: endpoint.id, provider: "slack", event: { channel: { id: "C-COMMAND-LOCK-PROBE", name: "command-lock-probe", isDM: false, post: vi.fn(), postEphemeral: vi.fn(), } as never, command: endpoint.setup.command, text: "prove provider latency does not hold database locks", triggerId: "trigger-command-lock-probe", user: { userId: "U-COMMAND-LOCK-PROBE", userName: "command-lock-probe", fullName: "Command Lock Probe", isBot: false, isMe: false, isSystem: false, }, raw: { trigger_id: "trigger-command-lock-probe" }, adapter: {} as never, openModal: async () => undefined, }, }); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack provider runtime"); let releasePost!: () => void; const postEntered = vi.fn(); providerRuntime.postHook = async () => { postEntered(); await new Promise((resolve) => { releasePost = resolve; }); }; const processing = service.processPendingDeliveries(1_000); await vi.waitFor(() => expect(postEntered).toHaveBeenCalledTimes(1)); const resourceUpdate = db .update(chatEndpointResources) .set({ enabled: false, updatedAt: new Date() }) .where(eq(chatEndpointResources.id, resource!.id)) .returning({ id: chatEndpointResources.id }) .then(() => "updated" as const); const updateOutcome = await Promise.race([ resourceUpdate, new Promise<"timed_out">((resolve) => setTimeout(() => resolve("timed_out"), 500), ), ]); releasePost(); await processing; await resourceUpdate; providerRuntime.postHook = undefined; // The worker snapshots its inbound batch before running action outboxes. // The confirmed Slack root is admitted with the normal short reorder // window. This fixture disables deferred scheduling, so make that durable // delivery ready before a second explicit pass drains it without another // provider call. await db .update(chatDeliveries) .set({ nextAttemptAt: null, updatedAt: new Date() }) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.state, "received"), ), ); await service.processPendingDeliveries(1_000); expect(updateOutcome).toBe("updated"); expect(providerRuntime.posts).toEqual([ { threadId: "slack:C-COMMAND-LOCK-PROBE:", text: "Starting a task…", }, ]); // Provider transport owns only its short authorization snapshot. The // resource update completed while Slack was in flight, so the later // Paperclip task mutation must honor the now-disabled destination. await expect(service.listConversations(endpoint.id)).resolves.toEqual([]); await expect( db .select({ id: issues.id }) .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).resolves.toEqual([]); await expect( db .select({ normalizedEvent: chatDeliveries.normalizedEvent, principalId: chatDeliveries.principalId, redactedError: chatDeliveries.redactedError, }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)), ).resolves.toEqual([ { normalizedEvent: expect.objectContaining({ filtering: { contentRetained: false }, message: expect.objectContaining({ providerMessageId: expect.any(String), }), }), principalId: null, redactedError: "Destination is not enabled in Paperclip", }, ]); await expect( db .select({ enabled: chatEndpointResources.enabled }) .from(chatEndpointResources) .where(eq(chatEndpointResources.id, resource!.id)), ).resolves.toEqual([{ enabled: false }]); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slash_task_start"), ), ), ).resolves.toEqual([{ status: "processed" }]); await service.shutdown(); }); it("parks provider-confirmed Slack admissions on paused endpoints without starving active work", async () => { const pausedFixture = await seedCompany(); const activeFixture = await seedCompany(); const paused = await configuredSlackEndpoint(pausedFixture, { scheduleDeferredWork: () => undefined, }); const active = await configuredSlackEndpoint(activeFixture, { scheduleDeferredWork: () => undefined, }); const pausedEndpoint = await paused.service.get(paused.endpoint.id); const activeEndpoint = await active.service.get(active.endpoint.id); if ( !pausedEndpoint.providerAccountId || !pausedEndpoint.setup.command || !activeEndpoint.providerAccountId || !activeEndpoint.setup.command ) { throw new Error("Slack endpoint setup was incomplete"); } await db .update(chatEndpoints) .set({ status: "paused", updatedAt: new Date() }) .where(eq(chatEndpoints.id, paused.endpoint.id)); const [pausedResource, activeResource] = await db .insert(chatEndpointResources) .values([ { companyId: pausedFixture.companyId, endpointId: paused.endpoint.id, type: "channel" as const, providerResourceId: "C-PAUSED-CONFIRMED-STARTER", label: "paused-confirmed-starter", availability: "available" as const, enabled: true, }, { companyId: activeFixture.companyId, endpointId: active.endpoint.id, type: "channel" as const, providerResourceId: "C-ACTIVE-CONFIRMED-STARTER", label: "active-confirmed-starter", availability: "available" as const, enabled: true, }, ]) .returning(); const [pausedPrincipal, activePrincipal] = await db .insert(chatExternalPrincipals) .values([ { companyId: pausedFixture.companyId, provider: "slack" as const, providerAccountId: pausedEndpoint.providerAccountId, externalId: "U-PAUSED-CONFIRMED-STARTER", kind: "user" as const, displayName: "Paused Confirmed Starter", handle: "paused-confirmed-starter", isBot: false, }, { companyId: activeFixture.companyId, provider: "slack" as const, providerAccountId: activeEndpoint.providerAccountId, externalId: "U-ACTIVE-CONFIRMED-STARTER", kind: "user" as const, displayName: "Active Confirmed Starter", handle: "active-confirmed-starter", isBot: false, }, ]) .returning(); const createdAt = new Date("2026-09-06T15:00:00.000Z"); const [pausedAction, activeAction] = await db .insert(chatActions) .values([ { companyId: pausedFixture.companyId, endpointId: paused.endpoint.id, principalId: pausedPrincipal!.id, kind: "slash_task_start" as const, providerActionId: `slash_task:${randomUUID()}`, payload: { version: 1, channelId: pausedResource!.providerResourceId, command: pausedEndpoint.setup.command, syntheticMessageId: randomUUID(), taskText: "keep this confirmed admission parked", }, status: "provider_confirmed" as const, result: { threadId: `slack:${pausedResource!.providerResourceId}:6270.1`, providerMessageId: "6270.1", }, createdAt, }, { companyId: activeFixture.companyId, endpointId: active.endpoint.id, principalId: activePrincipal!.id, kind: "slash_task_start" as const, providerActionId: `slash_task:${randomUUID()}`, payload: { version: 1, channelId: activeResource!.providerResourceId, command: activeEndpoint.setup.command, syntheticMessageId: randomUUID(), taskText: "admit this active task despite the older parked row", }, status: "provider_confirmed" as const, result: { threadId: `slack:${activeResource!.providerResourceId}:6270.2`, providerMessageId: "6270.2", }, createdAt: new Date(createdAt.getTime() + 1), }, ]) .returning(); await active.service.processPendingDeliveries(1); const [activeDelivery] = await db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, active.endpoint.id)); if (!activeDelivery) throw new Error("Expected active Slack admission delivery"); await db .update(chatDeliveries) .set({ nextAttemptAt: null, updatedAt: new Date() }) .where(eq(chatDeliveries.id, activeDelivery.id)); await active.service.processPendingDeliveries(1, activeDelivery.id); await expect( db .select({ id: chatActions.id, status: chatActions.status }) .from(chatActions) .where(inArray(chatActions.id, [pausedAction!.id, activeAction!.id])) .orderBy(asc(chatActions.createdAt)), ).resolves.toEqual([ { id: pausedAction!.id, status: "provider_confirmed" }, { id: activeAction!.id, status: "processed" }, ]); await expect( paused.service.listConversations(paused.endpoint.id), ).resolves.toEqual([]); await expect( active.service.listConversations(active.endpoint.id), ).resolves.toEqual([ expect.objectContaining({ externalThreadId: `slack:${activeResource!.providerResourceId}:6270.2`, }), ]); await db .update(chatActions) .set({ status: "cancelled", result: { code: "test_fixture_complete" }, updatedAt: new Date(), }) .where(eq(chatActions.id, pausedAction!.id)); await paused.service.shutdown(); await active.service.shutdown(); }); it("reconciles a provider-confirmed Slack starter into one task without replaying Slack", async () => { const fixture = await seedCompany(); const { endpoint, runtime, service, wakeup } = await configuredSlackEndpoint(fixture); const configured = await service.get(endpoint.id); if (!configured.providerAccountId) { throw new Error("Slack endpoint did not expose its provider account"); } const command = configured.setup.command; if (!command) throw new Error("Slack endpoint did not expose its command"); const [resource] = await db .insert(chatEndpointResources) .values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: "C-CONFIRMED-STARTER", label: "confirmed-starter", availability: "available", enabled: true, }) .returning(); const [principal] = await db .insert(chatExternalPrincipals) .values({ companyId: fixture.companyId, provider: "slack", providerAccountId: configured.providerAccountId, externalId: "U-CONFIRMED-STARTER", kind: "user", displayName: "Confirmed Starter User", handle: "confirmed-starter", isBot: false, }) .returning(); const syntheticMessageId = `confirmed-starter-${randomUUID()}`; const starterThreadId = "slack:C-CONFIRMED-STARTER:6200.1"; const [action] = await db .insert(chatActions) .values({ companyId: fixture.companyId, endpointId: endpoint.id, principalId: principal!.id, kind: "slash_task_start", providerActionId: `slash_task:${randomUUID()}`, payload: { version: 1, channelId: "C-CONFIRMED-STARTER", command, syntheticMessageId, taskText: "admit this already-confirmed starter exactly once", }, status: "provider_confirmed", result: { threadId: starterThreadId, providerMessageId: "6200.1", }, }) .returning(); // This row models the crash window after Slack returned the root timestamp // but before the original callback could durably admit the Paperclip task. await service.processPendingDeliveries(1_000); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.endpointId, endpoint.id)); await service.processPendingDeliveries(1_000); expect(runtime.endpoints.get(endpoint.id)?.posts).toEqual([]); expect(wakeup).toHaveBeenCalledTimes(1); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, action!.id)), ).resolves.toEqual([ { status: "processed", result: { threadId: starterThreadId, providerMessageId: "6200.1", }, }, ]); const conversations = await service.listConversations(endpoint.id); expect(conversations).toEqual([ expect.objectContaining({ resourceId: resource!.id, externalThreadId: starterThreadId, state: "active", }), ]); await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, conversations[0]!.issueId)), ).resolves.toEqual([ { body: "admit this already-confirmed starter exactly once" }, ]); await expect( db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)), ).resolves.toEqual([{ state: "processed" }]); }); it("fences a reclaimed Slack task admission from the obsolete worker's settlement", async () => { const fixture = await seedCompany(); let claimCount = 0; let markFirstClaimed!: () => void; let markSecondClaimed!: () => void; let releaseFirstClaim!: () => void; let releaseSecondClaim!: () => void; const firstClaimed = new Promise((resolve) => { markFirstClaimed = resolve; }); const secondClaimed = new Promise((resolve) => { markSecondClaimed = resolve; }); const firstClaimHold = new Promise((resolve) => { releaseFirstClaim = resolve; }); const secondClaimHold = new Promise((resolve) => { releaseSecondClaim = resolve; }); const { endpoint, service } = await configuredSlackEndpoint(fixture, { scheduleDeferredWork: () => undefined, slackTaskAdmissionClaimBarrier: async () => { claimCount += 1; if (claimCount === 1) { markFirstClaimed(); await firstClaimHold; } else if (claimCount === 2) { markSecondClaimed(); await secondClaimHold; } }, }); const configured = await service.get(endpoint.id); if (!configured.providerAccountId || !configured.setup.command) { throw new Error("Slack endpoint setup was incomplete"); } await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: "C-RECLAIMED-ADMISSION", label: "reclaimed-admission", availability: "available", enabled: true, }); const [principal] = await db .insert(chatExternalPrincipals) .values({ companyId: fixture.companyId, provider: "slack", providerAccountId: configured.providerAccountId, externalId: "U-RECLAIMED-ADMISSION", kind: "user", displayName: "Reclaimed Admission User", handle: "reclaimed-admission", isBot: false, }) .returning(); const syntheticMessageId = `reclaimed-admission-${randomUUID()}`; const starterThreadId = "slack:C-RECLAIMED-ADMISSION:6250.1"; const [action] = await db .insert(chatActions) .values({ companyId: fixture.companyId, endpointId: endpoint.id, principalId: null, kind: "slash_task_start", providerActionId: `slash_task:${randomUUID()}`, payload: { version: 1, channelId: "C-RECLAIMED-ADMISSION", command: configured.setup.command, syntheticMessageId, taskText: "admit this reclaimed Slack task exactly once", }, status: "provider_confirmed", result: { threadId: starterThreadId, providerMessageId: "6250.1", }, }) .returning(); const obsoleteWorker = service.processPendingDeliveries(); await firstClaimed; await db .update(chatActions) .set({ principalId: principal!.id, updatedAt: new Date(0) }) .where(eq(chatActions.id, action!.id)); const successorWorker = service.processPendingDeliveries(); await secondClaimed; releaseFirstClaim(); await obsoleteWorker; await expect( db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, action!.id)), ).resolves.toEqual([{ status: "admitting" }]); releaseSecondClaim(); await successorWorker; await expect( db .select({ result: chatActions.result, status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, action!.id)), ).resolves.toEqual([ { result: { threadId: starterThreadId, providerMessageId: "6250.1", }, status: "processed", }, ]); await expect( db .select({ providerEventId: chatDeliveries.providerEventId }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)), ).resolves.toEqual([ { providerEventId: `${starterThreadId}:${syntheticMessageId}` }, ]); // Admission creates a durable inbound receipt after this worker's delivery // batch was selected. Drain it with its own fixture before another test's // global worker observes that otherwise-valid pending message. await db .update(chatDeliveries) .set({ nextAttemptAt: null, updatedAt: new Date() }) .where(eq(chatDeliveries.endpointId, endpoint.id)); await service.processPendingDeliveries(1_000); await expect( db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)), ).resolves.toEqual([{ state: "processed" }]); }); it("rechecks linked authority and channel reach before admitting a provider-confirmed Slack task", async () => { for (const authorizationChange of [ "link_revoked", "viewer", "resource_disabled", ] as const) { const fixture = await seedCompany(); const { endpoint, runtime, service, wakeup } = await configuredSlackEndpoint(fixture, { allowUnlinkedPeople: false, scheduleDeferredWork: () => undefined, }); const configured = await service.get(endpoint.id); if (!configured.providerAccountId || !configured.setup.command) { throw new Error("Slack endpoint setup was incomplete"); } const externalPrincipalId = `U-CONFIRMED-AUTH-${authorizationChange}`; const [resource] = await db .insert(chatEndpointResources) .values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: `C-CONFIRMED-AUTH-${authorizationChange}`, label: `confirmed-auth-${authorizationChange}`, availability: "available", enabled: true, }) .returning(); const [principal] = await db .insert(chatExternalPrincipals) .values({ companyId: fixture.companyId, provider: "slack", providerAccountId: configured.providerAccountId, externalId: externalPrincipalId, kind: "user", displayName: "Confirmed Slack User", handle: "confirmed-slack-user", isBot: false, }) .returning(); await db.insert(chatIdentityLinks).values({ companyId: fixture.companyId, endpointId: endpoint.id, principalId: principal!.id, paperclipUserId: "owner-user", status: "linked", confirmedAt: new Date(), }); const syntheticMessageId = `confirmed-auth-${authorizationChange}-${randomUUID()}`; const starterThreadId = `slack:${resource!.providerResourceId}:6260.1`; const [action] = await db .insert(chatActions) .values({ companyId: fixture.companyId, endpointId: endpoint.id, principalId: principal!.id, kind: "slash_task_start", providerActionId: `slash_task:${randomUUID()}`, payload: { version: 1, channelId: resource!.providerResourceId, command: configured.setup.command, syntheticMessageId, taskText: `must-not-create-${authorizationChange}`, }, status: "provider_confirmed", result: { authorizedUserId: "owner-user", threadId: starterThreadId, providerMessageId: "6260.1", }, }) .returning(); if (authorizationChange === "link_revoked") { await db .update(chatIdentityLinks) .set({ paperclipUserId: null, status: "revoked", revokedAt: new Date(), updatedAt: new Date(), }) .where(eq(chatIdentityLinks.principalId, principal!.id)); } else if (authorizationChange === "viewer") { await db .update(companyMemberships) .set({ membershipRole: "viewer", updatedAt: new Date() }) .where( and( eq(companyMemberships.companyId, fixture.companyId), eq(companyMemberships.principalType, "user"), eq(companyMemberships.principalId, "owner-user"), ), ); } else { await db .update(chatEndpointResources) .set({ enabled: false, updatedAt: new Date() }) .where(eq(chatEndpointResources.id, resource!.id)); } await service.processPendingDeliveries(1_000); await db .update(chatDeliveries) .set({ nextAttemptAt: null, updatedAt: new Date() }) .where(eq(chatDeliveries.endpointId, endpoint.id)); await service.processPendingDeliveries(1_000); expect(runtime.endpoints.get(endpoint.id)?.posts).toEqual([]); expect(wakeup).not.toHaveBeenCalled(); await expect(service.listConversations(endpoint.id)).resolves.toEqual([]); await expect( db .select({ id: issues.id }) .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).resolves.toEqual([]); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, action!.id)), ).resolves.toEqual([{ status: "processed" }]); const [delivery] = await db .select({ normalizedEvent: chatDeliveries.normalizedEvent, principalId: chatDeliveries.principalId, redactedError: chatDeliveries.redactedError, state: chatDeliveries.state, }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(delivery).toMatchObject({ normalizedEvent: { filtering: { contentRetained: false }, message: { providerMessageId: syntheticMessageId }, }, principalId: null, redactedError: authorizationChange === "resource_disabled" ? "Destination is not enabled in Paperclip" : authorizationChange === "viewer" ? "Linked Paperclip account is not currently permitted" : "External identity must be linked to a Paperclip account", state: "filtered", }); expect(JSON.stringify(delivery?.normalizedEvent)).not.toContain( `must-not-create-${authorizationChange}`, ); await service.shutdown(); } }); it("recovers a provider-confirmed Slack starter after restart but rechecks revoked reach", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredSlackEndpoint(fixture, { // Keep the synthetic inbound delivery queued so the test can discard // every process-local callback object before a replacement service // reconstructs it from the durable ledger. deferWebhookProcessing: true, scheduleDeferredWork: () => undefined, }); const configured = await service.get(endpoint.id); if (!configured.providerAccountId || !configured.setup.command) { throw new Error("Slack endpoint setup was incomplete"); } const [resource] = await db .insert(chatEndpointResources) .values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: "C-QUEUED-STARTER", label: "queued-starter", availability: "available", enabled: true, }) .returning(); const [principal] = await db .insert(chatExternalPrincipals) .values({ companyId: fixture.companyId, provider: "slack", providerAccountId: configured.providerAccountId, externalId: "U-QUEUED-STARTER", kind: "user", displayName: "Queued Starter User", handle: "queued-starter", isBot: false, }) .returning(); const providerMessageId = randomUUID(); const syntheticMessageId = randomUUID(); const confirmedThreadId = `slack:C-QUEUED-STARTER:${providerMessageId}`; const [action] = await db .insert(chatActions) .values({ companyId: fixture.companyId, endpointId: endpoint.id, principalId: principal!.id, kind: "slash_task_start", providerActionId: `slash_task:${randomUUID()}`, payload: { version: 1, channelId: "C-QUEUED-STARTER", command: configured.setup.command, syntheticMessageId, taskText: "recover this safely queued slash task", }, status: "provider_confirmed", result: { attemptCount: 1, authorizedUserId: null, providerMessageId, threadId: confirmedThreadId, }, }) .returning(); // Seed the exact crash boundary deterministically: Slack has confirmed the // starter and Paperclip has durably normalized it, but no delivery drain // has begun. A global reconciliation sweep intentionally runs action and // delivery lanes concurrently, so using it to create this fixture made // the pre-shutdown assertion depend on query scheduling under load. const endpointRuntime = runtime.endpoints.get(endpoint.id); if (!endpointRuntime) throw new Error("Slack runtime was unavailable"); await callbacks.onMessage({ endpointId: endpoint.id, provider: "slack", thread: endpointRuntime.thread(confirmedThreadId), message: makeMessage({ id: syntheticMessageId, text: "recover this safely queued slash task", userId: "U-QUEUED-STARTER", }), trigger: "mention", }); const [normalizedStarter] = await db .select({ id: chatDeliveries.id, normalizedEvent: chatDeliveries.normalizedEvent, }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); if (!normalizedStarter) { throw new Error("Expected normalized Slack starter delivery"); } await db .update(chatDeliveries) .set({ normalizedEvent: { ...(normalizedStarter.normalizedEvent as Record), admission: { origin: "provider_confirmed_action" }, acknowledgement: { receiptReactionSupported: false }, }, }) .where(eq(chatDeliveries.id, normalizedStarter.id)); expect(action).toMatchObject({ status: "provider_confirmed", result: { providerMessageId, threadId: confirmedThreadId }, }); const [stagedDelivery] = await db .select({ normalizedEvent: chatDeliveries.normalizedEvent, state: chatDeliveries.state, }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(stagedDelivery?.state).toBe("received"); expect(stagedDelivery?.normalizedEvent).not.toHaveProperty( "admissionAuthorization", ); expect(stagedDelivery?.normalizedEvent).toMatchObject({ admission: { origin: "provider_confirmed_action" }, }); if (!stagedDelivery) throw new Error("Expected staged Slack delivery"); // Simulate a row written by a pre-upgrade worker. Recovery must ignore the // historical authorization decision, while recognizing its presence as a // restrictive action-origin marker that cannot reactivate setup reach. const { admission: _currentAdmission, ...legacyNormalizedEvent } = stagedDelivery.normalizedEvent as Record; await db .update(chatDeliveries) .set({ normalizedEvent: { ...legacyNormalizedEvent, admissionAuthorization: { version: 1, authorizedUserId: null, }, }, updatedAt: new Date(), }) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(wakeup).not.toHaveBeenCalled(); await service.shutdown(); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0), updatedAt: new Date() }) .where(eq(chatDeliveries.endpointId, endpoint.id)); await db .update(chatEndpoints) .set({ allowUnlinkedPeople: false, updatedAt: new Date() }) .where(eq(chatEndpoints.id, endpoint.id)); await db .update(chatEndpointResources) .set({ enabled: false, updatedAt: new Date() }) .where(eq(chatEndpointResources.id, resource!.id)); const recoveredRuntime = new FakeChatSdkRuntime(); const { service: recoveredService, wakeup: recoveredWakeup } = createService(recoveredRuntime); await recoveredService.processPendingDeliveries(1_000); // Slack already accepted the starter, so recovery must never post it again. // Task admission is a distinct Paperclip mutation and must still honor the // destination and identity policy that is current after restart. expect(recoveredRuntime.endpoints.get(endpoint.id)?.posts).toEqual([]); expect(recoveredWakeup).not.toHaveBeenCalled(); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, action!.id)), ).resolves.toEqual([{ status: "processed" }]); await expect( recoveredService.listConversations(endpoint.id), ).resolves.toEqual([]); await expect( db .select({ id: issues.id }) .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).resolves.toEqual([]); await expect( db .select({ normalizedEvent: chatDeliveries.normalizedEvent, principalId: chatDeliveries.principalId, redactedError: chatDeliveries.redactedError, state: chatDeliveries.state, }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)), ).resolves.toEqual([ expect.objectContaining({ normalizedEvent: expect.objectContaining({ filtering: { contentRetained: false }, }), principalId: null, redactedError: "Destination is not enabled in Paperclip", state: "filtered", }), ]); await recoveredService.shutdown(); }); it("cancels a queued Slack slash start when channel reach is revoked before transport", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture, { // Model a process disappearing after the durable callback receipt but // before its deferred transport worker starts. scheduleDeferredWork: () => undefined, }); if (!callbacks.onSlashCommand || !endpoint.setup.command) { throw new Error("Slack slash command setup was incomplete"); } const [resource] = await db .insert(chatEndpointResources) .values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: "C-QUEUED-REVOKE", label: "queued-revoke", availability: "available", enabled: true, }) .returning(); await callbacks.onSlashCommand({ endpointId: endpoint.id, provider: "slack", event: { channel: { id: "C-QUEUED-REVOKE", name: "queued-revoke", isDM: false, post: vi.fn(), postEphemeral: vi.fn(), } as never, command: endpoint.setup.command, text: "must not escape after reach is revoked", triggerId: "trigger-queued-revoke", user: { userId: "U-QUEUED-REVOKE", userName: "queued-revoke", fullName: "Queued Revoke User", isBot: false, isMe: false, isSystem: false, }, raw: { trigger_id: "trigger-queued-revoke" }, adapter: {} as never, openModal: async () => undefined, }, }); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slash_task_start"), ), ), ).resolves.toEqual([{ status: "queued" }]); await service.replaceResources(endpoint.id, [ { id: resource!.id, enabled: false }, ]); await service.processPendingDeliveries(); expect(runtime.endpoints.get(endpoint.id)?.posts).toEqual([]); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slash_task_start"), ), ), ).resolves.toEqual([ { status: "cancelled", result: { code: "slash_task_no_longer_authorized", attemptCount: 1, }, }, ]); await expect(service.listConversations(endpoint.id)).resolves.toEqual([]); }); it("honors Slack slash-start rate-limit backoff without hammering transport", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture, { scheduleDeferredWork: () => undefined, }); if (!callbacks.onSlashCommand || !endpoint.setup.command) { throw new Error("Slack slash command setup was incomplete"); } await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: "C-QUEUED-RATE-LIMIT", label: "queued-rate-limit", availability: "available", enabled: true, }); await callbacks.onSlashCommand({ endpointId: endpoint.id, provider: "slack", event: { channel: { id: "C-QUEUED-RATE-LIMIT", name: "queued-rate-limit", isDM: false, post: vi.fn(), postEphemeral: vi.fn(), } as never, command: endpoint.setup.command, text: "respect the provider retry window", triggerId: "trigger-queued-rate-limit", user: { userId: "U-QUEUED-RATE-LIMIT", userName: "queued-rate-limit", fullName: "Queued Rate Limit User", isBot: false, isMe: false, isSystem: false, }, raw: { trigger_id: "trigger-queued-rate-limit" }, adapter: {} as never, openModal: async () => undefined, }, }); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack provider runtime"); const transportAttempts = vi.fn(); providerRuntime.postHook = async () => { transportAttempts(); }; providerRuntime.postError = Object.assign(new Error("Slack rate limited"), { name: "RateLimitError", retryAfter: 30, }); await service.processPendingDeliveries(); expect(transportAttempts).toHaveBeenCalledTimes(1); const [deferred] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slash_task_start"), ), ); expect(deferred).toMatchObject({ status: "queued", result: { code: "slash_task_retry", retryable: true, attemptCount: 1, retryAt: expect.any(String), }, }); expect( new Date(String(deferred!.result?.retryAt)).getTime(), ).toBeGreaterThan(Date.now() + 25_000); await service.processPendingDeliveries(); expect(transportAttempts).toHaveBeenCalledTimes(1); await db .update(chatActions) .set({ result: { ...deferred!.result, retryAt: new Date(0).toISOString(), }, updatedAt: new Date(), }) .where(eq(chatActions.id, deferred!.id)); providerRuntime.postError = null; await service.processPendingDeliveries(); expect(transportAttempts).toHaveBeenCalledTimes(2); expect(providerRuntime.posts).toEqual([ { threadId: "slack:C-QUEUED-RATE-LIMIT:", text: "Starting a task…", }, ]); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, deferred!.id)), ).resolves.toEqual([{ status: "processed" }]); }); it("recovers an ambiguous Slack slash-task start only after an audited explicit retry", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const command = endpoint.setup.command; if (!command) throw new Error("Slack endpoint did not expose its command"); if (!callbacks.onSlashCommand) { throw new Error("Slack slash command callback was not registered"); } await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: "C-COMMAND-RECOVERY", label: "command-recovery", availability: "available", enabled: true, }); const post = vi.fn(); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack provider runtime"); providerRuntime.postError = new Error("socket reset after write"); await callbacks.onSlashCommand({ endpointId: endpoint.id, provider: "slack", event: { channel: { id: "C-COMMAND-RECOVERY", name: "command-recovery", isDM: false, post, postEphemeral: vi.fn(), } as never, command, text: "recover this ambiguous command exactly once", triggerId: "trigger-command-recovery", user: { userId: "U-COMMAND-RECOVERY", userName: "command-recovery", fullName: "Command Recovery User", isBot: false, isMe: false, isSystem: false, }, raw: { trigger_id: "trigger-command-recovery" }, adapter: {} as never, openModal: async () => undefined, }, }); await vi.waitFor(async () => expect( await db .select({ status: chatActions.status }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slash_task_start"), ), ), ).toEqual([{ status: "delivery_unknown" }]), ); expect(post).not.toHaveBeenCalled(); await vi.waitFor(async () => expect( await db .select({ status: chatActions.status }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slash_task_start"), ), ), ).toEqual([{ status: "delivery_unknown" }]), ); const [action] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slash_task_start"), ), ); expect(action).toMatchObject({ kind: "slash_task_start", status: "delivery_unknown", payload: { channelId: "C-COMMAND-RECOVERY", taskText: "recover this ambiguous command exactly once", syntheticMessageId: expect.any(String), }, }); expect(await service.listActivity(endpoint.id)).toEqual( expect.arrayContaining([ expect.objectContaining({ id: action!.id, kind: "action", status: "delivery_unknown", replayable: false, resolutionActions: ["retry_anyway", "cancel"], }), ]), ); const app = routesApp(db, fixture.companyId, service); providerRuntime.postError = null; let releaseRetryPost!: () => void; const retryPostEntered = vi.fn(); providerRuntime.postHook = async () => { retryPostEntered(); await new Promise((resolve) => { releaseRetryPost = resolve; }); }; const firstRetry = request(app) .post(`/api/chat-endpoints/${endpoint.id}/actions/${action!.id}/resolve`) .send({ action: "retry_anyway" }) .then((response) => response); await vi.waitFor(() => expect(retryPostEntered).toHaveBeenCalledTimes(1)); const concurrentRetry = request(app) .post(`/api/chat-endpoints/${endpoint.id}/actions/${action!.id}/resolve`) .send({ action: "retry_anyway" }) .then((response) => response); releaseRetryPost(); const retryResponses = await Promise.all([firstRetry, concurrentRetry]); expect(retryResponses.map((response) => response.status).sort()).toEqual([ 204, 409, ]); providerRuntime.postHook = undefined; expect(runtime.endpoints.get(endpoint.id)?.posts).toEqual([ { threadId: "slack:C-COMMAND-RECOVERY:", text: "Starting a task…", }, ]); const conversations = await service.listConversations(endpoint.id); expect(conversations).toHaveLength(1); expect(conversations[0]?.externalThreadId).toBe( "slack:C-COMMAND-RECOVERY:outbound-1", ); await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, conversations[0]!.issueId)), ).resolves.toEqual([ { body: "recover this ambiguous command exactly once" }, ]); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, action!.id)), ).resolves.toEqual([ { status: "processed", result: expect.objectContaining({ authorizedUserId: null }), }, ]); const [delivery] = await db .select({ normalizedEvent: chatDeliveries.normalizedEvent }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(delivery?.normalizedEvent).not.toHaveProperty( "admissionAuthorization", ); await expect( db .select({ action: activityLog.action, actorId: activityLog.actorId }) .from(activityLog) .where(eq(activityLog.entityId, action!.id)), ).resolves.toEqual( expect.arrayContaining([ { action: "chat.slack_command_retry_anyway", actorId: "owner-user", }, ]), ); expect(runtime.endpoints.get(endpoint.id)?.posts).toHaveLength(1); const [cancelAction] = await db .insert(chatActions) .values({ companyId: fixture.companyId, endpointId: endpoint.id, kind: "slash_task_start", providerActionId: `slash_task:cancel-${randomUUID()}`, payload: {}, status: "received", updatedAt: new Date(Date.now() - 2 * 60_000), }) .returning(); expect(await service.listActivity(endpoint.id)).toEqual( expect.arrayContaining([ expect.objectContaining({ id: cancelAction!.id, kind: "action", status: "delivery_unknown", resolutionActions: ["cancel"], }), ]), ); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, cancelAction!.id)), ).resolves.toEqual([{ status: "received" }]); await request(app) .post( `/api/chat-endpoints/${endpoint.id}/actions/${cancelAction!.id}/resolve`, ) .send({ action: "mark_delivered" }) .expect(409); await request(app) .post( `/api/chat-endpoints/${endpoint.id}/actions/${cancelAction!.id}/resolve`, ) .send({ action: "cancel" }) .expect(204); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, cancelAction!.id)), ).resolves.toEqual([{ status: "cancelled" }]); await expect( db .select({ action: activityLog.action, actorId: activityLog.actorId }) .from(activityLog) .where(eq(activityLog.entityId, cancelAction!.id)), ).resolves.toEqual([ { action: "chat.slack_command_cancel", actorId: "owner-user" }, ]); }); it("binds Slack DM slash controls to the native root returned for the latest task", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const command = endpoint.setup.command; if (!command) throw new Error("Slack endpoint did not expose its command"); if (!callbacks.onSlashCommand) { throw new Error("Slack slash command callback was not registered"); } const starterThreadId = "slack:D09SLASHTASK:outbound-1"; const post = vi.fn(async () => ({ id: "6100.1", threadId: starterThreadId, })); const slashChannel = { id: "slack:D09SLASHTASK", name: "direct message", isDM: false, post, postEphemeral: vi.fn(), } as never; const invoke = async (text: string) => callbacks.onSlashCommand!({ endpointId: endpoint.id, provider: "slack", event: { channel: slashChannel, command, text, triggerId: `trigger-dm-slash-${text}-${randomUUID()}`, user: { userId: "U-DM-SLASH-TASK", userName: "dm-slash-task", fullName: "DM Slash Task User", isBot: false, isMe: false, isSystem: false, }, raw: { command, text }, adapter: {} as never, openModal: async () => undefined, }, }); await invoke("Start a task from this DM slash command"); await vi.waitFor(() => expect(runtime.endpoints.get(endpoint.id)?.posts).toContainEqual({ threadId: "slack:D09SLASHTASK:", text: "Starting a task…", }), ); expect(post).not.toHaveBeenCalled(); await vi.waitFor(async () => expect(await service.listConversations(endpoint.id)).toHaveLength(1), ); const [conversation] = await service.listConversations(endpoint.id); expect(conversation.externalThreadId).toBe(starterThreadId); await invoke("status"); await service.processPendingPublications(); expect(runtime.endpoints.get(endpoint.id)?.posts.at(-1)).toMatchObject({ threadId: starterThreadId, text: expect.stringMatching(/— todo$/), }); await invoke("close"); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ externalThreadId: starterThreadId, state: "active", }), ]); await service.processPendingPublications(); expect(await service.listConversations(endpoint.id)).toEqual([ expect.objectContaining({ externalThreadId: starterThreadId, state: "completed", }), ]); expect( await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toHaveLength(1); }); it("acknowledges a Slack retry while one durable worker owns the unrecorded command root", async () => { const fixture = await seedCompany(); const scheduledWork: Array<() => void> = []; const { callbacks, endpoint, runtime, service, wakeup } = await configuredSlackEndpoint(fixture, { // Transport and Paperclip admission are separate durable phases. // Run transport explicitly, then hold the delivery drain until reach // revocation has committed instead of racing their query scheduling. deferWebhookProcessing: true, scheduleDeferredWork: (task) => scheduledWork.push(task), }); const command = endpoint.setup.command; if (!command) throw new Error("Slack endpoint did not expose its command"); if (!callbacks.onSlashCommand) { throw new Error("Slack slash command callback was not registered"); } const [resource] = await db .insert(chatEndpointResources) .values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: "C-COMMAND-ORPHAN", label: "command-orphan", availability: "available", enabled: true, }) .returning(); let releasePost!: () => void; const post = vi.fn(); const slashEvent = { endpointId: endpoint.id, provider: "slack" as const, event: { channel: { id: "C-COMMAND-ORPHAN", name: "command-orphan", isDM: false, post, postEphemeral: vi.fn(async () => ({ id: "unexpected-ephemeral", threadId: "slack:C-COMMAND-ORPHAN:root", })), } as never, command, text: "do not acknowledge an unrecorded root", triggerId: "trigger-command-orphan", user: { userId: "U-COMMAND-ORPHAN", userName: "command-orphan", fullName: "Command Orphan User", isBot: false, isMe: false, isSystem: false, }, raw: { trigger_id: "trigger-command-orphan" }, adapter: {} as never, openModal: async () => undefined, }, }; const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack provider runtime"); const postEntered = vi.fn(); providerRuntime.postHook = async () => { postEntered(); await new Promise((resolve) => { releasePost = resolve; }); }; const owner = callbacks.onSlashCommand(slashEvent); await owner; expect(scheduledWork).toHaveLength(1); scheduledWork.shift()!(); await vi.waitFor(() => expect(postEntered).toHaveBeenCalledTimes(1)); await expect( Promise.race([ callbacks .onSlashCommand(slashEvent) .then(() => "acknowledged" as const), new Promise<"timed_out">((resolve) => setTimeout(() => resolve("timed_out"), 250), ), ]), ).resolves.toBe("acknowledged"); // The duplicate's worker must converge on the resolving action without // starting a second provider root while the first request remains held. expect(scheduledWork).toHaveLength(1); scheduledWork.shift()!(); const taskStart = await db .select({ id: chatActions.id, principalId: chatActions.principalId, status: chatActions.status, }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slash_task_start"), ), ) .then((rows) => rows[0] ?? null); expect(taskStart).toMatchObject({ status: "resolving" }); if (!taskStart?.principalId) { throw new Error("Expected the claimed Slack task-start principal"); } const authorizationLockProbe = db.transaction(async (tx) => { await tx.execute( sql`select pg_advisory_xact_lock(hashtextextended(${`chat-identity:${fixture.companyId}:${taskStart.principalId}`}, 0))`, ); return "authorization-lock-released" as const; }); const lockOutcome = await Promise.race([ authorizationLockProbe, new Promise<"authorization-lock-held">((resolve) => setTimeout(() => resolve("authorization-lock-held"), 500), ), ]); if (lockOutcome !== "authorization-lock-released") { releasePost(); await owner; await authorizationLockProbe; } expect(lockOutcome).toBe("authorization-lock-released"); let revocationSettled = false; const revokeReach = service .replaceResources(endpoint.id, [{ id: resource!.id, enabled: false }]) .then(() => { revocationSettled = true; }); await new Promise((resolve) => setTimeout(resolve, 25)); expect(revocationSettled).toBe(false); await expect( db .select({ enabled: chatEndpointResources.enabled }) .from(chatEndpointResources) .where(eq(chatEndpointResources.id, resource!.id)), ).resolves.toEqual([{ enabled: true }]); expect(postEntered).toHaveBeenCalledTimes(1); releasePost(); await revokeReach; expect(revocationSettled).toBe(true); providerRuntime.postHook = undefined; expect(post).not.toHaveBeenCalled(); await vi.waitFor(async () => { const [action] = await db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, taskStart.id)); expect(action?.status).toBe("processed"); }); const [stagedDelivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(stagedDelivery).toMatchObject({ state: "received" }); if (!stagedDelivery) throw new Error("Expected the durable Slack starter receipt"); expect(stagedDelivery.normalizedEvent).toMatchObject({ admission: { origin: "provider_confirmed_action" }, }); expect(wakeup).not.toHaveBeenCalled(); await expect(service.listConversations(endpoint.id)).resolves.toEqual([]); await expect( db .select({ enabled: chatEndpointResources.enabled }) .from(chatEndpointResources) .where(eq(chatEndpointResources.id, resource!.id)), ).resolves.toEqual([{ enabled: false }]); // Expire only this receipt's reorder window, then drain after revocation. // Provider acceptance is not a grant to create a Paperclip task later. await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, stagedDelivery.id)); await service.processPendingDeliveries(1, stagedDelivery.id); await expect( db .select({ state: chatDeliveries.state, principalId: chatDeliveries.principalId, redactedError: chatDeliveries.redactedError, normalizedEvent: chatDeliveries.normalizedEvent, }) .from(chatDeliveries) .where(eq(chatDeliveries.id, stagedDelivery.id)), ).resolves.toEqual([ expect.objectContaining({ state: "filtered", principalId: null, redactedError: "Destination is not enabled in Paperclip", normalizedEvent: expect.objectContaining({ filtering: { contentRetained: false }, }), }), ]); const [filteredDelivery] = await db .select({ normalizedEvent: chatDeliveries.normalizedEvent }) .from(chatDeliveries) .where(eq(chatDeliveries.id, stagedDelivery.id)); expect(JSON.stringify(filteredDelivery?.normalizedEvent)).not.toContain( slashEvent.event.text, ); await expect(service.listConversations(endpoint.id)).resolves.toEqual([]); await expect( db .select({ id: issues.id }) .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).resolves.toEqual([]); expect(wakeup).not.toHaveBeenCalled(); expect(providerRuntime.posts).toEqual([ { threadId: "slack:C-COMMAND-ORPHAN:", text: "Starting a task…", }, ]); await service.shutdown(); }); it("treats D-prefixed Slack status, new, and close callbacks as DM task controls even when the SDK flag is false", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); if (!callbacks.onSlashCommand) { throw new Error("Slack slash command callback was not registered"); } const dm = makeThread({ channelId: "D09CONTROLS", id: "slack:D09CONTROLS:", isDM: true, name: "direct message", }); const firstProviderTimestamp = `${Math.floor(Date.now() / 1_000) - 5}.000001`; await deliverMessage({ callbacks, endpointId: endpoint.id, thread: dm.thread, message: makeMessage({ id: firstProviderTimestamp, raw: { ts: firstProviderTimestamp }, text: "Start the first DM task", userId: "U-DM-CONTROLS", }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, "U-DM-CONTROLS"); await service.test(endpoint.id, "owner-user"); const command = endpoint.setup.command; if (!command) throw new Error("Slack endpoint did not expose its command"); const post = vi.fn(async () => ({ id: "unexpected-control-root", threadId: dm.thread.id, })); const postEphemeral = vi.fn(async () => ({ id: "unexpected-control-ephemeral", threadId: dm.thread.id, })); const slashChannel = { id: "slack:D09CONTROLS", name: "direct message", // Match the real Slack SDK slash callback observed during qualification: // the signed provider id is a DM even though this convenience flag is // false. isDM: false, post, postEphemeral, } as never; const invokeControl = async (control: "status" | "new" | "close") => { const reactionCount = runtime.endpoints.get(endpoint.id)?.reactions.length ?? 0; await callbacks.onSlashCommand!({ endpointId: endpoint.id, provider: "slack", event: { channel: slashChannel, command, text: control, triggerId: `trigger-${control}-${randomUUID()}`, user: { userId: "U-DM-CONTROLS", userName: "dm-controller", fullName: "DM Controller", isBot: false, isMe: false, isSystem: false, }, raw: { command, text: control }, adapter: {} as never, openModal: async () => undefined, }, }); expect(runtime.endpoints.get(endpoint.id)?.reactions).toHaveLength( reactionCount, ); }; const controlPostOffset = runtime.endpoints.get(endpoint.id)?.posts.length ?? 0; const issuesBeforeStatus = await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)); const firstIssue = issuesBeforeStatus.find( (issue) => issue.title === "Start the first DM task", ); if (!firstIssue) throw new Error("First Slack DM task was not created"); await db .update(chatConversations) .set({ providerUrl: null }) .where(eq(chatConversations.endpointId, endpoint.id)); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ externalUrl: "https://slack.com/app_redirect?channel=D09CONTROLS&team=T-PAPERCLIP", }), ]); await invokeControl("status"); expect( await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toHaveLength(issuesBeforeStatus.length); await service.processPendingPublications(); expect(runtime.endpoints.get(endpoint.id)?.posts.at(-1)?.text).toMatch( /— todo$/, ); await invokeControl("new"); expect( await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toHaveLength(issuesBeforeStatus.length); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ state: "active" }), ]); await service.processPendingPublications(); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ state: "completed" }), ]); const secondProviderTime = Date.now(); const secondProviderTimestamp = `${Math.floor(secondProviderTime / 1_000)}.${String((secondProviderTime % 1_000) * 1_000).padStart(6, "0")}`; await deliverMessage({ callbacks, endpointId: endpoint.id, thread: dm.thread, message: makeMessage({ id: secondProviderTimestamp, raw: { ts: secondProviderTimestamp }, text: "Start the second DM task", userId: "U-DM-CONTROLS", }), trigger: "direct_message", }); const issuesBeforeClose = await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)); expect(issuesBeforeClose).toHaveLength(issuesBeforeStatus.length + 1); await invokeControl("close"); expect( await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toHaveLength(issuesBeforeClose.length); expect( (await service.listConversations(endpoint.id)).map( (conversation) => conversation.state, ), ).toContain("active"); await service.processPendingPublications(); expect( (await service.listConversations(endpoint.id)).map( (conversation) => conversation.state, ), ).toEqual(["completed", "completed"]); expect( await db .select({ id: issues.id, status: issues.status }) .from(issues) .where(eq(issues.companyId, fixture.companyId)) .orderBy(asc(issues.id)), ).toEqual( issuesBeforeClose .map(({ id, status }) => ({ id, status })) .sort((left, right) => left.id.localeCompare(right.id)), ); const controlAuthorizations = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "task_control_authorization"), ), ); expect(controlAuthorizations).toHaveLength(3); expect( controlAuthorizations.map((authorization) => ({ kind: authorization.kind, status: authorization.status, })), ).toEqual([ { kind: "task_control_authorization", status: "processed" }, { kind: "task_control_authorization", status: "processed" }, { kind: "task_control_authorization", status: "processed" }, ]); expect( runtime.endpoints .get(endpoint.id) ?.posts.slice(controlPostOffset) .map((providerPost) => ({ threadId: providerPost.threadId, text: providerPost.text, })), ).toEqual([ { threadId: dm.thread.id, text: `${firstIssue.identifier}: Start the first DM task — todo`, }, { threadId: dm.thread.id, text: "Send your request to start a new Paperclip task.", }, { threadId: dm.thread.id, text: "This chat conversation is closed. Send another message to start a new task.", }, ]); expect( ( await db .select() .from(chatPublications) .where(eq(chatPublications.endpointId, endpoint.id)) .orderBy(asc(chatPublications.createdAt), asc(chatPublications.id)) ) .filter((publication) => publication.idempotencyKey.startsWith("control:"), ) .map((publication) => publication.idempotencyKey.split(":")[1]), ).toEqual(["status", "new", "close"]); expect(post).not.toHaveBeenCalled(); expect(postEphemeral).not.toHaveBeenCalled(); // Each slash control asserted above that it added no reaction. The second // real DM task message is still free to receive its normal acknowledgement. const controlDeliveries = ( await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)) ).filter((delivery) => { const normalized = delivery.normalizedEvent as { message?: { text?: unknown }; }; return ["/status", "/new", "/close"].includes( String(normalized.message?.text), ); }); expect(controlDeliveries).toHaveLength(3); expect(controlDeliveries).toEqual( expect.arrayContaining( ["/status", "/new", "/close"].map((text) => expect.objectContaining({ state: "processed", redactedError: null, normalizedEvent: expect.objectContaining({ acknowledgement: { receiptReactionSupported: false }, message: expect.objectContaining({ text }), }), }), ), ), ); }); it("returns ephemeral guidance for exact Slack controls in channels without creating tasks or actions", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service } = await configuredSlackEndpoint(fixture); if (!callbacks.onSlashCommand) { throw new Error("Slack slash command callback was not registered"); } await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: "C-CONTROL-GUIDANCE", label: "control-guidance", availability: "available", enabled: true, }); const command = endpoint.setup.command; if (!command) throw new Error("Slack endpoint did not expose its command"); const post = vi.fn(async () => ({ id: "unexpected-channel-control-post", threadId: "slack:C-CONTROL-GUIDANCE:root", })); const postEphemeral = vi.fn(async () => ({ id: "channel-control-guidance", threadId: "slack:C-CONTROL-GUIDANCE:root", })); for (const control of ["status", "new", "close"] as const) { await callbacks.onSlashCommand({ endpointId: endpoint.id, provider: "slack", event: { channel: { id: "slack:C-CONTROL-GUIDANCE", name: "control-guidance", isDM: false, post, postEphemeral, } as never, command, text: control, triggerId: `channel-control-${control}`, user: { userId: "U-CHANNEL-CONTROLLER", userName: "channel-controller", fullName: "Channel Controller", isBot: false, isMe: false, isSystem: false, }, raw: { command, text: control }, adapter: {} as never, openModal: async () => undefined, }, }); } expect(post).not.toHaveBeenCalled(); await vi.waitFor(() => expect(postEphemeral).toHaveBeenCalledTimes(3)); for (const call of postEphemeral.mock.calls) { expect(call[1]).toBe( "Use status, new, and close in a direct message with this agent. In a channel, open the Paperclip task from its Slack thread.", ); expect(call[2]).toEqual({ fallbackToDM: false }); } expect(await service.listConversations(endpoint.id)).toEqual([]); expect( await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toHaveLength(0); // The ephemeral post is observable before its provider_effect row // settles, so under suite load the third row can still be // mid-settlement when the mock resolves (drew a not-yet-processed row // in CI on 2026-09-10). Wait for the bookkeeping, bounded, like the // durable-receipt paths above do. await vi.waitFor(async () => { expect( await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "provider_effect"), ), ), ).toEqual([ expect.objectContaining({ kind: "provider_effect", status: "processed" }), expect.objectContaining({ kind: "provider_effect", status: "processed" }), expect.objectContaining({ kind: "provider_effect", status: "processed" }), ]); }); }); it("keeps Telegram start and unknown commands as terse guidance without creating work", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredTelegramEndpoint(fixture); if (!callbacks.onSlashCommand) { throw new Error("Telegram slash command callback was not registered"); } const channel = { id: "telegram:77112233", name: "Telegram direct message", isDM: true, } as never; const invoke = async (command: string, text: string, messageId: number) => { await callbacks.onSlashCommand!({ endpointId: endpoint.id, provider: "telegram", event: { channel, command, text, user: { userId: "77112233", userName: "telegram-user", fullName: "Telegram User", isBot: false, isMe: false, isSystem: false, }, raw: { message_id: messageId, date: 1_788_620_500 + messageId, chat: { id: 77112233, type: "private" }, from: { id: 77112233, is_bot: false }, text: `${command} ${text}`.trim(), entities: [ { offset: 0, length: command.length, type: "bot_command" }, ], }, adapter: {} as never, openModal: async () => undefined, }, }); }; await invoke("/start", "ignored payload", 501); await invoke("/danger", "create an administrator action", 502); expect( runtime.endpoints.get(endpoint.id)?.posts.map((post) => post.text), ).toEqual([ expect.stringMatching( /^Send a direct message to start work with Maya\. In a group, use \/task@paperclip_\d+_bot followed by your request\./, ), expect.stringMatching( /^Available commands: \/task@paperclip_\d+_bot followed by your request, \/status, \/new, and \/close\.$/, ), ]); expect(wakeup).not.toHaveBeenCalled(); expect(await service.listConversations(endpoint.id)).toEqual([]); expect( await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toHaveLength(0); expect( await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "provider_effect"), ), ), ).toEqual([ expect.objectContaining({ kind: "provider_effect", status: "processed" }), expect.objectContaining({ kind: "provider_effect", status: "processed" }), ]); const deliveries = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(deliveries).toHaveLength(2); expect(deliveries).toEqual( expect.arrayContaining([ expect.objectContaining({ providerEventId: "telegram:77112233:77112233:501", state: "processed", conversationId: null, }), expect.objectContaining({ providerEventId: "telegram:77112233:77112233:502", state: "processed", conversationId: null, }), ]), ); }); it("does not commit a Telegram close control when its provider reply is definitively rejected", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture); if (!callbacks.onSlashCommand) { throw new Error("Telegram slash command callback was not registered"); } const dm = makeThread({ channelId: "78112233", id: "telegram:78112233", isDM: true, name: "Telegram direct message", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: "78112233:1", raw: { message_id: 1 }, text: "Start a task before close", userId: "78112233", }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, "78112233"); await service.test(endpoint.id, "owner-user"); await db .update(chatConversations) .set({ state: "completed", updatedAt: new Date() }) .where(eq(chatConversations.endpointId, endpoint.id)); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Telegram runtime"); providerRuntime.postError = Object.assign( new Error("Telegram rejected the control reply"), { name: "ValidationError", code: "VALIDATION_ERROR", }, ); const closeEvent = { endpointId: endpoint.id, provider: "telegram" as const, event: { channel: { id: "telegram:78112233", name: "Telegram direct message", isDM: true, } as never, command: "/close", text: "", user: { userId: "78112233", userName: "telegram-user", fullName: "Telegram User", isBot: false, isMe: false, isSystem: false, }, raw: { message_id: 800, date: 1_788_622_800, chat: { id: 78112233, type: "private" }, from: { id: 78112233, is_bot: false }, text: "/close", entities: [{ offset: 0, length: 6, type: "bot_command" }], }, adapter: {} as never, openModal: async () => undefined, }, }; await expect(callbacks.onSlashCommand(closeEvent)).rejects.toThrow( "Telegram rejected the control reply", ); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ state: "completed" }), ]); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "provider_effect"), ), ), ).resolves.toEqual([{ status: "failed" }]); await expect( db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, "telegram:78112233:78112233:800", ), ), ), ).resolves.toEqual([{ state: "failed" }]); expect(providerRuntime.posts).toHaveLength(0); await expect(callbacks.onSlashCommand(closeEvent)).resolves.toBeUndefined(); expect(providerRuntime.posts).toHaveLength(0); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ state: "completed" }), ]); }); it("quarantines a Telegram control when the reply lands but its state commit crashes", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture); if (!callbacks.onSlashCommand) { throw new Error("Telegram slash command callback was not registered"); } const dm = makeThread({ channelId: "79112233", id: "telegram:79112233", isDM: true, name: "Telegram direct message", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: "79112233:1", raw: { message_id: 1 }, text: "Start a task before ambiguous close", userId: "79112233", }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, "79112233"); await service.test(endpoint.id, "owner-user"); await db .update(chatConversations) .set({ state: "completed", updatedAt: new Date() }) .where(eq(chatConversations.endpointId, endpoint.id)); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Telegram runtime"); const closeEvent = { endpointId: endpoint.id, provider: "telegram" as const, event: { channel: { id: "telegram:79112233", name: "Telegram direct message", isDM: true, } as never, command: "/close", text: "", user: { userId: "79112233", userName: "telegram-user", fullName: "Telegram User", isBot: false, isMe: false, isSystem: false, }, raw: { message_id: 801, date: 1_788_622_801, chat: { id: 79112233, type: "private" }, from: { id: 79112233, is_bot: false }, text: "/close", entities: [{ offset: 0, length: 6, type: "bot_command" }], }, adapter: {} as never, openModal: async () => undefined, }, }; const originalTransaction = db.transaction.bind(db); let injectedCrash = false; const transactionSpy = vi .spyOn(db, "transaction") .mockImplementation((async ( ...args: Parameters ) => { const [callback, config] = args; return originalTransaction(async (tx) => { const result = await callback(tx); if (!injectedCrash && providerRuntime.posts.length === 1) { injectedCrash = true; throw new Error("injected provider-effect commit crash"); } return result; }, config); }) as typeof db.transaction); try { await expect(callbacks.onSlashCommand(closeEvent)).rejects.toThrow( "injected provider-effect commit crash", ); } finally { transactionSpy.mockRestore(); } expect(injectedCrash).toBe(true); expect(providerRuntime.posts).toHaveLength(1); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ state: "completed" }), ]); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "provider_effect"), ), ), ).resolves.toEqual([{ status: "delivery_unknown" }]); await expect( db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.providerEventId, "telegram:79112233:79112233:801", ), ), ), ).resolves.toEqual([{ state: "failed" }]); await expect(callbacks.onSlashCommand(closeEvent)).resolves.toBeUndefined(); await service.processPendingProviderEffects(); expect(providerRuntime.posts).toHaveLength(1); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ state: "completed" }), ]); }); it("resolves ambiguous provider effects explicitly and never lets terminal failures starve reconciliation", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture); if (!callbacks.onSlashCommand) { throw new Error("Telegram slash command callback was not registered"); } const dm = makeThread({ channelId: "71112233", id: "telegram:71112233", isDM: true, name: "Telegram direct message", }); await callbacks.onSlashCommand({ endpointId: endpoint.id, provider: "telegram", event: { channel: dm.thread as never, command: "/start", text: "", user: { userId: "71112233", userName: "telegram-user", fullName: "Telegram User", isBot: false, isMe: false, isSystem: false, }, raw: { message_id: 901, date: 1_788_622_901, chat: { id: 71112233, type: "private" }, from: { id: 71112233, is_bot: false }, text: "/start", entities: [{ offset: 0, length: 6, type: "bot_command" }], }, adapter: {} as never, openModal: async () => undefined, }, }); const seedEffect = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "provider_effect"), ), ) .then((rows) => rows[0]); if (!seedEffect) throw new Error("Expected a provider-effect fixture"); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: "71112233:902", raw: { message_id: 902 }, text: "Create a task for provider-effect recovery", userId: "71112233", }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, "71112233"); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); if (!conversation) throw new Error("Expected a bound conversation"); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Telegram runtime"); const initialPostCount = providerRuntime.posts.length; const app = routesApp(db, fixture.companyId, service); const insertAmbiguousEffect = async (suffix: string, text: string) => { const [delivery] = await db .insert(chatDeliveries) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, providerEventId: `provider-effect-resolution:${suffix}`, deduplicationKey: `provider-effect-resolution:${suffix}`, eventKind: "message", normalizedEvent: {}, state: "failed", redactedError: "Provider response is unconfirmed", }) .returning(); const [action] = await db .insert(chatActions) .values({ companyId: fixture.companyId, endpointId: endpoint.id, deliveryId: delivery!.id, conversationId: conversation.id, principalId: seedEffect.principalId, kind: "provider_effect", providerActionId: `provider_effect:resolution:${suffix}`, payload: { ...seedEffect.payload, text, settleDelivery: true, completeConversationId: conversation.id, }, status: "delivery_unknown", result: { code: "provider_effect_delivery_unknown", attempts: 1, retryable: false, }, }) .returning(); return { action: action!, delivery: delivery! }; }; const marked = await insertAmbiguousEffect("mark", "Close after mark"); const activity = await service.listActivity(endpoint.id); expect(activity).toEqual( expect.arrayContaining([ expect.objectContaining({ id: marked.action.id, kind: "action", actionType: "provider_effect", status: "delivery_unknown", replayable: false, resolutionActions: ["mark_delivered", "retry_anyway", "cancel"], }), expect.objectContaining({ id: marked.delivery.id, kind: "delivery", status: "failed", replayable: false, }), ]), ); await request(app) .post( `/api/chat-endpoints/${endpoint.id}/deliveries/${marked.delivery.id}/replay`, ) .send({}) .expect(409); await request(app) .post( `/api/chat-endpoints/${endpoint.id}/actions/${marked.action.id}/resolve`, ) .send({ action: "mark_delivered" }) .expect(204); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, marked.action.id)), ).resolves.toEqual([{ status: "processed" }]); await expect( db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where(eq(chatDeliveries.id, marked.delivery.id)), ).resolves.toEqual([{ state: "processed" }]); await expect( db .select({ state: chatConversations.state }) .from(chatConversations) .where(eq(chatConversations.id, conversation.id)), ).resolves.toEqual([{ state: "completed" }]); expect(providerRuntime.posts).toHaveLength(initialPostCount); await db .update(chatConversations) .set({ state: "active", updatedAt: new Date() }) .where(eq(chatConversations.id, conversation.id)); const retried = await insertAmbiguousEffect("retry", "Close after retry"); const retryResponses = await Promise.all([ request(app) .post( `/api/chat-endpoints/${endpoint.id}/actions/${retried.action.id}/resolve`, ) .send({ action: "retry_anyway" }), request(app) .post( `/api/chat-endpoints/${endpoint.id}/actions/${retried.action.id}/resolve`, ) .send({ action: "retry_anyway" }), ]); expect(retryResponses.map((response) => response.status).sort()).toEqual([ 204, 409, ]); expect(providerRuntime.posts).toHaveLength(initialPostCount + 1); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, retried.action.id)), ).resolves.toEqual([{ status: "processed" }]); await expect( db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where(eq(chatDeliveries.id, retried.delivery.id)), ).resolves.toEqual([{ state: "processed" }]); await db .update(chatConversations) .set({ state: "active", updatedAt: new Date() }) .where(eq(chatConversations.id, conversation.id)); const cancelled = await insertAmbiguousEffect( "cancel", "Do not send this reply", ); await request(app) .post( `/api/chat-endpoints/${endpoint.id}/actions/${cancelled.action.id}/resolve`, ) .send({ action: "cancel" }) .expect(204); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, cancelled.action.id)), ).resolves.toEqual([{ status: "cancelled" }]); await expect( db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where(eq(chatDeliveries.id, cancelled.delivery.id)), ).resolves.toEqual([{ state: "filtered" }]); await expect( db .select({ state: chatConversations.state }) .from(chatConversations) .where(eq(chatConversations.id, conversation.id)), ).resolves.toEqual([{ state: "active" }]); expect(providerRuntime.posts).toHaveLength(initialPostCount + 1); const terminalCreatedAt = new Date("2026-01-01T00:00:00.000Z"); await db.insert(chatActions).values( Array.from({ length: 25 }, (_, index) => ({ companyId: fixture.companyId, endpointId: endpoint.id, kind: "provider_effect", providerActionId: `provider_effect:terminal:${randomUUID()}`, payload: seedEffect.payload, status: "failed", result: { retryable: false }, createdAt: new Date(terminalCreatedAt.getTime() + index), updatedAt: terminalCreatedAt, })), ); const [actionable] = await db .insert(chatActions) .values({ companyId: fixture.companyId, endpointId: endpoint.id, principalId: seedEffect.principalId, kind: "provider_effect", providerActionId: `provider_effect:actionable:${randomUUID()}`, payload: { ...seedEffect.payload, text: "This newer actionable effect must not starve", settleDelivery: false, completeConversationId: undefined, }, status: "received", }) .returning(); await service.processPendingProviderEffects(1); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, actionable!.id)), ).resolves.toEqual([{ status: "processed" }]); await expect( db .select({ action: activityLog.action, actorId: activityLog.actorId }) .from(activityLog) .where( inArray(activityLog.entityId, [ marked.action.id, retried.action.id, cancelled.action.id, ]), ), ).resolves.toEqual( expect.arrayContaining([ { action: "chat.provider_effect_mark_delivered", actorId: "owner-user", }, { action: "chat.provider_effect_retry_anyway", actorId: "owner-user", }, { action: "chat.provider_effect_cancel", actorId: "owner-user", }, ]), ); }); it("acknowledges a denied Slack action before its durable notice completes and deduplicates redelivery", async () => { const scheduled: Array<() => void> = []; const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture, { scheduleDeferredWork: (task) => scheduled.push(task), }); if (!callbacks.onAction) { throw new Error("Slack action callback was not registered"); } const channel = makeThread({ channelId: "C-DENIAL-OUTBOX", id: "slack:C-DENIAL-OUTBOX:9000.1", name: "denial-outbox", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "9000.1", text: "@maya create a denial fixture", mentioned: true, userId: "U-DENIAL-OUTBOX", }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id, "U-DENIAL-OUTBOX"); await service.test(endpoint.id, "owner-user"); // Setup publishes its own final and schedules exact-source eyes cleanup. // Settle and identify that work before measuring the denied-action queue. const setupRemovals = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "receipt_reaction"), sql`${chatActions.payload}->>'operation' = 'remove'`, ), ); expect(setupRemovals).toHaveLength(1); const setupRemoval = setupRemovals[0]!; expect(setupRemoval).toMatchObject({ providerActionId: `receipt_reaction_remove:${setupRemoval.deliveryId}`, status: "received", payload: { operation: "remove", reaction: "eyes", threadId: channel.thread.id, messageId: expect.stringMatching(/^setup-follow-up-/), }, }); expect(scheduled).toHaveLength(1); expect(channel.postEphemeral).not.toHaveBeenCalled(); scheduled.shift()!(); await vi.waitFor(async () => { const removal = await db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, setupRemoval.id)); expect(removal).toEqual([{ status: "processed" }]); }); const providerRuntime = runtime.endpoints.get(endpoint.id)!; expect(providerRuntime.removedReactions).toEqual([ { threadId: channel.thread.id, messageId: setupRemoval.payload.messageId, emoji: "eyes", }, ]); expect(scheduled).toHaveLength(0); expect(channel.postEphemeral).not.toHaveBeenCalled(); let releaseNotice!: () => void; const noticeRelease = new Promise((resolve) => { releaseNotice = resolve; }); channel.postEphemeral.mockImplementation(async () => { await noticeRelease; return { id: "ephemeral-denial-outbox", threadId: channel.thread.id, usedFallback: false, }; }); const deniedAction = { endpointId: endpoint.id, provider: "slack" as const, event: { actionId: "pcq:unissued-denial-outbox", adapter: {} as never, messageId: "outbound-unissued", openModal: async () => undefined, raw: { type: "block_actions" }, thread: channel.thread, threadId: channel.thread.id, user: { userId: "U-DENIAL-OUTBOX", userName: "denial-user", fullName: "Denial User", isBot: false, isMe: false, isSystem: false, }, value: "unissued", }, }; await expect(callbacks.onAction(deniedAction)).resolves.toBeUndefined(); expect(channel.postEphemeral).not.toHaveBeenCalled(); expect(scheduled).toHaveLength(1); scheduled.shift()!(); await vi.waitFor(() => expect(channel.postEphemeral).toHaveBeenCalledTimes(1), ); await expect(callbacks.onAction(deniedAction)).resolves.toBeUndefined(); expect(scheduled).toHaveLength(0); expect(channel.postEphemeral).toHaveBeenCalledTimes(1); expect(providerRuntime.removedReactions).toHaveLength(1); releaseNotice(); await retirePublicationFixture(service, endpoint.id); await expect( db .select({ kind: chatActions.kind, status: chatActions.status }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "provider_effect"), ), ), ).resolves.toEqual([ expect.objectContaining({ kind: "provider_effect", status: "processed" }), ]); await expect( db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "action"), ), ), ).resolves.toEqual([{ state: "filtered" }]); }); it("queues Telegram guidance on the durable publication FIFO when a task is active", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredTelegramEndpoint(fixture); if (!callbacks.onSlashCommand) { throw new Error("Telegram slash command callback was not registered"); } const directMessage = makeThread({ channelId: "77112233", id: "telegram:77112233", isDM: true, name: "Telegram direct message", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: directMessage.thread, message: makeMessage({ id: "77112233:700", raw: { message_id: 700, date: 1_788_621_200, chat: { id: 77112233, type: "private" }, }, text: "investigate the durable guidance lane", userId: "77112233", }), trigger: "direct_message", }); const slashEvent = { endpointId: endpoint.id, provider: "telegram" as const, event: { channel: { id: "telegram:77112233", name: "Telegram direct message", isDM: true, } as never, command: "/start", text: "", user: { userId: "77112233", userName: "telegram-user", fullName: "Telegram User", isBot: false, isMe: false, isSystem: false, }, raw: { message_id: 701, date: 1_788_621_201, chat: { id: 77112233, type: "private" }, from: { id: 77112233, is_bot: false }, text: "/start", entities: [{ offset: 0, length: 6, type: "bot_command" }], }, adapter: {} as never, openModal: async () => undefined, }, }; await callbacks.onSlashCommand(slashEvent); await callbacks.onSlashCommand(slashEvent); expect(runtime.endpoints.get(endpoint.id)?.posts).toEqual([]); const publicationsBeforeDrain = await db .select() .from(chatPublications) .where(eq(chatPublications.endpointId, endpoint.id)); expect(publicationsBeforeDrain).toEqual([ expect.objectContaining({ state: "pending", idempotencyKey: expect.stringMatching(/^control:guidance:/), payload: expect.objectContaining({ text: expect.stringMatching( /^Send a direct message to start work with Maya\. In a group, use \/task@paperclip_\d+_bot followed by your request\./, ), }), }), ]); await service.processPendingPublications(); await service.processPendingPublications(); expect( runtime.endpoints.get(endpoint.id)?.posts.map((post) => post.text), ).toEqual([ expect.stringMatching( /^Send a direct message to start work with Maya\. In a group, use \/task@paperclip_\d+_bot followed by your request\./, ), ]); expect(wakeup).toHaveBeenCalledTimes(1); expect( await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toHaveLength(1); expect( await db .select() .from(issueComments) .where(eq(issueComments.companyId, fixture.companyId)), ).toHaveLength(1); }); it("durably deduplicates Telegram guidance webhook retries and filters disabled DMs", async () => { const fixture = await seedCompany(); const botToken = "887766:telegram-guidance-webhook-test"; const botId = 887766; const apiCalls: Array<{ body: string; method: string }> = []; const wakeup = vi.fn(async () => ({ accepted: true })); let webhookSecret = ""; let nextProviderMessageId = 9_000; const telegramFetch = vi.fn( async (input: string | URL | Request, init?: RequestInit) => { const method = new URL(String(input)).pathname.split("/").at(-1) ?? ""; const body = typeof init?.body === "string" ? init.body : ""; apiCalls.push({ body, method }); if (method === "getMe") { return new Response( JSON.stringify({ ok: true, result: { id: botId, username: "paperclip_guidance_test_bot", first_name: "Paperclip Guidance Test", }, }), { status: 200, headers: { "content-type": "application/json" } }, ); } if (method === "getWebhookInfo") { return new Response( JSON.stringify({ ok: true, result: { url: "" } }), { status: 200, headers: { "content-type": "application/json" }, }, ); } if (method === "setWebhook") { webhookSecret = String( (JSON.parse(body) as { secret_token?: unknown }).secret_token ?? "", ); return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } if (method === "setMyCommands") { return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } if (method === "sendMessage") { const payload = JSON.parse(body) as { chat_id?: string; message_thread_id?: number; text?: string; }; nextProviderMessageId += 1; return new Response( JSON.stringify({ ok: true, result: { message_id: nextProviderMessageId, date: Math.floor(Date.now() / 1_000), chat: { id: Number(payload.chat_id), type: "private", first_name: "Telegram User", }, text: payload.text, }, }), { status: 200, headers: { "content-type": "application/json" } }, ); } if (method === "sendChatAction" || method === "setMessageReaction") { return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } throw new Error(`Unexpected Telegram API call: ${method}`); }, ) as typeof globalThis.fetch; const previousFetch = globalThis.fetch; globalThis.fetch = telegramFetch; const service = chatChannelService(db, { fetch: telegramFetch, heartbeat: { wakeup: receiptBackedWakeup(wakeup) }, publicBaseUrl: "https://paperclip.example", }); try { const endpoint = await service.create( fixture.companyId, { provider: "telegram", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await service.configure( endpoint.id, { action: "configure", credentials: { botToken } }, "owner-user", ); expect(webhookSecret).not.toBe(""); await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, endpoint.id)); const providerRequest = (input: { command: string; messageId: number; updateId: number; }) => new Request( `https://paperclip.example/api/chat-webhooks/${endpoint.publicId}/telegram`, { method: "POST", headers: { "content-type": "application/json", "x-telegram-bot-api-secret-token": webhookSecret, }, body: JSON.stringify({ update_id: input.updateId, message: { message_id: input.messageId, date: 1_788_620_500 + input.messageId, chat: { id: 417200359, type: "private", first_name: "Telegram User", }, from: { id: 417200359, is_bot: false, first_name: "Telegram User", username: "telegram-user", }, text: input.command, entities: [ { offset: 0, length: input.command.length, type: "bot_command", }, ], }, }), }, ); const deliverTwice = async (input: { command: string; messageId: number; updateId: number; }) => { for (let attempt = 0; attempt < 2; attempt += 1) { const response = await service.handleWebhook( endpoint.publicId, "telegram", providerRequest(input), ); expect(response.status).toBe(200); await expect(response.text()).resolves.toBe("OK"); } }; await deliverTwice({ command: "/start", messageId: 601, updateId: 7601 }); await deliverTwice({ command: "/danger", messageId: 602, updateId: 7602, }); expect( apiCalls .filter(({ method }) => method === "sendMessage") .map(({ body }) => (JSON.parse(body) as { text?: string }).text), ).toEqual([ "Send a direct message to start work with Maya. In a group, use /task@paperclip_guidance_test_bot followed by your request. Use /status, /new, or /close to manage the active task in this chat.", "Available commands: /task@paperclip_guidance_test_bot followed by your request, /status, /new, and /close.", ]); await service.update( endpoint.id, { allowDirectMessages: false }, "owner-user", ); await deliverTwice({ command: "/start", messageId: 603, updateId: 7603 }); expect( apiCalls.filter(({ method }) => method === "sendMessage"), ).toHaveLength(2); const deliveries = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(deliveries).toHaveLength(3); for (const messageId of [601, 602]) { expect(deliveries).toContainEqual( expect.objectContaining({ providerEventId: `telegram:417200359:417200359:${messageId}`, state: "processed", attempts: 1, conversationId: null, normalizedEvent: expect.objectContaining({ deduplication: expect.objectContaining({ duplicateCount: 1 }), }), }), ); } expect(deliveries).toContainEqual( expect.objectContaining({ providerEventId: "telegram:417200359:417200359:603", state: "filtered", attempts: 0, conversationId: null, redactedError: "Destination is not enabled in Paperclip", normalizedEvent: expect.objectContaining({ deduplication: expect.objectContaining({ duplicateCount: 1 }), }), }), ); expect(await service.listConversations(endpoint.id)).toEqual([]); expect( await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toHaveLength(0); expect(wakeup).not.toHaveBeenCalled(); } finally { await service.shutdown(); globalThis.fetch = previousFetch; } }); it("consumes Telegram forum-topic controls without remapping or waking the topic task", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredTelegramEndpoint(fixture); if (!callbacks.onSlashCommand) { throw new Error("Telegram slash command callback was not registered"); } const chatId = "-10077112233"; const topicId = 77; const topicThreadId = `telegram:${chatId}:${topicId}`; await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "chat", providerResourceId: chatId, label: "Production forum", availability: "available", enabled: true, }); const topic = makeThread({ channelId: chatId, id: topicThreadId, name: "Production forum", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: topic.thread, message: makeMessage({ id: `${chatId}:200`, raw: { message_id: 200, message_thread_id: topicId, chat: { id: Number(chatId), type: "supergroup" }, }, text: "@maya investigate the forum alert", mentioned: true, userId: "77112233", }), trigger: "mention", }); expect(wakeup).toHaveBeenCalledTimes(1); const [initialConversation] = await service.listConversations(endpoint.id); if (!initialConversation) throw new Error("Expected topic conversation"); const invoke = async ( command: "/status" | "/new" | "/close", messageId: number, ) => { await callbacks.onSlashCommand!({ endpointId: endpoint.id, provider: "telegram", event: { channel: { id: topicThreadId, name: "Production forum topic", isDM: false, post: vi.fn(), } as never, command, text: "", user: { userId: "77112233", userName: "telegram-forum-user", fullName: "Telegram Forum User", isBot: false, isMe: false, isSystem: false, }, raw: { message_id: messageId, message_thread_id: topicId, date: 1_788_620_500 + messageId, chat: { id: Number(chatId), type: "supergroup" }, from: { id: 77112233, is_bot: false }, text: command, entities: [ { offset: 0, length: command.length, type: "bot_command" }, ], }, adapter: {} as never, openModal: async () => undefined, }, }); }; await invoke("/status", 201); await service.processPendingPublications(); await invoke("/new", 202); await service.processPendingPublications(); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ id: initialConversation.id, issueId: initialConversation.issueId, state: "active", }), ]); await invoke("/close", 203); await service.processPendingPublications(); const conversations = await service.listConversations(endpoint.id); expect(conversations).toEqual([ expect.objectContaining({ id: initialConversation.id, issueId: initialConversation.issueId, state: "completed", }), ]); expect( await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toHaveLength(1); expect( await db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, initialConversation.issueId)), ).toEqual([{ body: "@maya investigate the forum alert" }]); expect(wakeup).toHaveBeenCalledTimes(1); expect( runtime.endpoints.get(endpoint.id)?.posts.map((post) => post.text), ).toEqual([ expect.stringMatching(/— todo$/), expect.stringContaining( "Open a new Telegram forum topic to start a new Paperclip task.", ), "This chat conversation is closed. A later message here will continue the same Paperclip task.", ]); }); it("keeps successive Telegram DM messages on one active Paperclip task", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service, wakeup } = await configuredTelegramEndpoint(fixture); const dm = makeThread({ channelId: "77112233", id: "telegram:77112233", isDM: true, name: "Telegram direct message", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: "77112233:11", text: "Start one DM task", userId: "77112233", }), trigger: "direct_message", }); expect(wakeup).toHaveBeenCalledTimes(1); await qualifySetupRoundTrip(service, endpoint.id, "77112233"); await service.test(endpoint.id, "owner-user"); wakeup.mockClear(); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: "77112233:12", text: "Continue that same DM task", userId: "77112233", }), trigger: "direct_message", }); const conversations = await service.listConversations(endpoint.id); expect(conversations).toEqual([ expect.objectContaining({ state: "active", sessionGeneration: 1 }), ]); expect( await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toHaveLength(1); expect( ( await db .select() .from(issueComments) .where(eq(issueComments.issueId, conversations[0]!.issueId)) ).map((comment) => comment.body), ).toEqual( expect.arrayContaining([ "Start one DM task", "Continue that same DM task", ]), ); expect(wakeup).toHaveBeenCalledTimes(1); }); it("fails closed for unbound rich callbacks and unknown slash commands", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service } = await configuredSlackEndpoint(fixture); const command = endpoint.setup.command; if (!command) throw new Error("Slack endpoint did not expose its command"); expect(callbacks.onAction).toBeTypeOf("function"); expect(callbacks.onModalSubmit).toBeTypeOf("function"); expect(callbacks.onModalClose).toBeUndefined(); expect(callbacks.onOptionsLoad).toBeUndefined(); expect(callbacks.onReaction).toBeTypeOf("function"); if (!callbacks.onSlashCommand) throw new Error("Slack slash command callback was not registered"); await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: "C-COMMAND-DENY", label: "denied commands", availability: "available", enabled: true, }); const post = vi.fn(async () => ({ id: "unexpected-post", threadId: "slack:C-COMMAND-DENY:6100.1", })); const postEphemeral = vi.fn(async () => ({ id: "ephemeral-deny", threadId: "slack:C-COMMAND-DENY:6100.1", })); await callbacks.onSlashCommand({ endpointId: endpoint.id, provider: "slack", event: { channel: { id: "C-COMMAND-DENY", name: "denied commands", isDM: false, post, postEphemeral, } as never, command: "/anything-else", text: "should not start work", triggerId: "trigger-deny", user: { userId: "U-COMMANDER", userName: "commander", fullName: "Command User", isBot: false, isMe: false, isSystem: false, }, raw: { trigger_id: "trigger-deny" }, adapter: {} as never, openModal: async () => undefined, }, }); expect(post).not.toHaveBeenCalled(); await vi.waitFor(() => expect(postEphemeral).toHaveBeenCalledWith( "U-COMMANDER", `This connection only accepts ${command}.`, { fallbackToDM: false, }, ), ); expect(await service.listConversations(endpoint.id)).toEqual([]); }); it("applies the direct-message toggle to Slack slash commands", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service } = await configuredSlackEndpoint(fixture); const command = endpoint.setup.command; if (!command) throw new Error("Slack endpoint did not expose its command"); if (!callbacks.onSlashCommand) throw new Error("Slack slash command callback was not registered"); await service.update( endpoint.id, { allowDirectMessages: false }, "owner-user", ); const post = vi.fn(async () => ({ id: "unexpected-dm-post", threadId: "slack:D-COMMANDS:6200.1", })); const postEphemeral = vi.fn(async () => ({ id: "ephemeral-dm-deny", threadId: "slack:D-COMMANDS:6200.1", })); await callbacks.onSlashCommand({ endpointId: endpoint.id, provider: "slack", event: { channel: { id: "D-COMMANDS", name: "direct message", isDM: true, post, postEphemeral, } as never, command, text: "should not start work", triggerId: "trigger-dm-deny", user: { userId: "U-COMMANDER", userName: "commander", fullName: "Command User", isBot: false, isMe: false, isSystem: false, }, raw: { trigger_id: "trigger-dm-deny" }, adapter: {} as never, openModal: async () => undefined, }, }); expect(post).not.toHaveBeenCalled(); await vi.waitFor(() => expect(postEphemeral).toHaveBeenCalledWith( "U-COMMANDER", "This channel or account is not allowed to start Paperclip work.", { fallbackToDM: false }, ), ); }); it.each(["metadata_only", "file_revisions"] as const)( "preserves exact Discord source authority through registered Gateway %s updates", async (mode) => { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, runtime, service, wakeup, cancelRun } = await configuredDiscordEndpoint(fixture, { storage: storage.storage, // This fixture explicitly replays the staged root below. Do not // race an automatic root drain against its setup endpoint mutation; // an unadmitted root correctly holds every later lifecycle receipt. scheduleDeferredWork: () => undefined, }); const guildId = "1457808928258658549"; const channelId = "333333333333333333"; const messageId = "555555555555555688"; const userId = "444444444444444444"; const threadId = `discord:${guildId}:${channelId}:${messageId}`; const handlers = new Map< string, (...args: unknown[]) => Promise | void >(); const logger = { child: () => logger, debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn(), }; const adapter = createDiscordAdapter({ applicationId: "123456789012345678", botToken: "discord-fixture-token", webhookVerifier: async () => false, logger, }); const providerRuntime = runtime.endpoints.get(endpoint.id)!; const processMessageUpdated = vi.fn( async (event: { message: Message; previousMessage?: Message; threadId: string; }) => { await callbacks.onMessageUpdated!({ endpointId: endpoint.id, provider: "discord", thread: providerRuntime.thread(event.threadId) as unknown as Thread, message: event.message, previousMessage: event.previousMessage, }); }, ); const processMessageDeleted = vi.fn( async (event: Record) => { await callbacks.onMessageDeleted!({ endpointId: endpoint.id, provider: "discord", event: event as never, }); }, ); await adapter.initialize({ processMessageUpdated, processMessageDeleted, } as never); const gateway = adapter as unknown as { setupLegacyGatewayHandlers( client: unknown, closing: () => boolean, ): void; gatewayChatMessage( message: unknown, threadId: string, mentioned?: boolean, ): Message; }; gateway.setupLegacyGatewayHandlers( { user: { id: "123456789012345678" }, ws: { handlePacket: () => false }, on( event: string, handler: (...args: unknown[]) => Promise | void, ) { handlers.set(event, handler); return this; }, }, () => false, ); const fileBody = Buffer.from("The exact original Discord source.\n"); const sourceFile = { id: "original-file", name: "original.txt", contentType: "text/plain", size: fileBody.length, url: "https://cdn.discordapp.com/attachments/333333333333333333/555555555555555689/original.txt?ex=ORIGINAL_PRIVATE", }; const original = { id: messageId, channelId, guildId, partial: false, content: "@maya keep this exact original request authoritative", attachments: new Map>([ [sourceFile.id, sourceFile], ]), messageSnapshots: new Map(), channel: { isThread: () => false, parentId: null }, author: { id: userId, username: "ada", displayName: "Ada", bot: false }, createdAt: new Date(), editedAt: null, }; let runId: string | undefined; const workspaceRoot = mkdtempSync( path.join(os.tmpdir(), "discord-source-update-"), ); let reader: NativeChatAttachmentReadScope | undefined; try { const message = gateway.gatewayChatMessage(original, threadId, true); // Substitute only transport bytes, not normalization or admission. message.attachments[0]!.fetchData = vi.fn(async () => fileBody); await expect( callbacks.onDiscordRootMentionAdmission!({ endpointId: endpoint.id, guildId, channelId, messageId, message, threadId, userId, }), ).resolves.toBe(false); const [delivery] = await db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.providerEventId, `${threadId}:${messageId}`), ), ); expect(delivery).toBeDefined(); await service.processPendingDeliveries(25, delivery.id); await expect( db .select({ status: chatActions.status }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.deliveryId, delivery.id), eq(chatActions.kind, "inbound_wakeup"), ), ), ).resolves.toEqual([{ status: "processed" }]); await qualifySetupRoundTrip(service, endpoint.id, userId); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); const contextSnapshot = await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "discord", providerMessageId: messageId, }); runId = randomUUID(); const binding = { companyId: fixture.companyId, agentId: fixture.assignedAgentId, issueId: conversation.issueId, runId, }; await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", runtimeMode: "native", nativeIssueId: conversation.issueId, contextSnapshot: { ...contextSnapshot, paperclipHarnessCheckedOut: true, paperclipWake: { checkedOutByHarness: true, externalChatProvider: "discord", issue: { id: conversation.issueId, workMode: "standard" }, commentIds: contextSnapshot.wakeCommentIds, }, }, }); await db .update(issues) .set({ executionRunId: runId, status: "in_progress" }) .where(eq(issues.id, conversation.issueId)); const authority = () => resolveChatRunPresentationAuthorizationReason(db, { companyId: fixture.companyId, issueId: conversation.issueId, runId: runId!, }); await expect(authority()).resolves.toBe("allow_chat_run_presentation"); const [attachment] = await issueService(db).listAttachments( conversation.issueId, ); expect(attachment).toBeDefined(); const selection = { sourceCommentId: attachment.issueCommentId!, attachmentId: attachment.id, }; reader = new NativeChatAttachmentReadScope({ db, binding, workspaceRoot, executionTargetKind: "local", storage: storage.storage, }); const tools = new PaperclipRunnerToolAuthority(db, { ...binding, workspaceRoot, storage: storage.storage, chatAttachmentReadScope: reader, }); await expect( tools.execute({ tool: "list_chat_attachments", callId: "before-update", arguments: { sourceCommentId: selection.sourceCommentId }, }), ).resolves.toMatchObject({ attachments: [expect.objectContaining(selection)], }); const commentsBefore = await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)); const wakesBefore = wakeup.mock.calls.length; const reactionsBefore = providerRuntime.reactions.length; const postsBefore = providerRuntime.posts.length; if (mode === "metadata_only") { const threadCreated = { ...original, flags: 32, thread: { id: messageId }, pinned: true, embeds: [{ title: "Link preview" }], }; await handlers.get("messageUpdate")!(original, threadCreated); await handlers.get("messageUpdate")!(original, threadCreated); expect(processMessageUpdated).not.toHaveBeenCalled(); expect( await db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "message_updated"), ), ), ).toEqual([]); expect( await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)), ).toEqual(commentsBefore); expect(wakeup).toHaveBeenCalledTimes(wakesBefore); expect(cancelRun).not.toHaveBeenCalled(); expect(providerRuntime.reactions).toHaveLength(reactionsBefore); expect(providerRuntime.posts).toHaveLength(postsBefore); await expect( tools.execute({ tool: "list_chat_attachments", callId: "after-metadata", arguments: { sourceCommentId: selection.sourceCommentId }, }), ).resolves.toMatchObject({ attachments: [expect.objectContaining(selection)], }); await expect(authority()).resolves.toBe( "allow_chat_run_presentation", ); await db .update(heartbeatRuns) .set({ status: "succeeded", finishedAt: new Date() }) .where(eq(heartbeatRuns.id, runId)); const comment = await addSelectedChatFinal({ agentId: fixture.assignedAgentId, body: "Exact source answer remains deliverable.", companyId: fixture.companyId, issueId: conversation.issueId, runId, }); await service.processPendingPublications(); await service.processPendingPublications(); expect( providerRuntime.posts.filter( (post) => post.text === "Exact source answer remains deliverable.", ), ).toHaveLength(1); expect( await db .select({ state: chatPublications.state, attempts: chatPublications.attempts, }) .from(chatPublications) .where(eq(chatPublications.commentId, comment.id)), ).toEqual([{ state: "published", attempts: 1 }]); } else { const firstFile = { id: "attachment-1", name: "source.txt", contentType: "text/plain", size: 123, url: "https://cdn.discordapp.com/attachments/channel/file/source.txt?ex=NEVER_PERSIST", }; const secondFile = { ...firstFile, id: "attachment-2" }; const snapshots = [ original, { ...original, attachments: new Map([[firstFile.id, firstFile]]) }, { ...original, attachments: new Map([[secondFile.id, secondFile]]), }, { ...original, attachments: new Map() }, ]; for (let index = 1; index < snapshots.length; index += 1) { await handlers.get("messageUpdate")!( snapshots[index - 1], snapshots[index], ); await handlers.get("messageUpdate")!( snapshots[index - 1], snapshots[index], ); } expect(processMessageUpdated).toHaveBeenCalledTimes(6); const lifecycle = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "message_updated"), ), ); expect(lifecycle).toHaveLength(3); expect(lifecycle.every((row) => row.state === "processed")).toBe( true, ); expect( new Set(lifecycle.map((row) => row.providerEventId)).size, ).toBe(3); expect(JSON.stringify(lifecycle)).not.toContain("NEVER_PERSIST"); expect(JSON.stringify(lifecycle)).not.toContain("cdn.discordapp.com"); await expect( tools.execute({ tool: "read_chat_attachment", callId: "edited-source-read", arguments: selection, }), ).rejects.toThrow("paperclip_runner_chat_attachment_source_denied"); await expect( tools.execute({ tool: "reuse_chat_attachment", callId: "edited-source-reuse", arguments: { ...selection, title: "Do not reuse", idempotencyKey: "edited-source-reuse", }, }), ).rejects.toThrow("paperclip_runner_chat_attachment_source_denied"); expect(storage.putFile).toHaveBeenCalledTimes(1); const textEdit = { ...original, content: "actual changed request", editedAt: new Date(), }; await handlers.get("messageUpdate")!(original, textEdit); await handlers.get("messageDelete")!(textEdit); expect(processMessageDeleted).toHaveBeenCalledTimes(1); const allLifecycle = await db .select({ eventKind: chatDeliveries.eventKind, state: chatDeliveries.state, }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), inArray(chatDeliveries.eventKind, [ "message_updated", "message_deleted", ]), ), ); expect(allLifecycle).toHaveLength(5); expect(allLifecycle.every((row) => row.state === "processed")).toBe( true, ); } } finally { await reader?.close(); if (runId) { await db .update(heartbeatRuns) .set({ status: "succeeded", finishedAt: new Date() }) .where(eq(heartbeatRuns.id, runId)); await db .update(issues) .set({ executionRunId: null }) .where(eq(issues.executionRunId, runId)); } await retirePublicationFixture(service, endpoint.id); rmSync(workspaceRoot, { recursive: true, force: true }); } }, ); it("records message edits and deletes durably and deduplicates lifecycle callbacks", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service, wakeup } = await configuredSlackEndpoint(fixture); const channel = makeThread({ channelId: "C-LIFECYCLE", id: "slack:C-LIFECYCLE:7000.1", name: "lifecycle", }); const original = makeMessage({ id: "7000.1", text: "@maya original request", mentioned: true, }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: original, trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); if (!callbacks.onMessageUpdated || !callbacks.onMessageDeleted) { throw new Error("Slack lifecycle callbacks were not registered"); } const edited = { ...makeMessage({ id: "7000.1", text: "@maya corrected request" }), metadata: { dateSent: new Date("2026-09-04T10:00:00.000Z"), edited: true, editedAt: new Date("2026-09-04T10:01:00.000Z"), }, } as Message; const updateEvent = { endpointId: endpoint.id, provider: "slack" as const, thread: channel.thread, message: edited, previousMessage: original, }; await callbacks.onMessageUpdated(updateEvent); await callbacks.onMessageUpdated(updateEvent); const sameTimestampUpdate = { ...updateEvent, message: { ...edited, text: "@maya second correction in the same millisecond", } as Message, }; await callbacks.onMessageUpdated(sameTimestampUpdate); await callbacks.onMessageUpdated(sameTimestampUpdate); const deleteEvent = { endpointId: endpoint.id, provider: "slack" as const, event: { adapter: {} as never, channelId: "C-LIFECYCLE", deletedAt: new Date("2026-09-04T10:02:00.000Z"), messageId: "7000.1", platform: "slack", raw: {}, threadId: channel.thread.id, }, }; await callbacks.onMessageDeleted(deleteEvent); await callbacks.onMessageDeleted(deleteEvent); await callbacks.onMessageUpdated({ ...updateEvent, message: { ...edited, text: "@maya stale edit delivered after deletion", metadata: { ...edited.metadata, editedAt: new Date("2026-09-04T10:03:00.000Z"), }, } as Message, }); const deliveries = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(deliveries.map((delivery) => delivery.eventKind).sort()).toEqual([ "mention", "message", "message_deleted", "message_updated", "message_updated", "message_updated", ]); expect( deliveries.find( (delivery) => delivery.eventKind === "message_updated" && delivery.state === "filtered", ), ).toMatchObject({ redactedError: "Message edit arrived after the provider message was deleted", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const comments = await db .select() .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)); expect( comments .filter( (comment) => comment.body !== "Setup round trip complete" && comment.body !== "Setup follow-up", ) .map((comment) => comment.body) .sort(), ).toEqual( [ "@maya original request", "An external message was edited:\n\n@maya corrected request", "An external message was edited:\n\n@maya second correction in the same millisecond", "An external message in this conversation was deleted.", ].sort(), ); expect(wakeup).toHaveBeenCalledTimes(2); }); it("redacts lifecycle edits and ignores reactions after channel reach is revoked", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service } = await configuredSlackEndpoint(fixture); const channel = makeThread({ channelId: "C-LIFECYCLE-REVOKED", id: "slack:C-LIFECYCLE-REVOKED:7040.1", name: "lifecycle-revoked", }); const original = makeMessage({ id: "7040.1", text: "@maya establish a task before reach changes", mentioned: true, userId: "U-LIFECYCLE-REVOKED", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: original, trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); if (!callbacks.onMessageUpdated || !callbacks.onReaction) { throw new Error("Slack lifecycle callbacks were not registered"); } const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const commentCount = await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation!.issueId)) .then((rows) => rows.length); await service.replaceResources(endpoint.id, [ { id: conversation!.resourceId!, enabled: false }, ]); const secretEdit = "@maya do not retain this revoked edit"; await callbacks.onMessageUpdated({ endpointId: endpoint.id, provider: "slack", thread: channel.thread, message: { ...original, text: secretEdit, metadata: { dateSent: new Date("2026-09-05T15:10:00Z"), edited: true, editedAt: new Date("2026-09-05T15:11:00Z"), }, } as Message, previousMessage: original, }); const emoji = { name: "thumbs_up", toJSON: () => ":thumbs_up:", toString: () => ":thumbs_up:", }; await callbacks.onReaction({ endpointId: endpoint.id, provider: "slack", event: { adapter: {} as never, added: true, emoji, message: original, messageId: original.id, raw: { event_ts: "7041.1" }, rawEmoji: "+1", thread: channel.thread, threadId: channel.thread.id, user: original.author, }, }); await expect( db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation!.issueId)), ).resolves.toHaveLength(commentCount); const lifecycle = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "message_updated"), ), ) .then((rows) => rows[0]); expect(lifecycle).toMatchObject({ state: "processed", conversationId: conversation!.id, principalId: expect.any(String), redactedError: "Provider edit invalidation retained without task content while destination access is disabled", normalizedEvent: { filtering: { contentRetained: false }, message: { providerSentAt: "2026-09-05T15:11:00.000Z" }, }, }); expect(lifecycle.normalizedEvent.message).not.toHaveProperty("text"); expect(JSON.stringify(lifecycle?.normalizedEvent)).not.toContain( secretEdit, ); await expect( db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "reaction_added"), ), ), ).resolves.toEqual([]); }); async function linkLifecycleFixtureActor(input: { companyId: string; endpointId: string; externalId: string; }) { const [principal] = await db .select() .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, input.companyId), eq(chatExternalPrincipals.externalId, input.externalId), ), ); if (!principal) throw new Error("Expected original lifecycle principal"); await db.insert(chatIdentityLinks).values({ companyId: input.companyId, endpointId: input.endpointId, principalId: principal.id, paperclipUserId: "owner-user", status: "linked", confirmedAt: new Date(), }); await db .update(chatEndpoints) .set({ allowUnlinkedPeople: false }) .where( and( eq(chatEndpoints.companyId, input.companyId), eq(chatEndpoints.id, input.endpointId), ), ); return principal.id; } it.each( ( [ "deletion", "edit", "file_removal", "revoked_edit", "revoked_file_removal", ] as const ).flatMap((kind) => (["image/png", "text/plain"] as const).map((contentType) => ({ kind, contentType, })), ), )( "keeps a verified Slack $kind authoritative for $contentType read and reuse after reach or identity returns", async ({ kind, contentType }) => { const relinkActor = kind.startsWith("revoked_"); const mutation = kind.replace("revoked_", ""); const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredSlackEndpoint(fixture, { storage: storage.storage }); const workspaceRoot = mkdtempSync( path.join(os.tmpdir(), "slack-deleted-source-"), ); let reader: NativeChatAttachmentReadScope | undefined; try { const channel = makeThread({ channelId: "C-DELETED-FILE", id: "slack:C-DELETED-FILE:7060.1", name: "deleted-file", }); const body = contentType === "image/png" ? Buffer.from( "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+a3XcAAAAASUVORK5CYII=", "base64", ) : Buffer.from( "A deleted source must not become reusable when access returns.\n", ); const original = makeMessage({ id: "7060.1", text: "@maya retain this exact file", mentioned: true, raw: { files: [ { id: "F-ORIGINAL", name: "source", mimetype: contentType, size: body.length, }, ], }, attachments: [ { type: contentType === "image/png" ? "image" : "file", name: contentType === "image/png" ? "source.png" : "source.txt", mimeType: contentType, size: body.length, fetchData: async () => body, fetchMetadata: { testRecoveryKey: "slack-deleted-file" }, } as Attachment, ], }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: original, trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); const linkedPrincipalId = relinkActor ? await linkLifecycleFixtureActor({ companyId: fixture.companyId, endpointId: endpoint.id, externalId: original.author.userId, }) : null; const [attachment] = await db .select() .from(issueAttachments) .where(eq(issueAttachments.issueId, conversation.issueId)); expect(attachment?.issueCommentId).toBeTruthy(); const runId = randomUUID(); const binding = { companyId: fixture.companyId, agentId: fixture.assignedAgentId, issueId: conversation.issueId, runId, }; const context = { ...(await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "slack", providerMessageId: original.id, })), paperclipHarnessCheckedOut: true, paperclipWake: { reason: "External chat message received", externalChatProvider: "slack", checkedOutByHarness: true, issue: { id: conversation.issueId, workMode: "standard" }, commentIds: [attachment.issueCommentId!], }, }; await db.insert(heartbeatRuns).values({ ...binding, nativeIssueId: conversation.issueId, id: runId, runtimeMode: "native", status: "running", contextSnapshot: context, }); await db .update(issues) .set({ executionRunId: runId, status: "in_progress" }) .where(eq(issues.id, conversation.issueId)); reader = new NativeChatAttachmentReadScope({ db, binding, workspaceRoot, executionTargetKind: "local", storage: storage.storage, }); const authority = new PaperclipRunnerToolAuthority(db, { ...binding, workspaceRoot, storage: storage.storage, chatAttachmentReadScope: reader, }); const selection = { sourceCommentId: attachment.issueCommentId!, attachmentId: attachment.id, }; await expect( authority.execute({ tool: "list_chat_attachments", callId: "before-deletion", arguments: { sourceCommentId: selection.sourceCommentId }, }), ).resolves.toMatchObject({ attachments: [expect.objectContaining(selection)], }); const beforeComments = await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)); const beforeWakes = wakeup.mock.calls.length; const providerRuntime = runtime.endpoints.get(endpoint.id)!; const beforeProvider = { posts: providerRuntime.posts.length, reactions: providerRuntime.reactions.length, }; if (linkedPrincipalId) { await db .update(chatIdentityLinks) .set({ status: "revoked", paperclipUserId: null, revokedAt: new Date(), updatedAt: new Date(), }) .where( and( eq(chatIdentityLinks.endpointId, endpoint.id), eq(chatIdentityLinks.principalId, linkedPrincipalId), ), ); } else { await service.replaceResources(endpoint.id, [ { id: conversation.resourceId!, enabled: false }, ]); } const providerSentAt = new Date(); const replacementFetch = vi.fn(async () => body); if (mutation === "deletion") { await callbacks.onMessageDeleted!({ endpointId: endpoint.id, provider: "slack", event: { adapter: {} as never, channelId: channel.thread.channelId, deletedAt: providerSentAt, messageId: original.id, platform: "slack", raw: {}, threadId: channel.thread.id, }, }); } else { await callbacks.onMessageUpdated!({ endpointId: endpoint.id, provider: "slack", thread: channel.thread, previousMessage: original, message: { ...original, text: mutation === "edit" ? "PRIVATE_DISABLED_EDIT_MUST_NOT_PERSIST" : original.text, metadata: { ...original.metadata, edited: true, editedAt: providerSentAt, }, attachments: mutation === "edit" ? [ { ...original.attachments[0], fetchData: replacementFetch, }, ] : [], raw: { files: mutation === "edit" ? [ { id: "F-REPLACEMENT", name: "replacement", mimetype: contentType, size: body.length, url_private: "https://files.slack.com/PRIVATE_DISABLED_ATTACHMENT_URL", }, ] : [], }, }, }); } if (linkedPrincipalId) { await expect( authority.execute({ tool: "read_chat_attachment", callId: "before-relink", arguments: selection, }), ).rejects.toThrow( "paperclip_runner_chat_attachment_read_not_authorized", ); await db .update(chatIdentityLinks) .set({ status: "linked", paperclipUserId: "owner-user", revokedAt: null, confirmedAt: new Date(), updatedAt: new Date(), }) .where( and( eq(chatIdentityLinks.endpointId, endpoint.id), eq(chatIdentityLinks.principalId, linkedPrincipalId), ), ); } else { await service.replaceResources(endpoint.id, [ { id: conversation.resourceId!, enabled: true }, ]); } const readsBefore = vi.mocked(storage.storage.getObject).mock.calls .length; await expect( authority.execute({ tool: "reuse_chat_attachment", callId: "deleted-source-reuse", arguments: { ...selection, idempotencyKey: "deleted-source-reuse", title: "Must remain unavailable", }, }), ).rejects.toThrow("paperclip_runner_chat_attachment_source_denied"); await expect( authority.execute({ tool: "read_chat_attachment", callId: "deleted-source-read", arguments: selection, }), ).rejects.toThrow("paperclip_runner_chat_attachment_source_denied"); await expect( authority.execute({ tool: "list_chat_attachments", callId: "after-deletion", arguments: { sourceCommentId: selection.sourceCommentId }, }), ).resolves.toEqual({ attachments: [], nextCursor: null, complete: true, }); expect(vi.mocked(storage.storage.getObject).mock.calls.length).toBe( readsBefore, ); expect(storage.putFile).toHaveBeenCalledTimes(1); expect(replacementFetch).not.toHaveBeenCalled(); await expect( db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)), ).resolves.toEqual(beforeComments); expect(wakeup.mock.calls.length).toBe(beforeWakes); expect({ posts: providerRuntime.posts.length, reactions: providerRuntime.reactions.length, }).toEqual(beforeProvider); const [tombstone] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.eventKind, mutation === "deletion" ? "message_deleted" : "message_updated", ), ), ); expect(tombstone).toMatchObject({ state: "processed", conversationId: conversation.id, principalId: mutation === "deletion" ? null : expect.any(String), normalizedEvent: { filtering: { contentRetained: false }, message: { providerSentAt: providerSentAt.toISOString() }, }, }); expect(tombstone.normalizedEvent.message).not.toHaveProperty("text"); expect(JSON.stringify(tombstone.normalizedEvent)).not.toContain( "PRIVATE_DISABLED_", ); } finally { await reader?.close(); // Global milestone scans include paused endpoints. Retire this exact // fixture's conversation without rewriting its asserted run/audit rows. await db .update(chatConversations) .set({ state: "completed" }) .where( and( eq(chatConversations.companyId, fixture.companyId), eq(chatConversations.endpointId, endpoint.id), ), ); await retirePublicationFixture(service, endpoint.id); rmSync(workspaceRoot, { recursive: true, force: true }); } }, ); it.each(["deletion", "edit", "revoked_edit"] as const)( "refuses an otherwise eligible exact Slack retry after %s without admitting source changes", async (kind) => { const context = await failedChatRetryFixture( "slack", "U-SAFE-PROGRESS", kind === "revoked_edit", ); try { const linkedPrincipalId = kind === "revoked_edit" ? ( await db .select() .from(chatIdentityLinks) .where(eq(chatIdentityLinks.endpointId, context.endpoint.id)) )[0]!.principalId : null; const input = { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user", }; const rollbackProbe = new Error( "rollback authorized pre-deletion retry probe", ); await expect( db.transaction(async (tx) => { await expect( context.service.prepareFailedChatRunRetry(tx, input), ).resolves.toMatchObject({ issueId: context.issue.id }); throw rollbackProbe; }), ).rejects.toBe(rollbackProbe); const before = await db .select({ id: chatActions.id }) .from(chatActions) .where(eq(chatActions.endpointId, context.endpoint.id)); if (linkedPrincipalId) { await db .update(chatIdentityLinks) .set({ status: "revoked", paperclipUserId: null, revokedAt: new Date(), updatedAt: new Date(), }) .where( and( eq(chatIdentityLinks.endpointId, context.endpoint.id), eq(chatIdentityLinks.principalId, linkedPrincipalId), ), ); } else { await context.service.replaceResources(context.endpoint.id, [ { id: context.conversation.resourceId!, enabled: false }, ]); } const callbacks = context.runtime.configurations.get( context.endpoint.id, )!.callbacks; if (kind === "deletion") { await callbacks.onMessageDeleted!({ endpointId: context.endpoint.id, provider: "slack", event: { adapter: {} as never, channelId: context.thread.thread.channelId, deletedAt: new Date(), messageId: context.messageId, platform: "slack", raw: {}, threadId: context.thread.thread.id, }, }); } else { await callbacks.onMessageUpdated!({ endpointId: context.endpoint.id, provider: "slack", thread: context.thread.thread, message: makeMessage({ id: context.messageId, text: "PRIVATE_DISABLED_RETRY_EDIT", userId: "U-SAFE-PROGRESS", }), }); } if (linkedPrincipalId) { await db .update(chatIdentityLinks) .set({ status: "linked", paperclipUserId: "owner-user", revokedAt: null, confirmedAt: new Date(), updatedAt: new Date(), }) .where( and( eq(chatIdentityLinks.endpointId, context.endpoint.id), eq(chatIdentityLinks.principalId, linkedPrincipalId), ), ); } else { await context.service.replaceResources(context.endpoint.id, [ { id: context.conversation.resourceId!, enabled: true }, ]); } await expect( db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, input), ), ).rejects.toMatchObject({ details: { code: "chat_failed_run_retry_not_authorized" }, }); await expect( db .select({ id: chatActions.id }) .from(chatActions) .where(eq(chatActions.endpointId, context.endpoint.id)), ).resolves.toEqual(before); } finally { await db .update(chatConversations) .set({ state: "completed" }) .where( and( eq(chatConversations.companyId, context.fixture.companyId), eq(chatConversations.id, context.conversation.id), ), ); await retirePublicationFixture(context.service, context.endpoint.id); } }, ); it.each([ "unknown_target", "stale_runtime", "unauthorized_edit", "unknown_edit", "stale_edit", ] as const)( "keeps disabled-channel Slack %s scoped to its exact source without admitting content", async (mutation) => { const context = await safeNativeProgressFixture("slack", "97"); try { const callbacks = context.runtime.configurations.get( context.endpoint.id, )!.callbacks; const before = await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, context.conversation.issueId)); const wakeCount = context.wakeup.mock.calls.length; await context.service.replaceResources(context.endpoint.id, [ { id: context.conversation.resourceId!, enabled: false }, ]); if (mutation === "stale_runtime" || mutation === "stale_edit") await db .update(chatEndpoints) .set({ setup: sql`jsonb_set(${chatEndpoints.setup}, '{runtimeGeneration}', to_jsonb(coalesce((${chatEndpoints.setup}->>'runtimeGeneration')::int, 0) + 1))`, }) .where(eq(chatEndpoints.id, context.endpoint.id)); if (mutation === "unauthorized_edit") await db .update(chatEndpoints) .set({ allowUnlinkedPeople: false }) .where(eq(chatEndpoints.id, context.endpoint.id)); if (mutation.endsWith("edit")) { const original = makeMessage({ id: mutation === "unknown_edit" ? "unknown-message" : context.messageId, text: "@maya original", userId: "U-SAFE-PROGRESS", }); await callbacks.onMessageUpdated!({ endpointId: context.endpoint.id, provider: "slack", thread: context.thread.thread, message: { ...original, text: "PRIVATE_DISABLED_EDIT_MUST_NOT_PERSIST", }, previousMessage: original, }); } else { await callbacks.onMessageDeleted!({ endpointId: context.endpoint.id, provider: "slack", event: { adapter: {} as never, channelId: context.thread.thread.channelId, deletedAt: new Date(), messageId: mutation === "unknown_target" ? "unknown-message" : context.messageId, platform: "slack", raw: {}, threadId: context.thread.thread.id, }, }); } const lifecycle = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, context.endpoint.id), inArray(chatDeliveries.eventKind, [ "message_updated", "message_deleted", ]), ), ); expect(lifecycle).toHaveLength( mutation === "stale_runtime" || mutation === "stale_edit" ? 0 : 1, ); if (mutation === "unauthorized_edit") { // Revoking the exact old source is not permission to admit the edit. expect(lifecycle[0]).toMatchObject({ state: "processed", conversationId: context.conversation.id, normalizedEvent: { filtering: { contentRetained: false } }, }); } else { expect(lifecycle.some((row) => row.state === "processed")).toBe( false, ); expect(lifecycle.every((row) => row.conversationId === null)).toBe( true, ); } if (mutation === "unknown_edit") { // An unresolved source retains the existing bounded orphan-grace // receipt, but never becomes an authorized invalidation or task input. expect(lifecycle[0]).toMatchObject({ state: "retry", redactedError: "Waiting briefly for the original message", }); } else { expect(JSON.stringify(lifecycle)).not.toContain( "PRIVATE_DISABLED_EDIT_MUST_NOT_PERSIST", ); } await expect( db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, context.conversation.issueId)), ).resolves.toEqual(before); expect(context.wakeup.mock.calls.length).toBe(wakeCount); } finally { await db .update(chatConversations) .set({ state: "completed" }) .where( and( eq(chatConversations.companyId, context.fixture.companyId), eq(chatConversations.id, context.conversation.id), ), ); await retirePublicationFixture(context.service, context.endpoint.id); } }, ); it.each([ "different_user", "bot_editor", "invalid_signature", "wrong_thread", "unknown_message", ] as const)( "invalidates only an exact provider-verified GitHub source edit independently of editor admission: %s", async (mode) => { const context = await failedChatRetryFixture("github", "42"); try { const configuration = context.runtime.configurations.get( context.endpoint.id, )!; if (configuration.providerConfig.provider !== "github") throw new Error("Expected GitHub configuration"); const webhookSecret = configuration.providerConfig.credentials.webhookSecret; const input = { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user", }; const rollbackProbe = new Error( "rollback unchanged-source retry probe", ); const expectRetryEligible = async () => { await expect( db.transaction(async (tx) => { await expect( context.service.prepareFailedChatRunRetry(tx, input), ).resolves.toMatchObject({ issueId: context.issue.id }); throw rollbackProbe; }), ).rejects.toBe(rollbackProbe); }; await expectRetryEligible(); const beforeComments = await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, context.issue.id)); const beforeWakes = context.wakeup.mock.calls.length; const beforePosts = context.providerRuntime.posts.length; const providerSentAt = new Date().toISOString(); const deliveryId = `lifecycle-editor-${mode}-${randomUUID()}`; const signed = signedGitHubWebhookRequest({ delivery: deliveryId, event: "issue_comment", webhookSecret, payload: { action: "edited", installation: { id: 2468 }, repository: { id: 97531, full_name: "paperclipai/paperclip", name: "paperclip", owner: { id: 1357, login: "paperclipai" }, }, issue: { number: mode === "wrong_thread" ? 792 : 791 }, comment: { id: mode === "unknown_message" ? "990099" : context.messageId, body: "PRIVATE_UNADMITTED_EDITOR_CONTENT", updated_at: providerSentAt, user: { id: 42, login: "original-author", type: "User" }, }, sender: { id: 77, login: "different-editor", type: mode === "bot_editor" ? "Bot" : "User", }, }, }); if (mode === "invalid_signature") signed.headers.set("x-hub-signature-256", "sha256=invalid"); const response = await context.service.handleWebhook( context.endpoint.publicId, "github", signed, ); expect(response.status).toBe(mode === "invalid_signature" ? 401 : 202); if (response.ok) { const [ingress] = await db .select({ id: chatActions.id }) .from(chatActions) .where( and( eq(chatActions.endpointId, context.endpoint.id), eq(chatActions.kind, "github_webhook_ingress"), eq( chatActions.providerActionId, `github_webhook_ingress:${deliveryId}`, ), ), ); expect(ingress).toBeDefined(); // Exercise this verified ingress only. A global delivery sweep can // legitimately wake an unrelated earlier fixture's deferred task. await context.service.processPendingGitHubWebhookIngress( 1, ingress!.id, ); } const verifiedSourceChange = mode === "different_user" || mode === "bot_editor"; if (verifiedSourceChange) { await expect( db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, input), ), ).rejects.toMatchObject({ details: { code: "chat_failed_run_retry_not_authorized" }, }); } else { await expectRetryEligible(); } const deliveries = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, context.endpoint.id), eq(chatDeliveries.eventKind, "message_updated"), ), ); expect(deliveries).toHaveLength(mode === "invalid_signature" ? 0 : 1); if (verifiedSourceChange) { expect(deliveries[0]).toMatchObject({ state: "processed", conversationId: context.conversation.id, principalId: null, normalizedEvent: { filtering: { contentRetained: false }, message: { providerSentAt }, }, }); expect(JSON.stringify(deliveries[0].normalizedEvent)).not.toContain( "PRIVATE_UNADMITTED_EDITOR_CONTENT", ); expect(deliveries[0].normalizedEvent.message).not.toHaveProperty( "text", ); } else { expect( deliveries.every( (row) => row.state !== "processed" && row.conversationId === null, ), ).toBe(true); } await expect( db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, context.issue.id)), ).resolves.toEqual(beforeComments); expect(context.wakeup.mock.calls.length).toBe(beforeWakes); expect(context.providerRuntime.posts.length).toBe(beforePosts); } finally { await db .update(chatConversations) .set({ state: "completed" }) .where( and( eq(chatConversations.companyId, context.fixture.companyId), eq(chatConversations.id, context.conversation.id), ), ); await retirePublicationFixture(context.service, context.endpoint.id); } }, ); it("deduplicates Slack file-only revisions by consumed metadata without retaining private locators", async () => { const context = await safeNativeProgressFixture("slack", "98"); try { const callbacks = context.runtime.configurations.get( context.endpoint.id, )!.callbacks; await context.service.replaceResources(context.endpoint.id, [ { id: context.conversation.resourceId!, enabled: false }, ]); const providerSentAt = new Date(); const original = makeMessage({ id: context.messageId, text: "Unchanged caption", userId: "U-SAFE-PROGRESS", }); const file = (id: string) => ({ id, name: "source.png", mimetype: "image/png", size: 10, original_w: 1, original_h: 1, url_private: "https://files.slack.com/PRIVATE_LOCATOR_ONE", }); const revisions = [ [file("F-FIRST")], [file("F-SECOND")], [ { ...file("F-SECOND"), url_private: "https://files.slack.com/PRIVATE_LOCATOR_TWO", title: "unused title", }, ], [], [], ]; const beforeComments = await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, context.conversation.issueId)); const beforeWakes = context.wakeup.mock.calls.length; for (const files of revisions) { await callbacks.onMessageUpdated!({ endpointId: context.endpoint.id, provider: "slack", thread: context.thread.thread, previousMessage: original, message: { ...original, metadata: { ...original.metadata, edited: true, editedAt: providerSentAt, }, raw: { files }, }, }); } const deliveries = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, context.endpoint.id), eq(chatDeliveries.eventKind, "message_updated"), ), ); expect(deliveries).toHaveLength(3); expect(new Set(deliveries.map((row) => row.providerEventId)).size).toBe( 3, ); expect( deliveries.every( (row) => row.state === "processed" && row.conversationId === context.conversation.id, ), ).toBe(true); expect( JSON.stringify(deliveries.map((row) => row.normalizedEvent)), ).not.toContain("PRIVATE_LOCATOR"); expect( deliveries.every( (row) => !Object.hasOwn(row.normalizedEvent.message!, "text"), ), ).toBe(true); await expect( db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, context.conversation.issueId)), ).resolves.toEqual(beforeComments); expect(context.wakeup.mock.calls.length).toBe(beforeWakes); } finally { await db .update(chatConversations) .set({ state: "completed" }) .where( and( eq(chatConversations.companyId, context.fixture.companyId), eq(chatConversations.id, context.conversation.id), ), ); await retirePublicationFixture(context.service, context.endpoint.id); } }); it("retries lifecycle mutation atomically and reclaims it after restart", async () => { const fixture = await seedCompany(); const deferred: Array<() => void | Promise> = []; const first = await configuredSlackEndpoint(fixture, { deferWebhookProcessing: true, scheduleDeferredWork: (task) => deferred.push(task), }); const thread = makeThread({ channelId: "C-LIFECYCLE-RESTART", id: "slack:C-LIFECYCLE-RESTART:7050.1", name: "lifecycle-restart", }); const original = makeMessage({ id: "7050.1", text: "@maya preserve lifecycle recovery", mentioned: true, }); await deliverMessage({ callbacks: first.callbacks, endpointId: first.endpoint.id, thread: thread.thread, message: original, trigger: "mention", }); expect(deferred).toHaveLength(1); await deferred.shift()?.(); await vi.waitFor(async () => { await expect( db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, first.endpoint.id), eq( chatDeliveries.providerEventId, `${thread.thread.id}:${original.id}`, ), ), ), ).resolves.toEqual([{ state: "processed" }]); }, { timeout: 5_000 }); // The row becomes processed inside the mutation transaction, just before // the conversation drain releases its endpoint/thread lease. Synchronize // on that lease boundary before injecting the exact lifecycle commit fault. await vi.waitFor(async () => { await expect( db .select({ id: chatEndpointLeases.id }) .from(chatEndpointLeases) .where( and( eq(chatEndpointLeases.endpointId, first.endpoint.id), like(chatEndpointLeases.leaseKey, "inbound:%"), ), ), ).resolves.toEqual([]); }, { timeout: 5_000 }); if (!first.callbacks.onMessageUpdated) throw new Error("Slack lifecycle callback was not registered"); await first.callbacks.onMessageUpdated({ endpointId: first.endpoint.id, provider: "slack", thread: thread.thread, message: { ...original, text: "@maya corrected after restart", metadata: { dateSent: new Date("2026-09-05T15:00:00Z"), edited: true, editedAt: new Date("2026-09-05T15:01:00Z"), }, } as Message, previousMessage: original, }); const [lifecycle] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, first.endpoint.id), eq(chatDeliveries.eventKind, "message_updated"), ), ); expect(lifecycle).toMatchObject({ state: "received", attempts: 0 }); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, lifecycle.id)); const originalTransaction = db.transaction.bind(db); let injectedLifecycleFailure = false; const transaction = vi.spyOn(db, "transaction").mockImplementation((async ( ...args: Parameters ) => { const [callback, config] = args; return originalTransaction(async (tx) => { const result = await callback(tx); if (!injectedLifecycleFailure) { const [committedLifecycle] = await tx .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.id, lifecycle.id), eq(chatDeliveries.endpointId, first.endpoint.id), eq(chatDeliveries.state, "processed"), eq(chatDeliveries.attempts, 1), ), ); if (committedLifecycle) { // Only this transaction can see its uncommitted terminal row. // Roll back both its comment and terminal update; unrelated // Gateway renewals must never consume the injected failure. injectedLifecycleFailure = true; throw new Error("injected lifecycle comment failure"); } } return result; }, config); }) as typeof db.transaction); try { await first.service.processPendingDeliveries(25, lifecycle.id); } finally { transaction.mockRestore(); } expect(injectedLifecycleFailure).toBe(true); await expect( db .select({ attempts: chatDeliveries.attempts, state: chatDeliveries.state, }) .from(chatDeliveries) .where(eq(chatDeliveries.id, lifecycle.id)), ).resolves.toEqual([{ attempts: 1, state: "retry" }]); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, first.endpoint.id)); await expect( db .select() .from(issueComments) .where( and( eq(issueComments.issueId, conversation.issueId), eq( issueComments.body, "An external message was edited:\n\n@maya corrected after restart", ), ), ), ).resolves.toHaveLength(0); // Simulate a process dying after the durable claim but before its mutation // transaction. A new service instance must reclaim the stale claim and // commit the comment and terminal state together exactly once. await db .update(chatDeliveries) .set({ state: "processing", nextAttemptAt: null, updatedAt: new Date(Date.now() - 120_000), }) .where(eq(chatDeliveries.id, lifecycle.id)); await first.service.shutdown(); const restarted = createService( new FakeChatSdkRuntime(), fakeSlackFetch() as typeof globalThis.fetch, ); await restarted.service.processPendingDeliveries(25, lifecycle.id); await restarted.service.processPendingDeliveries(25, lifecycle.id); await expect( db .select({ attempts: chatDeliveries.attempts, state: chatDeliveries.state, }) .from(chatDeliveries) .where(eq(chatDeliveries.id, lifecycle.id)), ).resolves.toEqual([{ attempts: 2, state: "processed" }]); await expect( db .select() .from(issueComments) .where( and( eq(issueComments.issueId, conversation.issueId), eq( issueComments.body, "An external message was edited:\n\n@maya corrected after restart", ), ), ), ).resolves.toHaveLength(1); await restarted.service.shutdown(); }); it("filters a claimed message lifecycle delivery after pause and reconnect supersede its runtime", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service } = await configuredSlackEndpoint(fixture); const channel = makeThread({ channelId: "C-LIFECYCLE-FENCE", id: "slack:C-LIFECYCLE-FENCE:7060.1", name: "lifecycle-fence", }); const original = makeMessage({ id: "7060.1", text: "@maya establish lifecycle fencing", mentioned: true, }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: original, trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); if (!callbacks.onMessageUpdated) throw new Error("Slack lifecycle callback was not registered"); const correction = "An external message was edited:\n\n@maya stale correction"; await callbacks.onMessageUpdated({ endpointId: endpoint.id, provider: "slack", thread: channel.thread, message: { ...original, text: "@maya stale correction", metadata: { dateSent: new Date("2026-09-05T16:00:00Z"), edited: true, editedAt: new Date("2026-09-05T16:01:00Z"), }, } as Message, previousMessage: original, }); const [lifecycle] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "message_updated"), ), ); expect(lifecycle.normalizedEvent).toMatchObject({ runtimeContext: { credentialFingerprint: expect.any(String), generation: expect.any(Number), }, }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); await db .delete(issueComments) .where( and( eq(issueComments.issueId, conversation.issueId), eq(issueComments.body, correction), ), ); await db .update(chatDeliveries) .set({ state: "processing", processedAt: null, updatedAt: new Date(Date.now() - 120_000), }) .where(eq(chatDeliveries.id, lifecycle.id)); await service.configure(endpoint.id, { action: "pause" }, "owner-user"); await service.configure(endpoint.id, { action: "resume" }, "owner-user"); await service.processPendingDeliveries(25, lifecycle.id); await expect( db .select({ state: chatDeliveries.state, redactedError: chatDeliveries.redactedError, }) .from(chatDeliveries) .where(eq(chatDeliveries.id, lifecycle.id)), ).resolves.toEqual([ { state: "filtered", redactedError: "Message lifecycle callback belonged to a superseded runtime", }, ]); await expect( db .select({ id: issueComments.id }) .from(issueComments) .where( and( eq(issueComments.issueId, conversation.issueId), eq(issueComments.body, correction), ), ), ).resolves.toHaveLength(0); await service.shutdown(); }); it("acknowledges Telegram edits while verifying and when processing is suspended", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service, wakeup } = await configuredTelegramEndpoint(fixture); const chatId = "77112234"; const dm = makeThread({ channelId: `telegram:${chatId}`, id: `telegram:${chatId}`, isDM: true, name: "Telegram direct message", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: `${chatId}:51`, text: "Original setup request", userId: chatId, }), trigger: "direct_message", }); const sendEdit = (editDate: number, text: string) => service.handleWebhook( endpoint.publicId, "telegram", new Request("https://paperclip.example/telegram", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ update_id: editDate, edited_message: { message_id: 51, edit_date: editDate, chat: { id: Number(chatId), type: "private" }, from: { id: Number(chatId), first_name: "Telegram User" }, text, }, }), }), ); const transaction = vi.spyOn(db, "transaction"); transaction.mockRejectedValueOnce( new Error("injected lifecycle transaction failure"), ); await expect( sendEdit(1_788_620_100, "Edited during setup"), ).rejects.toThrow("injected lifecycle transaction failure"); transaction.mockRestore(); await expect( db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "message_updated"), ), ), ).resolves.toEqual([]); await expect( sendEdit(1_788_620_100, "Edited during setup"), ).resolves.toMatchObject({ ok: true }); expect(wakeup).toHaveBeenCalledTimes(1); for (const status of ["paused", "attention"] as const) { await db .update(chatEndpoints) .set({ status, updatedAt: new Date() }) .where(eq(chatEndpoints.id, endpoint.id)); await expect( sendEdit( status === "paused" ? 1_788_620_101 : 1_788_620_102, `Edited while ${status}`, ), ).resolves.toMatchObject({ ok: true }); } const lifecycle = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "message_updated"), ), ); expect(lifecycle).toEqual([ expect.objectContaining({ providerEventId: "telegram:update:1788620100", state: "processed", }), ]); expect(wakeup).toHaveBeenCalledTimes(1); }); it("durably admits a Telegram edit that races deferred original-message processing", async () => { const fixture = await seedCompany(); const deferred: Array<() => Promise> = []; const runtime = new FakeChatSdkRuntime(); const { service, wakeup } = createService( runtime, fakeTelegramFetch() as typeof globalThis.fetch, { deferWebhookProcessing: true, scheduleDeferredWork: (task) => deferred.push(task), }, ); const endpoint = await service.create( fixture.companyId, { provider: "telegram", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await service.configure( endpoint.id, { action: "configure", credentials: { botToken: "123456:telegram-ordering-test" }, }, "owner-user", ); const callbacks = runtime.configurations.get(endpoint.id)?.callbacks; if (!callbacks) throw new Error("Expected Telegram callbacks"); const chatId = "77112235"; const dm = makeThread({ channelId: `telegram:${chatId}`, id: `telegram:${chatId}`, isDM: true, name: "Telegram direct message", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: `${chatId}:61`, text: "Original deferred request", userId: chatId, }), trigger: "direct_message", }); const [originalDelivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(originalDelivery).toMatchObject({ state: "received" }); const sendEdit = (updateId: number, text: string) => service.handleWebhook( endpoint.publicId, "telegram", new Request("https://paperclip.example/telegram", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ update_id: updateId, edited_message: { message_id: 61, edit_date: 1_788_620_200, chat: { id: Number(chatId), type: "private" }, from: { id: Number(chatId), first_name: "Telegram User" }, text, }, }), }), ); await expect( sendEdit(7_004, "Second edit whose callback arrived first"), ).resolves.toMatchObject({ ok: true }); await expect( sendEdit(7_003, "First edit whose callback arrived second"), ).resolves.toMatchObject({ ok: true }); await expect( sendEdit(7_003, "First edit whose callback arrived second"), ).resolves.toMatchObject({ ok: true }); // The authenticated provider callback can be acknowledged as soon as both // events are durable. The shared per-conversation drain owns ordering and // no longer depends on Telegram redelivering the edit. await service.processPendingDeliveries(25, originalDelivery.id); const admitted = await db .select({ eventKind: chatDeliveries.eventKind, state: chatDeliveries.state, }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(admitted).toHaveLength(3); expect(admitted).toEqual( expect.arrayContaining([ expect.objectContaining({ eventKind: "direct_message", state: "received", }), expect.objectContaining({ eventKind: "message_updated", state: "received", }), expect.objectContaining({ eventKind: "message_updated", state: "received", }), ]), ); expect(deferred).toHaveLength(1); deferred.shift()?.(); await vi.waitFor( async () => { const states = await db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(states).toHaveLength(3); expect(states.every((row) => row.state === "processed")).toBe(true); }, { timeout: 3_000 }, ); const lifecycle = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "message_updated"), ), ); expect(lifecycle).toHaveLength(2); expect(lifecycle).toEqual( expect.arrayContaining([ expect.objectContaining({ providerEventId: "telegram:update:7003", state: "processed", attempts: 1, normalizedEvent: expect.objectContaining({ message: expect.objectContaining({ providerUpdateId: 7003 }), }), }), expect.objectContaining({ providerEventId: "telegram:update:7004", state: "processed", attempts: 1, normalizedEvent: expect.objectContaining({ message: expect.objectContaining({ providerUpdateId: 7004 }), }), }), ]), ); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)) .orderBy(asc(issueComments.createdAt), asc(issueComments.id)), ).resolves.toEqual([ { body: "Original deferred request" }, { body: "An external message was edited:\n\nFirst edit whose callback arrived second", }, { body: "An external message was edited:\n\nSecond edit whose callback arrived first", }, ]); expect(wakeup).toHaveBeenCalledTimes(1); }); it("filters a durable orphan Telegram edit after the bounded reorder window", async () => { const fixture = await seedCompany(); const { endpoint, service, wakeup } = await configuredTelegramEndpoint(fixture); const chatId = "77112239"; await expect( service.handleWebhook( endpoint.publicId, "telegram", new Request("https://paperclip.example/telegram", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ update_id: 7_039, edited_message: { message_id: 91, edit_date: 1_788_620_390, chat: { id: Number(chatId), type: "private" }, from: { id: Number(chatId), first_name: "Telegram User" }, text: "Edit whose original never reached Paperclip", }, }), }), ), ).resolves.toMatchObject({ ok: true }); const [admitted] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.providerEventId, "telegram:update:7039"), ), ); expect(admitted).toMatchObject({ state: "retry", attempts: 1 }); expect(admitted.redactedError).toContain("Waiting briefly"); for (let expectedAttempt = 2; expectedAttempt <= 13; expectedAttempt += 1) { await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, admitted.id)); await service.processPendingDeliveries(25, admitted.id); await expect( db .select({ attempts: chatDeliveries.attempts, redactedError: chatDeliveries.redactedError, state: chatDeliveries.state, }) .from(chatDeliveries) .where(eq(chatDeliveries.id, admitted.id)), ).resolves.toEqual([ { attempts: expectedAttempt, redactedError: expectedAttempt === 13 ? "Original message was not admitted to this conversation" : "Waiting briefly for the original message", state: expectedAttempt === 13 ? "filtered" : "retry", }, ]); } await expect( db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)), ).resolves.toHaveLength(0); expect(wakeup).not.toHaveBeenCalled(); }); it("supplements verified Teams edit, soft-delete, and restore activities against the existing task", async () => { const fixture = await seedCompany(); const runtime = new FakeChatSdkRuntime(); const { service } = createService( runtime, (async () => new Response(JSON.stringify({ access_token: "teams-edit-access" }), { status: 200, headers: { "content-type": "application/json" }, })) as typeof globalThis.fetch, ); const endpoint = await service.create( fixture.companyId, { provider: "microsoft-teams", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); const clientId = randomUUID(); const tenantId = randomUUID(); await service.configure( endpoint.id, { action: "configure", credentials: { clientId, tenantId, clientSecret: "teams-edit-secret", }, }, "owner-user", ); const callbacks = runtime.configurations.get(endpoint.id)?.callbacks; if (!callbacks) throw new Error("Expected Teams callbacks"); const conversationId = "19:teams-edit@thread.tacv2;messageid=teams-root-1"; const serviceUrl = "https://smba.trafficmanager.net/amer/"; const threadId = `teams:${Buffer.from(conversationId).toString("base64url")}:${Buffer.from(serviceUrl).toString("base64url")}`; const channel = makeThread({ channelId: threadId, id: threadId, name: "Teams edit channel", }); const aadObjectId = "00000000-0000-4000-8000-000000000433"; const providerUserId = "29:teams-edit-user"; const messageId = "teams-message-1"; const originalRaw = { id: messageId, type: "message", text: "Maya original Teams request", serviceUrl, from: { id: providerUserId, aadObjectId, name: "Teams Edit User", }, conversation: { id: conversationId, conversationType: "channel", tenantId, }, channelData: { tenant: { id: tenantId } }, }; await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: channel.thread, message: makeMessage({ id: messageId, raw: originalRaw, text: "@maya original Teams request", mentioned: true, userId: providerUserId, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id, providerUserId); await service.test(endpoint.id, "owner-user"); const endpointRuntime = runtime.endpoints.get(endpoint.id); if (!endpointRuntime) throw new Error("Expected Teams endpoint runtime"); const initialRouteCount = endpointRuntime.recordedMicrosoftTeamsRoutes.length; const sendLifecycle = (payload: Record) => service.handleWebhook( endpoint.publicId, "microsoft-teams", new Request("https://paperclip.example/microsoft-teams", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(payload), }), ); const foreignTenantId = randomUUID(); const rejectedLifecyclePayloads = [ { ...originalRaw, type: "messageUpdate", text: "FOREIGN_TENANT_EDIT_MUST_NOT_BE_RETAINED", timestamp: "2026-09-06T13:58:00.000Z", conversation: { ...originalRaw.conversation, tenantId: foreignTenantId, }, channelData: { eventType: "editMessage", tenant: { id: foreignTenantId }, }, }, { ...originalRaw, type: "messageDelete", text: undefined, timestamp: "2026-09-06T13:59:00.000Z", conversation: { id: conversationId, conversationType: "channel", }, channelData: { eventType: "softDeleteMessage" }, }, { ...originalRaw, type: "messageUpdate", text: "CONFLICTING_TENANT_RESTORE_MUST_NOT_BE_RETAINED", timestamp: "2026-09-06T14:00:00.000Z", channelData: { eventType: "undeleteMessage", tenant: { id: foreignTenantId }, }, }, { ...originalRaw, type: "messageUpdate", text: "TARGETED_EDIT_MUST_NOT_BE_RETAINED", timestamp: "2026-09-06T14:00:30.000Z", recipient: { id: clientId, isTargeted: true }, channelData: { eventType: "editMessage", tenant: { id: tenantId }, }, }, ]; for (const payload of rejectedLifecyclePayloads) { await expect(sendLifecycle(payload)).resolves.toMatchObject({ ok: true }); } expect(endpointRuntime.recordedMicrosoftTeamsRoutes).toHaveLength( initialRouteCount, ); await expect( db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), inArray(chatDeliveries.eventKind, [ "message_updated", "message_deleted", "message_restored", ]), ), ), ).resolves.toEqual([]); const editPayload = { ...originalRaw, type: "messageUpdate", text: "Corrected Teams request", timestamp: "2026-09-06T14:01:00.000Z", channelData: { eventType: "editMessage", tenant: { id: tenantId } }, }; const sendEdit = (payload = editPayload) => sendLifecycle(payload); const duplicateResponses = await Promise.all( Array.from({ length: 12 }, () => sendEdit()), ); expect(duplicateResponses).toHaveLength(12); expect(duplicateResponses.every((response) => response.ok)).toBe(true); const sameTimestampRevision = { ...editPayload, text: "Corrected Teams request again in the same millisecond", }; const revisedResponses = await Promise.all( Array.from({ length: 8 }, () => sendEdit(sameTimestampRevision)), ); expect(revisedResponses).toHaveLength(8); expect(revisedResponses.every((response) => response.ok)).toBe(true); const deletePayload = { ...originalRaw, type: "messageDelete", text: undefined, timestamp: "2026-09-06T14:02:00.000Z", channelData: { eventType: "softDeleteMessage", tenant: { id: tenantId }, }, }; const deleteResponses = await Promise.all( Array.from({ length: 12 }, () => sendLifecycle(deletePayload)), ); expect(deleteResponses.every((response) => response.ok)).toBe(true); await expect( sendEdit({ ...editPayload, text: "An edit must not resurrect a soft-deleted Teams message", timestamp: "2026-09-06T14:03:00.000Z", }), ).resolves.toMatchObject({ ok: true }); const restorePayload = { ...originalRaw, type: "messageUpdate", text: "Restored Teams request", timestamp: "2026-09-06T14:04:00.000Z", channelData: { eventType: "undeleteMessage", tenant: { id: tenantId }, }, }; const restoreResponses = await Promise.all( Array.from({ length: 8 }, () => sendLifecycle(restorePayload)), ); expect(restoreResponses.every((response) => response.ok)).toBe(true); await expect( sendEdit({ ...editPayload, text: "Edited after Teams restored the message", timestamp: "2026-09-06T14:05:00.000Z", }), ).resolves.toMatchObject({ ok: true }); await expect( sendLifecycle({ ...deletePayload, timestamp: "2026-09-06T14:02:30.000Z", }), ).resolves.toMatchObject({ ok: true }); await expect( sendLifecycle({ ...restorePayload, text: "STALE_TEAMS_RESTORE_CONTENT_MUST_BE_REDACTED", timestamp: "2026-09-06T14:03:30.000Z", }), ).resolves.toMatchObject({ ok: true }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const editComments = await db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)) .then((rows) => rows.filter((comment) => comment.body.startsWith("An external message was edited:"), ), ); expect(editComments).toHaveLength(3); expect(editComments).toEqual( expect.arrayContaining([ { body: "An external message was edited:\n\nCorrected Teams request" }, { body: "An external message was edited:\n\nCorrected Teams request again in the same millisecond", }, { body: "An external message was edited:\n\nEdited after Teams restored the message", }, ]), ); await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)) .then((rows) => rows.filter( (comment) => comment.body === "An external message in this conversation was deleted." || comment.body === "An external message was restored:\n\nRestored Teams request", ), ), ).resolves.toEqual( expect.arrayContaining([ { body: "An external message in this conversation was deleted." }, { body: "An external message was restored:\n\nRestored Teams request", }, ]), ); await expect( db .select({ principalId: chatDeliveries.principalId, state: chatDeliveries.state, }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "message_updated"), ), ), ).resolves.toEqual( expect.arrayContaining([ { principalId: expect.any(String), state: "processed" }, { principalId: expect.any(String), state: "processed" }, { principalId: null, state: "filtered", }, { principalId: expect.any(String), state: "processed" }, ]), ); await expect( db .select({ eventKind: chatDeliveries.eventKind, principalId: chatDeliveries.principalId, state: chatDeliveries.state, }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), inArray(chatDeliveries.eventKind, [ "message_deleted", "message_restored", ]), ), ), ).resolves.toEqual( expect.arrayContaining([ { eventKind: "message_deleted", principalId: null, state: "processed", }, { eventKind: "message_restored", principalId: expect.any(String), state: "processed", }, { eventKind: "message_deleted", principalId: null, state: "filtered", }, { eventKind: "message_restored", principalId: null, state: "filtered", }, ]), ); const filteredLifecyclePayloads = await db .select({ normalizedEvent: chatDeliveries.normalizedEvent }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.state, "filtered"), inArray(chatDeliveries.eventKind, [ "message_updated", "message_deleted", "message_restored", ]), ), ); expect(filteredLifecyclePayloads).toHaveLength(3); expect(JSON.stringify(filteredLifecyclePayloads)).not.toContain( "An edit must not resurrect a soft-deleted Teams message", ); expect(JSON.stringify(filteredLifecyclePayloads)).not.toContain( "STALE_TEAMS_RESTORE_CONTENT_MUST_BE_REDACTED", ); expect(filteredLifecyclePayloads).toEqual( expect.arrayContaining([ { normalizedEvent: expect.objectContaining({ filtering: { contentRetained: false }, }), }, ]), ); // The shared deletion-only tombstone path must not authorize a restore or // let an older delete override the current provider revision while disabled. const commentsBeforeDisabledLifecycle = await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)); const priorLifecycleIds = new Set( ( await db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)) ).map((row) => row.id), ); await service.replaceResources(endpoint.id, [ { id: conversation.resourceId!, enabled: false }, ]); await sendLifecycle({ ...restorePayload, timestamp: "2026-09-06T14:06:00.000Z", text: "DISABLED_RESTORE_MUST_NOT_PERSIST", }); await sendLifecycle({ ...deletePayload, timestamp: "2026-09-06T14:02:45.000Z", }); const latestLifecycle = ( await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)) ).filter((row) => !priorLifecycleIds.has(row.id)); expect(latestLifecycle).toHaveLength(2); expect(latestLifecycle.every((row) => row.state === "filtered")).toBe(true); expect(JSON.stringify(latestLifecycle)).not.toContain( "DISABLED_RESTORE_MUST_NOT_PERSIST", ); await sendLifecycle({ ...deletePayload, timestamp: "2026-09-06T14:07:00.000Z", }); await service.replaceResources(endpoint.id, [ { id: conversation.resourceId!, enabled: true }, ]); await sendLifecycle({ ...restorePayload, timestamp: "2026-09-06T14:06:30.000Z", text: "OLDER_RESTORE_MUST_NOT_PERSIST", }); const afterDisabledDelete = ( await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)) ).filter((row) => !priorLifecycleIds.has(row.id)); expect( afterDisabledDelete.filter((row) => row.state === "processed"), ).toEqual([ expect.objectContaining({ eventKind: "message_deleted", conversationId: conversation.id, normalizedEvent: { providerEventId: expect.any(String), kind: "message_deleted", runtimeContext: expect.objectContaining({ generation: expect.any(Number), credentialFingerprint: expect.any(String), }), conversation: { externalThreadId: `teams:${Buffer.from(conversationId).toString("base64url")}`, }, message: { providerMessageId: messageId, targetProviderEventId: expect.any(String), providerSentAt: "2026-09-06T14:07:00.000Z", }, filtering: { contentRetained: false }, }, }), ]); expect(afterDisabledDelete).toHaveLength(4); expect( afterDisabledDelete.filter((row) => row.state === "filtered"), ).toHaveLength(3); expect(JSON.stringify(afterDisabledDelete)).not.toContain( "OLDER_RESTORE_MUST_NOT_PERSIST", ); await expect( db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)), ).resolves.toEqual(commentsBeforeDisabledLifecycle); await service.shutdown(); }); it("records verified Telegram edited_message updates against the existing DM task", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service, wakeup } = await configuredTelegramEndpoint(fixture); const chatId = "77112233"; const dm = makeThread({ channelId: `telegram:${chatId}`, id: `telegram:${chatId}`, isDM: true, name: "Telegram direct message", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: `${chatId}:41`, text: "Original Telegram request", userId: chatId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, chatId); await service.test(endpoint.id, "owner-user"); const editPayload = { update_id: 7001, edited_message: { message_id: 41, edit_date: 1_788_620_000, chat: { id: Number(chatId), type: "private" }, from: { id: Number(chatId), first_name: "Telegram User" }, text: "Corrected Telegram request", }, }; await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: { ...makeMessage({ id: `${chatId}:41`, text: "Corrected Telegram request", userId: chatId, }), raw: editPayload.edited_message, } as Message, trigger: "direct_message", }); const sendEdit = () => service.handleWebhook( endpoint.publicId, "telegram", new Request("https://paperclip.example/telegram", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(editPayload), }), ); await Promise.all(Array.from({ length: 12 }, sendEdit)); await service.handleWebhook( endpoint.publicId, "telegram", new Request("https://paperclip.example/telegram", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ ...editPayload, update_id: 7002, edited_message: { ...editPayload.edited_message, text: "Corrected Telegram request again in the same second", }, }), }), ); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const comments = await db .select() .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)); expect( comments .filter((comment) => comment.body.startsWith("An external message was edited:"), ) .map((comment) => comment.body), ).toEqual( expect.arrayContaining([ "An external message was edited:\n\nCorrected Telegram request", "An external message was edited:\n\nCorrected Telegram request again in the same second", ]), ); const updateDeliveries = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "message_updated"), ), ); expect(updateDeliveries).toHaveLength(2); expect(updateDeliveries).toEqual( expect.arrayContaining([ expect.objectContaining({ providerEventId: "telegram:update:7001", state: "processed", normalizedEvent: expect.objectContaining({ message: expect.objectContaining({ providerUpdateId: 7001 }), }), }), expect.objectContaining({ providerEventId: "telegram:update:7002", state: "processed", normalizedEvent: expect.objectContaining({ message: expect.objectContaining({ providerUpdateId: 7002 }), }), }), ]), ); const [originalDelivery] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.providerEventId, `telegram:${chatId}:${chatId}:41`), ), ); expect(originalDelivery.normalizedEvent.deduplication).toBeUndefined(); expect(wakeup).toHaveBeenCalledTimes(2); }); it("retains only a source invalidation for a Telegram edit when the original actor's link is revoked", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service } = await configuredTelegramEndpoint(fixture); const chatId = "77112239"; const dm = makeThread({ channelId: `telegram:${chatId}`, id: `telegram:${chatId}`, isDM: true, name: "Telegram direct message", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: `${chatId}:91`, text: "Original authorized Telegram request", userId: chatId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, chatId); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const principal = await db .select() .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, fixture.companyId), eq(chatExternalPrincipals.provider, "telegram"), eq(chatExternalPrincipals.externalId, chatId), ), ) .then((rows) => rows[0]); if (!conversation || !principal) throw new Error("Expected Telegram conversation and principal"); const linkedUserId = `telegram-edit-user-${randomUUID()}`; const now = new Date(); await db.insert(authUsers).values({ id: linkedUserId, name: "Telegram Edit User", email: `${linkedUserId}@example.com`, emailVerified: true, createdAt: now, updatedAt: now, }); await db.insert(companyMemberships).values({ companyId: fixture.companyId, principalType: "user", principalId: linkedUserId, status: "active", membershipRole: "operator", }); await db.insert(chatIdentityLinks).values({ companyId: fixture.companyId, endpointId: endpoint.id, principalId: principal.id, paperclipUserId: linkedUserId, status: "linked", confirmedAt: now, }); await service.update( endpoint.id, { allowUnlinkedPeople: false }, "owner-user", ); await db .update(chatIdentityLinks) .set({ paperclipUserId: null, status: "revoked", revokedAt: new Date(), updatedAt: new Date(), }) .where(eq(chatIdentityLinks.principalId, principal.id)); const commentCount = await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)) .then((rows) => rows.length); const secretEdit = "Revoked Telegram edit must not enter Paperclip"; await expect( service.handleWebhook( endpoint.publicId, "telegram", new Request("https://paperclip.example/telegram", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ update_id: 7_009, edited_message: { message_id: 91, edit_date: 1_788_620_900, chat: { id: Number(chatId), type: "private" }, from: { id: Number(chatId), first_name: "Telegram User" }, text: secretEdit, }, }), }), ), ).resolves.toMatchObject({ ok: true }); await expect( db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)), ).resolves.toHaveLength(commentCount); const [editDelivery] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.providerEventId, "telegram:update:7009"), ), ); expect(editDelivery).toMatchObject({ principalId: principal.id, conversationId: conversation.id, state: "processed", redactedError: "Provider edit invalidation retained without admitting content from an unauthorized actor", normalizedEvent: { filtering: { contentRetained: false } }, }); expect(JSON.stringify(editDelivery.normalizedEvent)).not.toContain( secretEdit, ); }); it("replaces working with one safe ownership-attention message without terminalizing the run", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture); const chatId = "77112249"; const dm = makeThread({ channelId: `telegram:${chatId}`, id: `telegram:${chatId}`, isDM: true, name: "Telegram direct message", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: `${chatId}:101`, text: "Recover this turn safely", userId: chatId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, chatId); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); if (!conversation) throw new Error("Expected Telegram conversation"); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "telegram", providerMessageId: `${chatId}:101`, }), }); await expect(enqueueChatRunMilestones(db)).resolves.toBe(1); await service.processPendingPublications(); await db .update(heartbeatRuns) .set({ errorCode: "native_execution_ownership_unverified", error: "Private runner PID and executable authentication diagnostic", updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, runId)); await expect(enqueueChatRunMilestones(db)).resolves.toBe(1); await service.processPendingPublications(); await expect(enqueueChatRunMilestones(db)).resolves.toBe(0); await service.processPendingPublications(); const providerRuntime = runtime.endpoints.get(endpoint.id); expect(providerRuntime?.posts).toEqual([ { threadId: dm.thread.id, text: "Maya is working…" }, ]); expect(providerRuntime?.edits).toEqual([ { threadId: dm.thread.id, messageId: "outbound-2", text: "Maya needs a Paperclip admin to safely recover this turn before more work can start. Open the task in Paperclip for details.", }, ]); expect(JSON.stringify(providerRuntime?.edits)).not.toContain( "Private runner", ); const [retainedRun] = await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, runId)); expect(retainedRun).toMatchObject({ status: "running", finishedAt: null }); }); it.each(["queued", "working", "waiting_for_input"] as const)( "does not let a late %s milestone replace authoritative ownership attention or final status", async (lateMilestone) => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture); const chatId = "77112250"; const dm = makeThread({ channelId: `telegram:${chatId}`, id: `telegram:${chatId}`, isDM: true, name: "Telegram direct message", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: `${chatId}:101`, text: "Recover safely", userId: chatId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, chatId); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); if (!conversation) throw new Error("Expected Telegram conversation"); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: lateMilestone === "queued" ? "running" : "queued", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "telegram", providerMessageId: `${chatId}:101`, }), }); await expect(enqueueChatRunMilestones(db)).resolves.toBe(1); await service.processPendingPublications(); const terminal = lateMilestone === "waiting_for_input"; await db .update(heartbeatRuns) .set({ status: terminal ? "failed" : "running", errorCode: terminal ? "adapter_failed" : "native_execution_ownership_unverified", error: "Private diagnostic must not leave Paperclip", updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, runId)); await expect(enqueueChatRunMilestones(db)).resolves.toBe(1); await service.processPendingPublications(); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Telegram runtime"); const settledEdits = [...providerRuntime.edits]; expect(settledEdits).toHaveLength(1); // Model a sweep that sampled an earlier run state, then finished its // binding lookup after a newer milestone had already reached the provider. const [stale] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${runId}:${lateMilestone}:${endpoint.id}`, payload: { text: "Stale status must not replace the authoritative message", progressState: lateMilestone, }, state: "pending", }) .returning(); await service.processPendingPublications(); expect(providerRuntime.posts).toHaveLength(1); expect(providerRuntime.edits).toEqual(settledEdits); expect(JSON.stringify(providerRuntime.edits)).not.toContain( "Private diagnostic", ); const [retained] = await db .select() .from(chatPublications) .where(eq(chatPublications.id, stale!.id)); expect(retained).toMatchObject({ state: "cancelled", attempts: 0 }); }, ); it("keeps an internal Telegram run summary private and completes its progress message", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture); const chatId = "77112240"; const dm = makeThread({ channelId: `telegram:${chatId}`, id: `telegram:${chatId}`, isDM: true, name: "Telegram direct message", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: `${chatId}:101`, text: "Finish without an externally authored reply", userId: chatId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, chatId); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); if (!conversation) throw new Error("Expected Telegram conversation"); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "telegram", providerMessageId: `${chatId}:101`, }), }); await expect(enqueueChatRunMilestones(db)).resolves.toBe(1); await service.processPendingPublications(); // Heartbeat persists success before its presentation resolver runs. A // reconciliation sweep in this window must not race ahead with a generic // completion that could hide a later explicit response. await db .update(heartbeatRuns) .set({ status: "succeeded", updatedAt: new Date() }) .where(eq(heartbeatRuns.id, runId)); await expect(enqueueChatRunMilestones(db)).resolves.toBe(0); const internalSummary = await issueService(db).addComment( conversation.issueId, "Internal presentation summary that must never reach Telegram", { agentId: fixture.assignedAgentId, runId }, { authorType: "agent", authorizationReason: "internal_agent_write" }, ); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.commentId, internalSummary.id)), ).resolves.toHaveLength(0); const internalAttachmentBody = Buffer.from("internal attachment"); await issueService(db).createAttachment({ issueId: conversation.issueId, issueCommentId: internalSummary.id, provider: "local_disk", objectKey: `chat-tests/${randomUUID()}`, contentType: "text/plain", byteSize: internalAttachmentBody.byteLength, sha256: createHash("sha256").update(internalAttachmentBody).digest("hex"), originalFilename: "internal.txt", createdByAgentId: fixture.assignedAgentId, createdByRunId: runId, }); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.commentId, internalSummary.id)), ).resolves.toHaveLength(0); await db .update(heartbeatRuns) .set({ resultJson: { presentationDecision: { chosenSource: "final_agent_message", commentAction: "create", }, }, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, runId)); await expect(enqueueChatRunMilestones(db)).resolves.toBe(1); await service.processPendingPublications(); const providerRuntime = runtime.endpoints.get(endpoint.id); expect(providerRuntime?.posts).toEqual([ { threadId: dm.thread.id, text: "Maya is working…" }, ]); expect(providerRuntime?.edits).toEqual([ { threadId: dm.thread.id, messageId: "outbound-2", text: "Maya completed this turn.", }, ]); expect( JSON.stringify({ posts: providerRuntime?.posts, edits: providerRuntime?.edits, }), ).not.toContain("Internal presentation summary"); }); it.each([ { authorizationReason: "paperclip_runner_protocol", chatId: "77112241", publishesComment: false, }, { authorizationReason: "allow_visible_issue_write", chatId: "77112242", publishesComment: false, }, { authorizationReason: "allow_chat_run_presentation", chatId: "77112243", publishesComment: true, }, ] as const)( "publishes only a resolver-selected Telegram reply for $authorizationReason", async ({ authorizationReason, chatId, publishesComment }) => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture); const dm = makeThread({ channelId: `telegram:${chatId}`, id: `telegram:${chatId}`, isDM: true, name: "Telegram direct message", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: `${chatId}:102`, text: "Return an explicit reply", userId: chatId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, chatId); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); if (!conversation) throw new Error("Expected Telegram conversation"); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "telegram", providerMessageId: `${chatId}:102`, }), }); await expect(enqueueChatRunMilestones(db)).resolves.toBe(1); await service.processPendingPublications(); const resolvedAuthorizationReason = authorizationReason === "allow_chat_run_presentation" ? await resolveChatRunPresentationAuthorizationReason(db, { companyId: fixture.companyId, issueId: conversation.issueId, runId, }) : authorizationReason; expect(resolvedAuthorizationReason).toBe(authorizationReason); const comment = await issueService(db).addComment( conversation.issueId, `Explicit Telegram reply via ${authorizationReason}`, { agentId: fixture.assignedAgentId, runId }, { authorType: "agent", authorizationReason: resolvedAuthorizationReason, }, ); await service.processPendingPublications(); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.commentId, comment.id)), ).resolves.toHaveLength(publishesComment ? 1 : 0); await db .update(heartbeatRuns) .set({ status: "succeeded", resultJson: { presentationDecision: { chosenSource: publishesComment ? "existing_issue_comment" : "none", commentAction: "none", }, }, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, runId)); const laterQueuedRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: laterQueuedRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "queued", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "telegram", providerMessageId: `${chatId}:102`, }), updatedAt: new Date(Date.now() + 1_000), }); // The settled successful run must be filtered before LIMIT. Otherwise a // page of explicit replies can starve later run milestones forever. await expect(enqueueChatRunMilestones(db, { limit: 1 })).resolves.toBe(1); if (!publishesComment) { await service.processPendingPublications(); await expect(enqueueChatRunMilestones(db, { limit: 1 })).resolves.toBe( 1, ); } const providerRuntime = runtime.endpoints.get(endpoint.id); expect(providerRuntime?.posts).toEqual([ { threadId: dm.thread.id, text: "Maya is working…" }, ]); expect(providerRuntime?.edits).toEqual([ { threadId: dm.thread.id, messageId: "outbound-2", text: publishesComment ? `Explicit Telegram reply via ${authorizationReason}` : "Maya completed this turn.", }, ]); await expect( db .select() .from(chatPublications) .where( like(chatPublications.idempotencyKey, `run:${runId}:completed:%`), ), ).resolves.toHaveLength(publishesComment ? 0 : 1); await expect( db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `run:${laterQueuedRunId}:queued:${endpoint.id}`, ), ), ).resolves.toHaveLength(1); }, ); describe("reconciled chat retry delivery ownership", () => { async function fixture(deferScheduling = false) { const context = await failedChatRetryFixture( "slack", "U-SAFE-PROGRESS", true, ); const [action] = await db .insert(issueRecoveryActions) .values({ companyId: context.fixture.companyId, sourceIssueId: context.issue.id, kind: "active_run_watchdog", status: "active", ownerType: "board", returnOwnerAgentId: context.fixture.assignedAgentId, cause: "uncertain_external_action", fingerprint: context.runId, nextAction: "Continue only the exact verified failed chat request.", evidence: { runId: context.runId }, }) .returning(); const genericWake = vi.fn(async () => null); const app = express(); app.use(express.json()); app.use((req, _res, next) => { req.actor = boardActor(context.fixture.companyId); next(); }); app.use( "/api", issueRoutes(db, createStorageService().storage, { chatRunRetries: { prepareFailedChatRunRetry: context.service.prepareFailedChatRunRetry, processFailedChatRunRetry: deferScheduling ? async () => { throw new Error("fixture holds post-commit scheduling"); } : context.service.processFailedChatRunRetry, }, recoveryActionEnqueueWakeup: genericWake, }), ); app.use(errorHandler); const server = createServer(app); await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", () => { server.removeListener("error", reject); resolve(); }); }); const api = request( `http://127.0.0.1:${(server.address() as AddressInfo).port}`, ); const body = { actionId: action!.id, outcome: "restored", sourceIssueStatus: "todo", executionReconciliation: { runId: context.runId, providerStopped: true, actionOutcome: "not_performed", outcomeEvidence: "The stopped local fixture performed no provider action; the original source is still exact.", }, }; context.wakeup.mockClear(); return { ...context, action: action!, genericWake, resolve: () => api .post(`/api/issues/${context.issue.id}/recovery-actions/resolve`) .send(body), async cleanup() { try { await db .update(issueRecoveryActions) .set({ status: "cancelled", resolvedAt: new Date() }) .where(eq(issueRecoveryActions.id, action!.id)); await db .update(chatConversations) .set({ state: "completed" }) .where(eq(chatConversations.id, context.conversation.id)); await retirePublicationFixture( context.service, context.endpoint.id, ); } finally { await new Promise((resolve) => { server.close(() => resolve()); server.closeAllConnections(); }); } }, }; } it.each(["concurrent", "lost_scheduler_receipt"] as const)( "keeps one exact chat delivery owner through %s reconciliation", async (mode) => { const context = await fixture(); try { if (mode === "lost_scheduler_receipt") { // The scheduler is simulated, but its durable authorization and // receipt are the real service transaction. No model turn runs. context.wakeup.mockImplementationOnce(async (agentId, opts) => { const receipt = opts.durableChatRequest!; await db.transaction(async (tx) => { await receipt.authorize(tx); await tx.insert(agentWakeupRequests).values({ id: receipt.id, companyId: receipt.companyId, agentId, source: opts.source!, triggerDetail: opts.triggerDetail, reason: opts.reason, payload: opts.payload, requestedByActorType: opts.requestedByActorType, requestedByActorId: opts.requestedByActorId, idempotencyKey: receipt.idempotencyKey, requestedAt: receipt.requestedAt, status: "queued", }); }); throw new Error( "fixture lost acknowledgement after durable chat receipt", ); }); } const responses = await Promise.all([ context.resolve(), context.resolve(), ]); expect(responses.map((response) => response.status)).toEqual([ 200, 200, ]); const retries = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, context.endpoint.id), eq(chatActions.kind, "failed_run_retry"), ), ); expect(retries).toHaveLength(1); const retry = retries[0]!; const [resolved] = await db .select() .from(issueRecoveryActions) .where(eq(issueRecoveryActions.id, context.action.id)); expect(resolved).toMatchObject({ status: "resolved", evidence: { continuationDelivery: "delegated", continuationDeliveryOwner: { kind: "chat_failed_run_retry", actionId: retry.id, }, executionReconciliation: { runId: context.runId }, }, }); await Promise.all([ context.service.processFailedChatRunRetry(retry.id), context.service.processFailedChatRunRetry(retry.id), deliverReconciledExecutions(db, context.genericWake), deliverReconciledExecutions(db, context.genericWake), ]); await context.resolve().expect(200); expect(context.genericWake).not.toHaveBeenCalled(); expect(context.wakeup).toHaveBeenCalledTimes(1); const receipts = await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, retry.id)); expect(receipts).toHaveLength(1); expect(receipts[0]).toMatchObject({ status: "queued", payload: { retryOfRunId: context.runId, source: "chat:slack" }, }); expect( ( await db .select() .from(issueRecoveryActions) .where(eq(issueRecoveryActions.id, context.action.id)) )[0], ).toEqual(resolved); } finally { await context.cleanup(); } }, ); it.each(["revoked_principal", "closed_conversation"] as const)( "retains the sole owner but refuses its deferred dispatch after %s", async (mode) => { const context = await fixture(true); try { await context.resolve().expect(200); const [retry] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, context.endpoint.id), eq(chatActions.kind, "failed_run_retry"), ), ); expect(retry).toMatchObject({ status: "issued" }); if (mode === "revoked_principal") { await db .update(chatIdentityLinks) .set({ status: "revoked", revokedAt: new Date() }) .where(eq(chatIdentityLinks.endpointId, context.endpoint.id)); } else { await db .update(chatConversations) .set({ state: "completed" }) .where(eq(chatConversations.id, context.conversation.id)); } await context.service.processFailedChatRunRetry(retry!.id); await deliverReconciledExecutions(db, context.genericWake); expect(context.wakeup).not.toHaveBeenCalled(); expect(context.genericWake).not.toHaveBeenCalled(); expect( await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, retry!.id)), ).toEqual([]); expect( ( await db .select() .from(issueRecoveryActions) .where(eq(issueRecoveryActions.id, context.action.id)) )[0]?.evidence, ).toMatchObject({ continuationDelivery: "delegated", continuationDeliveryOwner: { kind: "chat_failed_run_retry", actionId: retry!.id, }, }); } finally { await context.cleanup(); } }, ); it.each([ "revoked_principal", "closed_conversation", "missing_source", ] as const)( "does not resolve or mint a substitute owner for %s", async (mode) => { const context = await fixture(); try { if (mode === "revoked_principal") { await db .update(chatIdentityLinks) .set({ status: "revoked", revokedAt: new Date() }) .where(eq(chatIdentityLinks.endpointId, context.endpoint.id)); } else if (mode === "closed_conversation") { await db .update(chatConversations) .set({ state: "completed" }) .where(eq(chatConversations.id, context.conversation.id)); } else { await db .update(heartbeatRuns) .set({ contextSnapshot: { issueId: context.issue.id, source: "chat:slack", }, wakeupRequestId: null, }) .where(eq(heartbeatRuns.id, context.runId)); } const response = await context.resolve(); expect(response.status).toBe(409); expect( ( await db .select() .from(issueRecoveryActions) .where(eq(issueRecoveryActions.id, context.action.id)) )[0], ).toEqual(context.action); expect( ( await db .select() .from(issues) .where(eq(issues.id, context.issue.id)) )[0]?.status, ).toBe("blocked"); expect( await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, context.endpoint.id), eq(chatActions.kind, "failed_run_retry"), ), ), ).toEqual([]); await deliverReconciledExecutions(db, context.genericWake); expect(context.genericWake).not.toHaveBeenCalled(); expect(context.wakeup).not.toHaveBeenCalled(); } finally { await context.cleanup(); } }, ); }); async function failedChatRetryFixture( provider: "slack" | "telegram" | "github", externalActorId?: string, linkedOriginalActor = false, githubUnavailableFile = false, ) { const context = await safeNativeProgressFixture( provider, "91", "channel", externalActorId, linkedOriginalActor, githubUnavailableFile, ); const [issue] = await db .select() .from(issues) .where(eq(issues.id, context.conversation.issueId)); const [action] = await db .select() .from(chatActions) .where( and( eq(chatActions.conversationId, context.conversation.id), eq(chatActions.kind, "inbound_wakeup"), ), ); const [receipt] = await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, action.id)); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: context.fixture.companyId, agentId: context.fixture.assignedAgentId, status: "failed", errorCode: "adapter_failed", finishedAt: new Date(), wakeupRequestId: receipt.id, contextSnapshot: { issueId: issue.id, taskKey: issue.identifier, source: `chat:${provider}`, wakeCommentId: action.payload.commentId, wakeCommentIds: [action.payload.commentId], }, }); await db .update(agentWakeupRequests) .set({ status: "failed", runId }) .where(eq(agentWakeupRequests.id, receipt.id)); await db .update(issues) .set({ description: "ORIGINAL task, not the failed request", status: "blocked", executionRunId: null, }) .where(eq(issues.id, issue.id)); return { ...context, issue, action, receipt, runId }; } it.each([ "settled", "distinct_account", "null_account", "wrong_account", "missing_account", "malformed_account", "missing_thread", "older_warm_run", "committed_marker", "foreign_marker", "prepared_marker", "pending_lease_cleanup", "missing_receipt", "activation_prepared", "changed_checkpoint", "late_event", "leased", "wrong_thread", "source_edited", ])( "retries only the original pre-provider Telegram request after exact cleanup: %s", async (mode) => { const context = await committedChatResponseRecoveryFixture("telegram"); const providerAccount = mode === "null_account" ? null : mode === "distinct_account" ? "retained-backend-account" : "same-retained-thread"; const checkpointAccount = mode === "wrong_account" ? "another-account" : mode === "missing_account" ? undefined : mode === "malformed_account" ? 123 : providerAccount; const previous = process.env.PAPERCLIP_RUNNER_STATE_DIR; const directory = mkdtempSync( path.join(os.tmpdir(), "paperclip-chat-cleanup-retry-"), ); process.env.PAPERCLIP_RUNNER_STATE_DIR = directory; try { const runId = randomUUID(); const nativeSessionId = context.binding.normalizedSessionId; const runnerInstanceId = context.binding.runnerSourceInstanceId; const inputFor = (id: string) => ({ schema: "paperclip.native-execution-input.v1", provider: { kind: "codex", model: null }, binding: { companyId: context.fixture.companyId, runId: id, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, executionWorkspaceId: id, }, task: { identifier: context.issue.identifier, title: "Exact queued request", description: null, prompt: "Exact queued request", workMode: "standard", }, workspace: { cwd: "/tmp/paperclip-cleanup-retry", repoUrl: null, repoRef: null, branchName: null, }, session: { normalizedSessionId: nativeSessionId, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, }, completionContract: { id: context.binding.completionContractId, sha256: context.binding.completionContractSha256, schemaVersion: "paperclip.completion-contract.v1", contract: { revision: "recovered-response-v1", objective: "Exact request", criteria: [ { id: "response", requirement: "Return the requested answer" }, ], }, }, interactionResponses: [], credentialBindings: [], }); await db .update(heartbeatRuns) .set({ status: "succeeded", processPid: 99_999_999, processGroupId: 99_999_999, runnerProfileJson: { nativeExecutionInput: inputFor(context.runId), }, }) .where(eq(heartbeatRuns.id, context.runId)); if (mode === "older_warm_run") { const olderId = randomUUID(); await db.insert(heartbeatRuns).values({ id: olderId, companyId: context.fixture.companyId, agentId: context.fixture.assignedAgentId, status: "succeeded", finishedAt: new Date(Date.now() - 60_000), runtimeMode: "native", nativeIssueId: context.issue.id, nativeSessionId, runnerInstanceId, processPid: 99_999_999, processGroupId: 99_999_999, runnerProfileJson: { nativeExecutionInput: inputFor(olderId) }, }); } await deliverMessage({ callbacks: context.runtime.configurations.get(context.endpoint.id)! .callbacks, endpointId: context.endpoint.id, provider: "telegram", thread: context.thread.thread, message: makeMessage({ id: `${context.thread.thread.channelId}:102`, text: "The exact queued request B", userId: context.thread.thread.channelId, }), trigger: "direct_message", }); const [action] = await db .select() .from(chatActions) .where( and( eq(chatActions.conversationId, context.conversation.id), eq(chatActions.kind, "inbound_wakeup"), sql`${chatActions.id} <> ${context.action.id}::uuid`, ), ); expect(action).toBeDefined(); await db.insert(heartbeatRuns).values({ id: runId, companyId: context.fixture.companyId, agentId: context.fixture.assignedAgentId, status: "failed", errorCode: "adapter_failed", error: "runner_state_identity_mismatch", finishedAt: new Date(), wakeupRequestId: action.id, runtimeMode: "native", nativeIssueId: context.issue.id, nativeSessionId, runnerInstanceId, nativePhase: "observed", processPid: 99_999_999, processGroupId: 99_999_999, contextSnapshot: { issueId: context.issue.id, taskKey: context.issue.identifier, source: "chat:telegram", wakeCommentId: action.payload.commentId, wakeCommentIds: [action.payload.commentId], }, runnerProfileJson: { nativeExecutionInput: inputFor(runId), sessionCheckpoint: { sessionId: mode === "wrong_thread" ? "another-thread" : mode === "missing_thread" ? undefined : "same-retained-thread", providerSessionId: checkpointAccount, identity: { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, runId, sessionId: nativeSessionId, }, }, }, }); await db .update(agentWakeupRequests) .set({ status: "failed", runId }) .where(eq(agentWakeupRequests.id, action.id)); await db.insert(nativeRunFinalizations).values({ companyId: context.fixture.companyId, issueId: context.issue.id, runId, phase: "observed", attempt: 0, leaseOwner: mode === "leased" ? "other-owner" : null, }); await db.insert(environmentLeases).values({ companyId: context.fixture.companyId, issueId: context.issue.id, heartbeatRunId: runId, provider: "local", status: "failed", releasedAt: new Date(), cleanupStatus: mode === "pending_lease_cleanup" ? "pending" : null, }); await db .update(issues) .set({ status: "in_review", executionRunId: null }) .where(eq(issues.id, context.issue.id)); const canonical = (value: unknown): string => value && typeof value === "object" && !Array.isArray(value) ? `{${Object.entries(value) .sort(([a], [b]) => a.localeCompare(b)) .map( ([key, entry]) => `${JSON.stringify(key)}:${canonical(entry)}`, ) .join(",")}}` : JSON.stringify(value); const key = createHash("sha256") .update( canonical({ schema: "paperclip.native-session-scope.v2", companyId: context.fixture.companyId, agentId: context.fixture.assignedAgentId, workspace: { kind: "transient", ...inputFor(runId).workspace }, provider: { driverKind: "codex_app_server", identity: { kind: "codex" }, }, normalizedSessionId: nativeSessionId, }), ) .digest("hex"); const root = path.join(directory, key); mkdirSync(path.join(root, "runner"), { recursive: true }); mkdirSync(path.join(root, "control-plane"), { recursive: true }); const identity = { runId: context.runId, runnerInstanceId, environmentLeaseId: context.runId, normalizedSessionId: nativeSessionId, }; const files = [ [ "control-plane/control-plane-state.json", { schema: "paperclip.runner.durable.control-plane-state.v1", identity, }, ], [ "runner/runner-state.json", { schema: "paperclip.runner.durable.state.v1", ...identity, lifecycle: "suspended", outbox: [], }, ], [ "runner/codex-provider-state.json", { schema: "paperclip.runner.codex-provider-state.v1", lifecycle: "prepared", threadId: "same-retained-thread", providerSessionId: providerAccount, activeProviderTurnId: null, config: { provider: "codex", driver: "codex_app_server" }, pendingEvents: [], queuedEvents: [], toolBridge: { pending: {} }, }, ], ] as const; const bytes = files.map(([file, data]) => { const text = JSON.stringify(data); writeFileSync(path.join(root, file), text); return text; }); const fingerprint = createHash("sha256") .update( JSON.stringify( bytes.map((text) => createHash("sha256").update(text).digest("hex"), ), ), ) .digest("hex"); await db .update(nativeRunFinalizations) .set({ recoveryHistory: mode === "missing_receipt" ? [] : [ { kind: "native_cleanup_maintenance", version: 1, phase: mode === "activation_prepared" || mode === "prepared_marker" ? "activation_prepared" : "settled", requestId: "exact-cleanup-receipt", nativeSessionId, runnerInstanceId, providerSessionId: "same-retained-thread", sourceFingerprint: "a".repeat(64), settledFingerprint: fingerprint, }, ], }) .where(eq(nativeRunFinalizations.runId, context.runId)); if ( ["committed_marker", "foreign_marker", "prepared_marker"].includes( mode, ) ) { writeFileSync( path.join(root, "cleanup-activation.json"), JSON.stringify({ schema: "paperclip.native_cleanup_activation.v1", companyId: context.fixture.companyId, issueId: context.issue.id, runId: context.runId, requestId: mode === "foreign_marker" ? "another-receipt" : "exact-cleanup-receipt", sourceFingerprint: "a".repeat(64), settledFingerprint: fingerprint, }), ); } if (mode === "changed_checkpoint") writeFileSync( path.join(root, "runner/runner-state.json"), JSON.stringify({ ...files[1][1], lifecycle: "ready" }), ); if (mode === "late_event") await db.insert(heartbeatRunEvents).values({ companyId: context.fixture.companyId, agentId: context.fixture.assignedAgentId, runId, seq: 1, eventType: "session.started", sourceInstanceId: runnerInstanceId, payload: { prpEvent: { sourceKind: "runner" } }, }); if (mode === "source_edited") await db .update(issueComments) .set({ body: "Changed after admission", updatedAt: new Date() }) .where(eq(issueComments.id, String(action.payload.commentId))); const stage = () => db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: runId, initiatedByUserId: "owner-user", }), ); const [original] = await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, runId)); if ( mode === "settled" || mode === "distinct_account" || mode === "null_account" || mode === "older_warm_run" || mode === "committed_marker" ) { const intent = await stage(); expect(await stage()).toEqual(intent); await expect( context.service.processFailedChatRunRetry(intent.actionId), ).resolves.toMatchObject({ status: "queued" }); const [receipt] = await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, intent.actionId)); expect(receipt.payload).toMatchObject({ retryOfRunId: runId, wakeCommentIds: [action.payload.commentId], }); expect( await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, runId)), ).toEqual([original]); expect( ( await db .select() .from(nativeRunFinalizations) .where(eq(nativeRunFinalizations.runId, runId)) )[0], ).toMatchObject({ phase: "observed", attempt: 0 }); if (mode === "committed_marker") expect(existsSync(path.join(root, "cleanup-activation.json"))).toBe( true, ); } else await expect(stage()).rejects.toMatchObject({ details: { code: "chat_failed_run_retry_not_authorized" }, }); } finally { await context.service.shutdown(); if (previous === undefined) delete process.env.PAPERCLIP_RUNNER_STATE_DIR; else process.env.PAPERCLIP_RUNNER_STATE_DIR = previous; rmSync(directory, { recursive: true, force: true }); } }, ); it.each([ "bootstrap", "checkpoint", "distinct_account", "null_account", "wrong_account", "missing_account", "malformed_account", "wrong_thread", "missing_thread", "missing_coordinator", "retryable", "leased", "next_attempt", "ambiguous", "integrity", "cleanup", "active_environment", "live_pid", "missing_input", "late_provider_event", ] as const)( "requires terminal-safe native ownership for an exact chat retry: %s", async (kind) => { const context = await failedChatRetryFixture("telegram"); const hasCheckpoint = [ "checkpoint", "distinct_account", "null_account", "wrong_account", "missing_account", "malformed_account", "wrong_thread", "missing_thread", ].includes(kind); const providerAccount = kind === "null_account" ? null : kind === "distinct_account" ? "retry-backend-account" : "exact-retry-thread"; const checkpointAccount = kind === "wrong_account" ? "another-account" : kind === "missing_account" ? undefined : kind === "malformed_account" ? 123 : providerAccount; const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; let stateDirectory: string | null = null; try { const nativeSessionId = randomUUID(); const runnerInstanceId = randomUUID(); const nativeExecutionInput = { schema: "paperclip.native-execution-input.v1", provider: { kind: "codex", model: null }, binding: { companyId: context.fixture.companyId, runId: context.runId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, executionWorkspaceId: context.runId, }, task: { identifier: context.issue.identifier, title: "Exact failed chat", description: null, prompt: "Exact request", workMode: "standard", }, workspace: { cwd: "/tmp/paperclip-exact-chat-retry", repoUrl: null, repoRef: null, branchName: null, }, session: { normalizedSessionId: nativeSessionId, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, }, completionContract: { id: randomUUID(), sha256: "sha", schemaVersion: "paperclip.completion-contract.v1", contract: { revision: "1", objective: "Exact request", criteria: [ { id: "objective", requirement: "Respond to the exact request", }, ], }, }, interactionResponses: [], credentialBindings: [], }; await db .update(heartbeatRuns) .set({ runtimeMode: "native", nativeIssueId: context.issue.id, nativePhase: kind === "retryable" ? "retryable_failure" : "terminal_failure", nativeSessionId, runnerInstanceId, errorCode: "provider_initialize_timeout", processPid: kind === "live_pid" ? process.pid : null, runnerProfileJson: kind === "missing_input" ? {} : { nativeExecutionInput, ...(hasCheckpoint ? { sessionCheckpoint: { sessionId: kind === "wrong_thread" ? "another-thread" : kind === "missing_thread" ? undefined : "exact-retry-thread", providerSessionId: checkpointAccount, }, } : {}), }, }) .where(eq(heartbeatRuns.id, context.runId)); if (kind !== "missing_coordinator") await db.insert(nativeRunFinalizations).values({ runId: context.runId, companyId: context.fixture.companyId, issueId: context.issue.id, attempt: 3, phase: kind === "retryable" ? "retryable_failure" : "terminal_failure", recoveryState: "blocked", failureCode: "native_session_retry_exhausted", leaseOwner: kind === "leased" ? "active-owner" : null, nextAttemptAt: kind === "next_attempt" ? new Date(Date.now() + 30000) : null, failureDetail: { originalFailureCode: kind === "integrity" ? "native_event_replay_conflict" : kind === "cleanup" ? "native_session_cleanup_quarantined" : "provider_initialize_timeout", recoveryMode: kind === "ambiguous" ? "ambiguous_state" : hasCheckpoint ? "exact_checkpoint_resume" : "bootstrap_retry", checkpointExists: hasCheckpoint, providerEventsExist: false, providerSessionEstablished: hasCheckpoint, }, }); if (kind === "late_provider_event") await db.insert(heartbeatRunEvents).values({ companyId: context.fixture.companyId, agentId: context.fixture.assignedAgentId, runId: context.runId, seq: 1, eventType: "session.started", payload: {}, }); if (hasCheckpoint) { stateDirectory = mkdtempSync( path.join(os.tmpdir(), "paperclip-chat-retry-checkpoint-"), ); process.env.PAPERCLIP_RUNNER_STATE_DIR = stateDirectory; const canonical = (value: unknown): string => value && typeof value === "object" && !Array.isArray(value) ? `{${Object.entries(value) .sort(([a], [b]) => a.localeCompare(b)) .map( ([key, entry]) => `${JSON.stringify(key)}:${canonical(entry)}`, ) .join(",")}}` : JSON.stringify(value); const scope = { schema: "paperclip.native-session-scope.v2", companyId: context.fixture.companyId, agentId: context.fixture.assignedAgentId, workspace: { kind: "transient", ...nativeExecutionInput.workspace }, provider: { driverKind: "codex_app_server", identity: { kind: "codex" }, }, normalizedSessionId: nativeSessionId, }; const root = path.join( stateDirectory, createHash("sha256").update(canonical(scope)).digest("hex"), ); mkdirSync(path.join(root, "runner"), { recursive: true }); mkdirSync(path.join(root, "control-plane"), { recursive: true }); const identity = { runId: context.runId, runnerInstanceId, environmentLeaseId: context.runId, normalizedSessionId: nativeSessionId, }; writeFileSync( path.join(root, "control-plane", "control-plane-state.json"), JSON.stringify({ schema: "paperclip.runner.durable.control-plane-state.v1", identity, }), ); writeFileSync( path.join(root, "runner", "runner-state.json"), JSON.stringify({ schema: "paperclip.runner.durable.state.v1", ...identity, lifecycle: "suspended", outbox: [], }), ); writeFileSync( path.join(root, "runner", "codex-provider-state.json"), JSON.stringify({ schema: "paperclip.runner.codex-provider-state.v1", lifecycle: "prepared", threadId: "exact-retry-thread", providerSessionId: providerAccount, activeProviderTurnId: null, config: { provider: "codex", driver: "codex_app_server" }, pendingEvents: [], queuedEvents: [], toolBridge: { pending: {} }, }), ); } if (kind === "active_environment") await db.insert(environmentLeases).values({ companyId: context.fixture.companyId, heartbeatRunId: context.runId, status: "active", }); const attempt = db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user", }), ); // A null account without existing provider-session evidence remains // conservatively ineligible for the exhausted-run recovery lane. if (["bootstrap", "checkpoint", "distinct_account"].includes(kind)) { const staged = await attempt; await expect( context.service.processFailedChatRunRetry(staged.actionId), ).resolves.toMatchObject({ status: "queued" }); const [receipt] = await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, staged.actionId)); expect(receipt.payload).toMatchObject({ retryOfRunId: context.runId, wakeCommentIds: [context.action.payload.commentId], }); } else await expect(attempt).rejects.toMatchObject({ details: { code: "chat_failed_run_retry_not_authorized" }, }); } finally { await context.service.shutdown(); if (stateDirectory) { if (previousStateDirectory === undefined) delete process.env.PAPERCLIP_RUNNER_STATE_DIR; else process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; rmSync(stateDirectory, { recursive: true, force: true }); } } }, ); it.each(["slack", "telegram"] as const)( "retries the exact failed %s chat input with a new idempotent admission", async (provider) => { const context = await failedChatRetryFixture(provider); try { const sourceBefore = await db .select() .from(chatActions) .where(eq(chatActions.id, context.action.id)); const stage = () => db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user", }), ); const [first, duplicate] = await Promise.all([stage(), stage()]); expect(first).toEqual(duplicate); expect(first.actionId).not.toBe(context.action.id); await context.service.processFailedChatRunRetry(first.actionId); await context.service.processFailedChatRunRetry(first.actionId); const [retry] = await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, first.actionId)); expect(retry).toMatchObject({ companyId: context.fixture.companyId, agentId: context.fixture.assignedAgentId, requestedByActorType: context.receipt.requestedByActorType, requestedByActorId: context.receipt.requestedByActorId, payload: { issueId: context.issue.id, taskKey: context.issue.identifier, wakeCommentIds: [context.action.payload.commentId], retryOfRunId: context.runId, }, }); expect( await db .select() .from(chatActions) .where(eq(chatActions.id, context.action.id)), ).toEqual(sourceBefore); expect( await db .select() .from(agentWakeupRequests) .where( and( eq(agentWakeupRequests.companyId, context.fixture.companyId), eq(agentWakeupRequests.id, first.actionId), ), ), ).toHaveLength(1); } finally { await context.service.shutdown(); } }, ); it("denies a staged failed chat retry after its original lifecycle epoch changes", async () => { const context = await failedChatRetryFixture("telegram"); try { const staged = await db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user", }), ); await db .update(chatEndpoints) .set({ setup: sql`jsonb_set(${chatEndpoints.setup}, '{runtimeGeneration}', to_jsonb(coalesce((${chatEndpoints.setup}->>'runtimeGeneration')::int, 0) + 1))`, }) .where(eq(chatEndpoints.id, context.endpoint.id)); await expect( context.service.processFailedChatRunRetry(staged.actionId), ).resolves.toMatchObject({ status: "failed", runId: null }); expect( await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, staged.actionId)), ).toHaveLength(0); } finally { await context.service.shutdown(); } }); it.each([ "paused", "generation", "retired", "edited", "deleted", "relinked", "reassigned", "action_cancelled", "agent_paused", ] as const)( "reauthorizes a failed chat retry and denies %s before scheduling", async (change) => { const context = await failedChatRetryFixture("telegram"); try { const staged = await db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user", }), ); if (change === "paused") await db .update(chatEndpoints) .set({ status: "paused" }) .where(eq(chatEndpoints.id, context.endpoint.id)); if (change === "generation") await db .update(chatEndpoints) .set({ setup: sql`jsonb_set(${chatEndpoints.setup}, '{runtimeGeneration}', to_jsonb(coalesce((${chatEndpoints.setup}->>'runtimeGeneration')::int, 0) + 1))`, }) .where(eq(chatEndpoints.id, context.endpoint.id)); if (change === "retired") await db .update(chatConversations) .set({ state: "completed" }) .where(eq(chatConversations.id, context.conversation.id)); if (change === "edited") await db .update(issueComments) .set({ body: "Changed after failure", updatedAt: new Date(Date.now() + 1000), }) .where( eq(issueComments.id, String(context.action.payload.commentId)), ); if (change === "deleted") { const [original] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, context.action.deliveryId!)); await db.insert(chatDeliveries).values({ companyId: context.fixture.companyId, endpointId: context.endpoint.id, conversationId: context.conversation.id, principalId: context.action.principalId, eventKind: "message_deleted", providerEventId: `deleted-${randomUUID()}`, deduplicationKey: randomUUID(), state: "processed", normalizedEvent: { runtimeContext: original.normalizedEvent.runtimeContext, message: { targetProviderEventId: original.providerEventId }, }, }); } if (change === "relinked") await db .insert(chatIdentityLinks) .values({ companyId: context.fixture.companyId, endpointId: context.endpoint.id, principalId: context.action.principalId!, status: "linked", paperclipUserId: "owner-user", }) .onConflictDoUpdate({ target: [ chatIdentityLinks.endpointId, chatIdentityLinks.principalId, ], set: { status: "linked", paperclipUserId: "owner-user" }, }); if (change === "reassigned") await db .update(issues) .set({ assigneeAgentId: context.fixture.replacementAgentId }) .where(eq(issues.id, context.issue.id)); if (change === "action_cancelled") await db .update(chatActions) .set({ status: "cancelled" }) .where(eq(chatActions.id, context.action.id)); if (change === "agent_paused") await db .update(agents) .set({ status: "paused" }) .where(eq(agents.id, context.fixture.assignedAgentId)); await expect( context.service.processFailedChatRunRetry(staged.actionId), ).resolves.toMatchObject({ status: "failed", runId: null }); expect( await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, staged.actionId)), ).toEqual([]); } finally { await context.service.shutdown(); } }, ); it.each([ "received", "processing", "retry", "failed", "processed", "filtered", "stale", "different_target", ] as const)( "reauthorizes a failed chat retry against a %s provider correction before lifecycle projection", async (state) => { const context = await failedChatRetryFixture("telegram"); try { const staged = await db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user", }), ); const [original] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, context.action.deliveryId!)); const fence = original.normalizedEvent.runtimeContext as Record< string, unknown >; await db.insert(chatDeliveries).values({ companyId: context.fixture.companyId, endpointId: context.endpoint.id, // This is the durable shape produced by recordLifecycleDelivery: // authenticated, but not yet assigned a conversation by its worker. conversationId: null, eventKind: "message_updated", providerEventId: `pending-edit-${randomUUID()}`, deduplicationKey: randomUUID(), state: ["stale", "different_target"].includes(state) ? "received" : state, normalizedEvent: { runtimeContext: state === "stale" ? { ...fence, generation: Number(fence.generation) - 1 } : fence, message: { targetProviderEventId: state === "different_target" ? "unrelated-message" : original.providerEventId, }, }, }); const denied = !["filtered", "stale", "different_target"].includes( state, ); await expect( context.service.processFailedChatRunRetry(staged.actionId), ).resolves.toMatchObject({ status: denied ? "failed" : "queued", runId: null, }); expect( await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, staged.actionId)), ).toHaveLength(denied ? 0 : 1); } finally { await context.service.shutdown(); } }, ); it("does not retry a chat comment edited on the Board during the failed execution", async () => { const context = await failedChatRetryFixture("telegram"); try { const [comment] = await db .select() .from(issueComments) .where(eq(issueComments.id, String(context.action.payload.commentId))); await db .update(issueComments) .set({ body: "Edited while the provider was executing the original request", updatedAt: new Date(comment.createdAt.getTime() + 1000), }) .where(eq(issueComments.id, comment.id)); await db .update(heartbeatRuns) .set({ finishedAt: new Date(comment.createdAt.getTime() + 2000) }) .where(eq(heartbeatRuns.id, context.runId)); await expect( db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user", }), ), ).rejects.toMatchObject({ details: { code: "chat_failed_run_retry_not_authorized" }, }); } finally { await context.service.shutdown(); } }); it("preserves the complete failed coalesced chat batch and rejects a dropped sibling", async () => { const context = await failedChatRetryFixture("telegram"); try { const callbacks = context.runtime.configurations.get( context.endpoint.id, )!.callbacks; const nextMessageId = `${context.thread.thread.channelId}:102`; await deliverMessage({ callbacks, endpointId: context.endpoint.id, provider: "telegram", thread: context.thread.thread, message: makeMessage({ id: nextMessageId, text: "SECOND exact failed request", userId: context.thread.thread.channelId, }), trigger: "direct_message", }); const [second] = await db .select() .from(chatActions) .where( and( eq(chatActions.conversationId, context.conversation.id), eq(chatActions.kind, "inbound_wakeup"), sql`${chatActions.id} <> ${context.action.id}::uuid`, ), ); await db .update(agentWakeupRequests) .set({ status: "coalesced", runId: null, payload: { issueId: context.issue.id, wakeCommentId: second.payload.commentId, coalescedIntoWakeupRequestId: context.receipt.id, }, }) .where(eq(agentWakeupRequests.id, second.id)); const commentIds = [ String(context.action.payload.commentId), String(second.payload.commentId), ]; const failedContext = { issueId: context.issue.id, taskKey: context.issue.identifier, source: "chat:telegram", wakeCommentId: commentIds[1], wakeCommentIds: commentIds, }; await db .update(heartbeatRuns) .set({ contextSnapshot: failedContext, finishedAt: new Date(Date.now() + 1), }) .where(eq(heartbeatRuns.id, context.runId)); const stage = () => db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user", }), ); await db .update(heartbeatRuns) .set({ contextSnapshot: { ...failedContext, wakeCommentIds: [commentIds[1]], }, }) .where(eq(heartbeatRuns.id, context.runId)); await expect(stage()).rejects.toMatchObject({ details: { code: "chat_failed_run_retry_not_authorized" }, }); await db .update(heartbeatRuns) .set({ contextSnapshot: failedContext }) .where(eq(heartbeatRuns.id, context.runId)); const staged = await stage(); await context.service.processFailedChatRunRetry(staged.actionId); const [receipt] = await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, staged.actionId)); expect(receipt.payload).toMatchObject({ wakeCommentIds: commentIds, wakeCommentId: commentIds[1], }); } finally { await context.service.shutdown(); } }); it.each(["slack", "telegram"] as const)( "keeps a %s retry's queue, working, and selected final on its own provider identity", async (provider) => { const context = await failedChatRetryFixture(provider); try { const staged = await db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user", }), ); await context.service.processFailedChatRunRetry(staged.actionId); await db .update(agentWakeupRequests) .set({ status: "deferred_issue_execution" }) .where(eq(agentWakeupRequests.id, staged.actionId)); await context.service.enqueueInboundWakeupPublications(100); await context.service.processPendingPublications(100); const [queued] = await db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `wake:${staged.actionId}:queued:${context.endpoint.id}:${context.conversation.id}`, ), ); expect(queued).toMatchObject({ state: "published", attempts: 1 }); const [receipt] = await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, staged.actionId)); const retryRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: retryRunId, companyId: context.fixture.companyId, agentId: context.fixture.assignedAgentId, status: "running", wakeupRequestId: staged.actionId, retryOfRunId: context.runId, contextSnapshot: receipt.payload, }); await db .update(agentWakeupRequests) .set({ runId: retryRunId, status: "claimed" }) .where(eq(agentWakeupRequests.id, staged.actionId)); await db .update(issues) .set({ status: "in_progress", executionRunId: retryRunId }) .where(eq(issues.id, context.issue.id)); await db.insert(chatPublications).values({ companyId: context.fixture.companyId, endpointId: context.endpoint.id, conversationId: context.conversation.id, issueId: context.issue.id, idempotencyKey: `run:${retryRunId}:working:${context.endpoint.id}`, payload: { text: "Maya is retrying the requested work…", progressState: "working", }, state: "pending", }); await context.service.processPendingPublications(100); await db .update(heartbeatRuns) .set({ status: "succeeded", finishedAt: new Date() }) .where(eq(heartbeatRuns.id, retryRunId)); const final = await addSelectedChatFinal({ companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, runId: retryRunId, body: "EXACT-RETRY-ANSWER", }); await context.service.processPendingPublications(100); const [finalPublication] = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, final.id)); expect(finalPublication).toMatchObject({ state: "published", attempts: 1, providerMessageId: queued.providerMessageId, }); expect(context.providerRuntime.posts).toHaveLength(1); expect(context.providerRuntime.edits.at(-1)?.text).toContain( "EXACT-RETRY-ANSWER", ); } finally { await context.service.shutdown(); } }, ); it("retains an exact chat retry through a transient scheduling failure and a lost committed receipt acknowledgement", async () => { const context = await failedChatRetryFixture("telegram"); try { const staged = await db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user", }), ); context.wakeup.mockClear(); context.wakeup.mockRejectedValueOnce( new Error("temporary scheduler failure"), ); await expect( context.service.processFailedChatRunRetry(staged.actionId), ).resolves.toMatchObject({ status: "queued", runId: null }); expect( await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, staged.actionId)), ).toHaveLength(0); await db .update(chatActions) .set({ result: sql`${chatActions.result} - 'retryAt'` }) .where(eq(chatActions.id, staged.actionId)); context.wakeup.mockImplementationOnce(async (agentId, opts) => { const request = opts.durableChatRequest!; await db.transaction(async (tx) => { await request.authorize(tx); await tx.insert(agentWakeupRequests).values({ id: request.id, companyId: request.companyId, agentId, source: opts.source!, triggerDetail: opts.triggerDetail, reason: opts.reason, payload: opts.payload, requestedByActorType: opts.requestedByActorType, requestedByActorId: opts.requestedByActorId, idempotencyKey: request.idempotencyKey, requestedAt: request.requestedAt, status: "queued", }); }); throw new Error("receipt acknowledgement lost after commit"); }); await expect( context.service.processFailedChatRunRetry(staged.actionId), ).resolves.toMatchObject({ status: "queued", runId: null }); await context.service.processFailedChatRunRetry(staged.actionId); expect(context.wakeup).toHaveBeenCalledTimes(2); expect( await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, staged.actionId)), ).toHaveLength(1); expect( await db .select() .from(chatActions) .where(eq(chatActions.id, staged.actionId)), ).toEqual([ expect.objectContaining({ status: "processed", result: expect.objectContaining({ attemptCount: 2 }), }), ]); } finally { await context.service.shutdown(); } }); it("retains a retry final when its source authorization read fails transiently", async () => { const context = await failedChatRetryFixture("telegram"); let spy: { mockRestore(): void } | undefined; try { const staged = await db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user", }), ); await context.service.processFailedChatRunRetry(staged.actionId); const [receipt] = await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, staged.actionId)); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: context.fixture.companyId, agentId: context.fixture.assignedAgentId, status: "succeeded", finishedAt: new Date(), wakeupRequestId: staged.actionId, retryOfRunId: context.runId, contextSnapshot: receipt.payload, }); await db .update(agentWakeupRequests) .set({ runId, status: "completed" }) .where(eq(agentWakeupRequests.id, staged.actionId)); const final = await addSelectedChatFinal({ companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, runId, body: "Retry answer survives temporary source read failure", }); const transaction = db.transaction.bind(db); let injected = false; spy = vi.spyOn(db, "transaction").mockImplementation((callback, config) => transaction(async (tx) => { const select = tx.select.bind(tx); vi.spyOn(tx, "select").mockImplementation((( fields?: Record, ) => { if (!injected && fields?.status === agents.status) { injected = true; throw new Error("temporary source read unavailable"); } return fields ? select(fields as Parameters[0]) : select(); }) as typeof tx.select); return callback(tx); }, config), ); await context.service.processPendingPublications(100); spy.mockRestore(); spy = undefined; expect(injected).toBe(true); const [pending] = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, final.id)); expect(pending.state).toBe("retry"); expect(context.providerRuntime.posts).toHaveLength(0); await db .update(chatPublications) .set({ nextAttemptAt: null }) .where(eq(chatPublications.id, pending.id)); await context.service.processPendingPublications(100); expect( await db .select() .from(chatPublications) .where(eq(chatPublications.id, pending.id)), ).toEqual([expect.objectContaining({ state: "published" })]); expect(context.providerRuntime.posts).toHaveLength(1); } finally { spy?.mockRestore(); await context.service.shutdown(); } }); it("allows a new exact failed chat retry after an ordinary retry fails, without rearming its ancestors", async () => { const context = await failedChatRetryFixture("telegram"); try { const input = { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user", }; const first = await db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, input), ); await context.service.processFailedChatRunRetry(first.actionId); const [receipt] = await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, first.actionId)); const failedRetryRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: failedRetryRunId, companyId: context.fixture.companyId, agentId: context.fixture.assignedAgentId, status: "failed", errorCode: "adapter_failed", finishedAt: new Date(), wakeupRequestId: receipt.id, retryOfRunId: context.runId, contextSnapshot: receipt.payload, }); await db .update(agentWakeupRequests) .set({ runId: failedRetryRunId, status: "failed" }) .where(eq(agentWakeupRequests.id, receipt.id)); const [ancestor] = await db .select() .from(chatActions) .where(eq(chatActions.id, first.actionId)); const cyclicContext = { ...receipt.payload, retryOfRunId: failedRetryRunId, chatFailedRunRetry: { version: 1, actionId: first.actionId, failedRunId: failedRetryRunId, }, }; await db .update(chatActions) .set({ payload: { ...ancestor.payload, failedRunId: failedRetryRunId }, }) .where(eq(chatActions.id, ancestor.id)); await db .update(heartbeatRuns) .set({ retryOfRunId: failedRetryRunId, contextSnapshot: cyclicContext }) .where(eq(heartbeatRuns.id, failedRetryRunId)); await expect( db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, { ...input, failedRunId: failedRetryRunId, }), ), ).rejects.toMatchObject({ details: { code: "chat_failed_run_retry_not_authorized" }, }); await db .update(chatActions) .set({ payload: ancestor.payload }) .where(eq(chatActions.id, ancestor.id)); await db .update(heartbeatRuns) .set({ retryOfRunId: context.runId, contextSnapshot: receipt.payload }) .where(eq(heartbeatRuns.id, failedRetryRunId)); const second = await db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, { ...input, failedRunId: failedRetryRunId, }), ); expect(second.actionId).not.toBe(first.actionId); await context.service.processFailedChatRunRetry(second.actionId); const [secondReceipt] = await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, second.actionId)); expect(secondReceipt.payload).toMatchObject({ retryOfRunId: failedRetryRunId, taskKey: context.issue.identifier, wakeCommentIds: [context.action.payload.commentId], }); expect( await db .select() .from(chatActions) .where(eq(chatActions.id, first.actionId)), ).toEqual([ancestor]); } finally { await context.service.shutdown(); } }); async function failedReviewedGitHubRetryFixture() { const context = await failedChatRetryFixture("github"); await db .update(issues) .set({ status: "in_review" }) .where(eq(issues.id, context.issue.id)); await db .update(heartbeatRuns) .set({ errorCode: "setup_failed", error: "reviewed_chat_execution_binding_not_authorized", startedAt: new Date(Date.now() - 10), finishedAt: new Date(), // Admission captures the prior display/session identity before any // execution starts. It is not evidence of a turn for this new run. sessionIdBefore: "prior-provider-session", }) .where(eq(heartbeatRuns.id, context.runId)); await db.insert(heartbeatRunEvents).values({ companyId: context.fixture.companyId, agentId: context.fixture.assignedAgentId, runId: context.runId, seq: 1, eventType: "error", stream: "system", level: "error", message: "reviewed_chat_execution_binding_not_authorized", }); return context; } it("retries an exact GitHub source denied by pre-provider reviewed attestation without rewriting its failure", async () => { const context = await failedReviewedGitHubRetryFixture(); try { const [original] = await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, context.runId)); const [originalSource] = await db .select() .from(chatActions) .where(eq(chatActions.id, context.action.id)); const stage = () => db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user", }), ); const first = await stage(); expect(await stage()).toEqual(first); await expect( context.service.processFailedChatRunRetry(first.actionId), ).resolves.toMatchObject({ status: "queued" }); await context.service.processFailedChatRunRetry(first.actionId); const receipts = await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, first.actionId)); expect(receipts).toEqual([ expect.objectContaining({ companyId: context.fixture.companyId, agentId: context.fixture.assignedAgentId, requestedByActorType: context.receipt.requestedByActorType, requestedByActorId: context.receipt.requestedByActorId, payload: expect.objectContaining({ issueId: context.issue.id, taskKey: context.issue.identifier, wakeCommentIds: [context.action.payload.commentId], retryOfRunId: context.runId, }), }), ]); expect( await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, context.runId)), ).toEqual([original]); expect( await db .select() .from(chatActions) .where(eq(chatActions.id, context.action.id)), ).toEqual([originalSource]); expect( await db .select() .from(nativeRunFinalizations) .where(eq(nativeRunFinalizations.runId, context.runId)), ).toEqual([]); expect( await db .select() .from(nativeRunResults) .where(eq(nativeRunResults.runId, context.runId)), ).toEqual([]); } finally { await context.service.shutdown(); } }); it("rechecks pre-provider retry evidence after staging and refuses a late execution event", async () => { const context = await failedReviewedGitHubRetryFixture(); try { const staged = await db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user", }), ); const wakeCount = context.wakeup.mock.calls.length; await db.insert(heartbeatRunEvents).values({ companyId: context.fixture.companyId, agentId: context.fixture.assignedAgentId, runId: context.runId, seq: 2, eventType: "prp", sourceInstanceId: "late-provider", payload: { type: "turn.started" }, }); await expect( context.service.processFailedChatRunRetry(staged.actionId), ).resolves.toMatchObject({ status: "failed", runId: null }); expect(context.wakeup.mock.calls.length).toBe(wakeCount); expect( await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, staged.actionId)), ).toEqual([]); } finally { await context.service.shutdown(); } }); it.each([ "other_setup_error", "missing_error_event", "provider_event", "native_profile", "native_identity", "process_identity", "provider_output", "environment_lease", "uncertain_publication", "selected_reply", "revoked_resource", "retired_conversation", "edited_source", ] as const)( "denies a pre-provider reviewed retry with %s", async (change) => { const context = await failedReviewedGitHubRetryFixture(); try { if (change === "other_setup_error") await db .update(heartbeatRuns) .set({ error: "Some other setup failure" }) .where(eq(heartbeatRuns.id, context.runId)); if (change === "missing_error_event") await db .delete(heartbeatRunEvents) .where(eq(heartbeatRunEvents.runId, context.runId)); if (change === "provider_event") await db.insert(heartbeatRunEvents).values({ companyId: context.fixture.companyId, agentId: context.fixture.assignedAgentId, runId: context.runId, seq: 2, eventType: "prp", sourceInstanceId: "provider-source", payload: { type: "turn.started" }, }); if (change === "native_profile") await db .update(heartbeatRuns) .set({ runnerProfileJson: { nativeExecutionInput: {} } }) .where(eq(heartbeatRuns.id, context.runId)); if (change === "native_identity") await db .update(heartbeatRuns) .set({ nativeSessionId: randomUUID() }) .where(eq(heartbeatRuns.id, context.runId)); if (change === "process_identity") await db .update(heartbeatRuns) .set({ processPid: 987654 }) .where(eq(heartbeatRuns.id, context.runId)); if (change === "provider_output") await db .update(heartbeatRuns) .set({ lastOutputSeq: 1 }) .where(eq(heartbeatRuns.id, context.runId)); if (change === "environment_lease") await db.insert(environmentLeases).values({ companyId: context.fixture.companyId, heartbeatRunId: context.runId, status: "released", }); if (change === "uncertain_publication") await db.insert(chatPublications).values({ companyId: context.fixture.companyId, endpointId: context.endpoint.id, conversationId: context.conversation.id, issueId: context.issue.id, state: "delivery_unknown", idempotencyKey: `run:${context.runId}:working:uncertain`, payload: { text: "Possibly sent" }, }); if (change === "selected_reply") { const [comment] = await db .insert(issueComments) .values({ companyId: context.fixture.companyId, issueId: context.issue.id, authorAgentId: context.fixture.assignedAgentId, authorType: "agent", createdByRunId: context.runId, body: "Already selected response", }) .returning(); await db.insert(chatPublications).values({ companyId: context.fixture.companyId, endpointId: context.endpoint.id, conversationId: context.conversation.id, issueId: context.issue.id, commentId: comment.id, state: "pending", idempotencyKey: `selected:${context.runId}`, payload: { text: comment.body }, }); } if (change === "revoked_resource") await db .update(chatEndpointResources) .set({ enabled: false }) .where( eq(chatEndpointResources.id, context.conversation.resourceId!), ); if (change === "retired_conversation") await db .update(chatConversations) .set({ state: "completed" }) .where(eq(chatConversations.id, context.conversation.id)); if (change === "edited_source") await db .update(issueComments) .set({ updatedAt: new Date(Date.now() + 1000) }) .where( eq(issueComments.id, String(context.action.payload.commentId)), ); await expect( db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user", }), ), ).rejects.toMatchObject({ details: { code: "chat_failed_run_retry_not_authorized" }, }); expect( await db .select() .from(chatActions) .where( and( eq(chatActions.companyId, context.fixture.companyId), eq(chatActions.kind, "failed_run_retry"), ), ), ).toEqual([]); } finally { await context.service.shutdown(); } }, ); it.each(["adapter_failed", "adapter_exit_code", "process_exit", "timeout"])( "admits the exact chat source for ordinary %s failure", async (errorCode) => { const context = await failedChatRetryFixture("telegram"); try { await db .update(heartbeatRuns) .set({ errorCode }) .where(eq(heartbeatRuns.id, context.runId)); await expect( db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user", }), ), ).resolves.toMatchObject({ issueId: context.issue.id }); } finally { await context.service.shutdown(); } }, ); it.each([ "native_event_replay_conflict", "native_session_cleanup_quarantined", "native_session_operator_recovery_required", "native_execution_ownership_unverified", "runner_state_identity_mismatch", "unknown", ])("does not bypass %s through an exact chat retry", async (errorCode) => { const context = await failedChatRetryFixture("telegram"); try { await db .update(heartbeatRuns) .set({ errorCode }) .where(eq(heartbeatRuns.id, context.runId)); await expect( db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user", }), ), ).rejects.toMatchObject({ details: { code: "chat_failed_run_retry_not_authorized" }, }); expect( await db .select() .from(chatActions) .where( and( eq(chatActions.companyId, context.fixture.companyId), eq(chatActions.kind, "failed_run_retry"), ), ), ).toEqual([]); } finally { await context.service.shutdown(); } }); it.each(["accepted_result", "unknown_delivery", "selected_answer"] as const)( "does not repeat a failed chat run with %s", async (kind) => { const context = await failedChatRetryFixture("telegram"); try { if (kind === "accepted_result") { const contractId = randomUUID(); await db.insert(completionContracts).values({ id: contractId, companyId: context.fixture.companyId, issueId: context.issue.id, revision: 1, schemaVersion: "1", policyVersion: "1", risk: "low", completionAuthority: "agent", incompleteCriteriaPolicy: "block", contractJson: {}, canonicalSha256: "a".repeat(64), createdByActorType: "system", createdByActorId: "test", }); await db .update(heartbeatRuns) .set({ nativeIssueId: context.issue.id, completionContractId: contractId, }) .where(eq(heartbeatRuns.id, context.runId)); await db.insert(nativeRunResults).values({ companyId: context.fixture.companyId, issueId: context.issue.id, runId: context.runId, completionContractId: contractId, serverFingerprint: "test-accepted", schemaStatus: "accepted", resultJson: {}, canonicalSha256: "b".repeat(64), }); } else { const commentId = kind === "selected_answer" ? randomUUID() : null; if (commentId) await db.insert(issueComments).values({ id: commentId, companyId: context.fixture.companyId, issueId: context.issue.id, authorType: "agent", authorAgentId: context.fixture.assignedAgentId, createdByRunId: context.runId, body: "Already selected answer", }); await db.insert(chatPublications).values({ companyId: context.fixture.companyId, endpointId: context.endpoint.id, conversationId: context.conversation.id, issueId: context.issue.id, commentId, idempotencyKey: `run:${context.runId}:working:${context.endpoint.id}`, payload: { text: "Prior output" }, state: kind === "unknown_delivery" ? "delivery_unknown" : "published", }); } await expect( db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user", }), ), ).rejects.toMatchObject({ details: { code: "chat_failed_run_retry_not_authorized" }, }); } finally { await context.service.shutdown(); } }, ); it("rehydrates exact chat retry authority and verifies its persisted projection after service restart", async () => { const context = await failedChatRetryFixture("telegram"); let restarted: ChatChannelService | undefined; try { const staged = await db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user", }), ); await context.service.shutdown(); restarted = createService(context.runtime).service; await restarted.processFailedChatRunRetry(staged.actionId); const [receipt] = await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, staged.actionId)); const authorize = (contextSnapshot: Record) => db.transaction((tx) => authorizeFailedChatRunRetryWake(db, tx as unknown as TestDb, { phase: "promotion", companyId: context.fixture.companyId, agentId: context.fixture.assignedAgentId, issueId: context.issue.id, wakeupRequestId: staged.actionId, contextSnapshot, }), ); await expect(authorize(receipt.payload!)).resolves.toBe(true); await expect( authorize({ ...receipt.payload, wakeCommentIds: [randomUUID()] }), ).rejects.toMatchObject({ details: { code: "chat_failed_run_retry_not_authorized" }, }); await expect( authorize({ ...receipt.payload, externalAttachmentOmissions: [] }), ).rejects.toMatchObject({ details: { code: "chat_failed_run_retry_not_authorized" }, }); await expect( authorize({ ...receipt.payload, forceFreshSession: true }), ).rejects.toMatchObject({ details: { code: "chat_failed_run_retry_not_authorized" }, }); } finally { await restarted?.shutdown(); await context.service.shutdown(); } }); async function safeNativeProgressFixture( provider: ChatProvider, suffix: string, teamsSurface: "channel" | "personal" = "channel", externalActorId?: string, linkedOriginalActor = false, githubUnavailableFile = false, ) { const fixture = await seedCompany(); const configured = provider === "slack" ? await configuredSlackEndpoint(fixture) : provider === "github" ? await configuredGitHubEndpoint( fixture, githubUnavailableFile ? { storage: createStorageService().storage } : {}, ) : provider === "discord" ? await configuredDiscordEndpoint(fixture) : provider === "microsoft-teams" ? await configuredTeamsEndpoint(fixture) : await configuredTelegramEndpoint(fixture); const { callbacks, endpoint, runtime, service } = configured; const telegramChatId = `77113${suffix.padStart(3, "0")}`; const teamsConversationId = `19:safe-progress-${suffix}@thread.${teamsSurface === "channel" ? "tacv2" : "v2"}`; const teamsServiceUrl = "https://smba.trafficmanager.net/amer/"; const messageId = provider === "slack" ? `1789000${suffix}.100001` : provider === "github" ? `9900${suffix}` : provider === "discord" ? `55555555555556${suffix.padStart(3, "0")}` : provider === "microsoft-teams" ? `17400000${suffix.padStart(5, "0")}` : `${telegramChatId}:101`; const thread = provider === "slack" ? makeThread({ channelId: `C-SAFE-PROGRESS-${suffix}`, id: `slack:C-SAFE-PROGRESS-${suffix}:${messageId}`, name: `safe-progress-${suffix}`, }) : provider === "github" ? makeThread({ channelId: "github:paperclipai/paperclip", id: `github:paperclipai/paperclip:issue:${700 + Number(suffix)}`, name: "paperclipai/paperclip", }) : provider === "discord" ? makeThread({ channelId: `33333333333334${suffix.padStart(3, "0")}`, id: `discord:1457808928258658549:33333333333334${suffix.padStart(3, "0")}:${messageId}`, name: `safe-progress-${suffix}`, }) : provider === "microsoft-teams" ? makeThread({ channelId: `teams:${Buffer.from(teamsConversationId).toString("base64url")}:${Buffer.from(teamsServiceUrl).toString("base64url")}`, id: `teams:${Buffer.from(`${teamsConversationId}${teamsSurface === "channel" ? `;messageid=${messageId}` : ""}`).toString("base64url")}:${Buffer.from(teamsServiceUrl).toString("base64url")}`, isDM: teamsSurface === "personal", name: `safe-progress-${suffix}`, }) : makeThread({ channelId: telegramChatId, id: `telegram:${telegramChatId}`, isDM: true, name: `safe-progress-${suffix}`, }); if (provider === "microsoft-teams" && teamsSurface === "channel") { await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "channel", providerResourceId: teamsConversationId, label: "Native progress channel", availability: "available", enabled: true, }); } if (linkedOriginalActor) { const currentEndpoint = await service.get(endpoint.id); const originalActorId = externalActorId ?? (provider === "telegram" ? telegramChatId : "U-SAFE-PROGRESS"); await db.insert(chatExternalPrincipals).values({ companyId: fixture.companyId, provider, providerAccountId: currentEndpoint.providerAccountId!, externalId: originalActorId, kind: "user", isBot: false, }); await linkLifecycleFixtureActor({ companyId: fixture.companyId, endpointId: endpoint.id, externalId: originalActorId, }); } const sourceMessage = makeMessage({ id: messageId, mentioned: !thread.thread.isDM, text: thread.thread.isDM ? "Show safe progress" : "@maya show safe progress", userId: externalActorId ?? (provider === "telegram" ? telegramChatId : "U-SAFE-PROGRESS"), }); if (githubUnavailableFile) { const url = "https://github.com/user-attachments/files/31967808/private-current.txt"; sourceMessage.text = `@maya inspect only this exact file: [file](${url})`; sourceMessage.threadId = thread.thread.id; sourceMessage.raw = { type: "issue_comment", threadType: "issue", prNumber: 700 + Number(suffix), repository: { full_name: "paperclipai/paperclip" }, comment: { id: Number(messageId), body: sourceMessage.text, user: { id: 42 }, }, }; sourceMessage.formatted = { type: "root", children: [{ type: "link", url, children: [] }], }; sourceMessage.attachments.push( ...githubPublicAttachmentsFromMessage(sourceMessage), ); } await deliverMessage({ callbacks, endpointId: endpoint.id, provider, thread: thread.thread, message: sourceMessage, trigger: thread.thread.isDM ? "direct_message" : "mention", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); if (!conversation) throw new Error("Expected safe-progress conversation"); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected safe-progress runtime"); const createRun = async (label: string) => { const runId = randomUUID(); const baseCreatedAt = new Date(Date.now() - 60_000); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, runtimeMode: "native", status: "running", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider, providerMessageId: messageId, }), }); await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${runId}:working:${endpoint.id}`, payload: { text: `Maya is working on ${label}…`, progressState: "working", }, state: "pending", createdAt: baseCreatedAt, updatedAt: baseCreatedAt, }); await service.processPendingPublications(100); return { baseCreatedAt, runId }; }; const addEvent = async ( run: Awaited>, eventType = "item.completed", seq = 1, createdAt = new Date(run.baseCreatedAt.getTime() + 30_000 + seq), ) => { await db.insert(heartbeatRunEvents).values({ companyId: fixture.companyId, runId: run.runId, agentId: fixture.assignedAgentId, seq, eventType, message: "PRIVATE native event prose must stay in Paperclip", payload: { toolName: "secret_internal_tool", arguments: { token: "PRIVATE-NATIVE-TOKEN" }, result: "PRIVATE-NATIVE-RESULT", target: "PRIVATE-NATIVE-TARGET", }, createdAt, }); }; return { addEvent, conversation, createRun, endpoint, fixture, messageId, providerRuntime, runtime, service, thread, wakeup: configured.wakeup, providerFetch: "providerFetch" in configured ? configured.providerFetch : undefined, }; } it.each(["issue", "run"] as const)( "skips a locked native progress %s while unrelated final and question publications deliver", async (lockTarget) => { const blocked = await safeNativeProgressFixture("telegram", "71"); const final = await safeNativeProgressFixture("telegram", "72"); const question = await safeNativeProgressFixture("telegram", "73"); const blockedRun = await blocked.createRun("contended progress"); const finalRun = await final.createRun("ready final"); const questionRun = await question.createRun("ready question"); await blocked.addEvent(blockedRun); const finalComment = await addSelectedChatFinal({ agentId: final.fixture.assignedAgentId, body: "The unrelated final is ready", companyId: final.fixture.companyId, issueId: final.conversation.issueId, runId: finalRun.runId, }); const interaction = await issueThreadInteractionService(db).create( { id: question.conversation.issueId, companyId: question.fixture.companyId, }, { kind: "ask_user_questions", continuationPolicy: "wake_assignee", sourceRunId: questionRun.runId, payload: { version: 1, prompt: "Choose a color", questions: [ { id: "color", prompt: "Choose a color", selectionMode: "single", required: true, options: [ { id: "amber", label: "Amber" }, { id: "cobalt", label: "Cobalt" }, ], }, ], }, }, { agentId: question.fixture.assignedAgentId }, ); let release!: () => void; let entered!: () => void; const held = new Promise((resolve) => { release = resolve; }); const acquired = new Promise((resolve) => { entered = resolve; }); const holder = db.transaction(async (tx) => { if (lockTarget === "issue") { await tx .select({ id: issues.id }) .from(issues) .where(eq(issues.id, blocked.conversation.issueId)) .for("update"); } else { await tx .select({ id: heartbeatRuns.id }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, blockedRun.runId)) .for("update"); } entered(); await held; }); let flush: Promise | undefined; let flushError: unknown; try { try { await acquired; flush = (async () => { await enqueueChatRunMilestones(db, { since: new Date(0) }); await blocked.service.processPendingPublications(100); })().catch((error) => { flushError = error; }); await vi.waitFor( async () => { if (flushError) throw flushError; const ready = await db .select() .from(chatPublications) .where( or( eq(chatPublications.commentId, finalComment.id), eq( chatPublications.idempotencyKey, `interaction:${interaction.id}:${question.endpoint.id}`, ), ), ); expect(ready).toHaveLength(2); expect( ready.every((publication) => publication.state === "published"), ).toBe(true); }, { // Keep the lock held until these exact receipts arrive. The // global sweep may still be settling unrelated fixture work; // its completion is not the nonblocking-delivery assertion. timeout: 5_000, }, ); expect( blocked.runtime.endpoints.get(final.endpoint.id)?.edits.at(-1) ?.text, ).toBe("The unrelated final is ready"); expect( blocked.runtime.endpoints.get(question.endpoint.id)?.edits.at(-1) ?.text, ).toContain("Choose a color"); expect(blocked.providerRuntime.edits).toEqual([]); } finally { release(); await Promise.allSettled([holder, flush]); } if (flushError) throw flushError; await expect( enqueueChatRunMilestones(db, { since: new Date(0) }), ).resolves.toBe(1); await blocked.service.processPendingPublications(100); expect(blocked.providerRuntime.edits).toEqual([ expect.objectContaining({ messageId: "outbound-1", text: "Maya is making progress…", }), ]); await expect( enqueueChatRunMilestones(db, { since: new Date(0) }), ).resolves.toBe(0); } finally { await Promise.allSettled([ blocked.service.shutdown(), final.service.shutdown(), question.service.shutdown(), ]); } }, ); it.each(["question", "final", "revoked"] as const)( "reauthorizes skipped native progress after %s wins before retry", async (winner) => { const context = await safeNativeProgressFixture("telegram", "74"); const run = await context.createRun("progress that must remain current"); await context.addEvent(run); let release!: () => void; let entered!: () => void; const held = new Promise((resolve) => { release = resolve; }); const acquired = new Promise((resolve) => { entered = resolve; }); const holder = db.transaction(async (tx) => { await tx .select({ id: issues.id }) .from(issues) .where(eq(issues.id, context.conversation.issueId)) .for("update"); entered(); await held; }); try { try { await acquired; await expect( enqueueChatRunMilestones(db, { since: new Date(0) }), ).resolves.toBe(0); } finally { release(); await holder; } if (winner === "question") { await issueThreadInteractionService(db).create( { id: context.conversation.issueId, companyId: context.fixture.companyId, }, { kind: "ask_user_questions", continuationPolicy: "wake_assignee", sourceRunId: run.runId, payload: { version: 1, prompt: "Choose a color", questions: [ { id: "color", prompt: "Choose a color", selectionMode: "single", required: true, options: [ { id: "amber", label: "Amber" }, { id: "cobalt", label: "Cobalt" }, ], }, ], }, }, { agentId: context.fixture.assignedAgentId }, ); } else if (winner === "final") { await addSelectedChatFinal({ agentId: context.fixture.assignedAgentId, body: "Current selected final", companyId: context.fixture.companyId, issueId: context.conversation.issueId, runId: run.runId, }); await db .update(heartbeatRuns) .set({ status: "succeeded", resultJson: { presentationDecision: { chosenSource: "existing_issue_comment", commentAction: "none", }, }, }) .where(eq(heartbeatRuns.id, run.runId)); } else { await db .update(chatEndpoints) .set({ allowDirectMessages: false, updatedAt: new Date() }) .where(eq(chatEndpoints.id, context.endpoint.id)); } await enqueueChatRunMilestones(db, { since: new Date(0) }); await context.service.processPendingPublications(100); expect( context.providerRuntime.edits.map((edit) => edit.text), ).not.toContain("Maya is making progress…"); const progress = await db .select() .from(chatPublications) .where( like( chatPublications.idempotencyKey, `run:${run.runId}:working:${context.endpoint.id}:native:%`, ), ); expect( progress.every((publication) => publication.state === "cancelled"), ).toBe(true); if (winner === "final") expect(context.providerRuntime.edits.at(-1)?.text).toBe( "Current selected final", ); if (winner === "question") expect(context.providerRuntime.edits.at(-1)?.text).toContain( "Choose a color", ); if (winner === "revoked") expect(context.providerRuntime.edits).toEqual([]); } finally { release(); await Promise.allSettled([holder, context.service.shutdown()]); } }, ); it("settles an interrupted native run once without overwriting its successor", async () => { const context = await safeNativeProgressFixture("telegram", "61"); try { const interrupted = await context.createRun("the original request"); const [originalWorking] = await db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `run:${interrupted.runId}:working:${context.endpoint.id}`, ), ); expect(originalWorking?.providerMessageId).toBeTruthy(); await db .update(heartbeatRuns) .set({ status: "interrupted", finishedAt: new Date(), errorCode: "server_shutdown_interrupted", error: "PRIVATE stdout tool arguments token=PRIVATE-INTERRUPTION-TOKEN", resultJson: { summary: "PRIVATE internal interruption summary" }, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, interrupted.runId)); // Model delayed milestone reconciliation after a successor has already // published its own working placeholder in the same conversation. const successor = await context.createRun("the successor request"); const [successorWorking] = await db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `run:${successor.runId}:working:${context.endpoint.id}`, ), ); expect(successorWorking?.providerMessageId).toBeTruthy(); expect(successorWorking?.providerMessageId).not.toBe( originalWorking?.providerMessageId, ); await expect(enqueueChatRunMilestones(db)).resolves.toBe(1); await context.service.processPendingPublications(100); await expect(enqueueChatRunMilestones(db)).resolves.toBe(0); await expect( context.service.processPendingPublications(100), ).resolves.toBe(0); expect(context.providerRuntime.posts).toHaveLength(2); expect(context.providerRuntime.edits).toEqual([ { threadId: context.thread.thread.id, messageId: originalWorking!.providerMessageId, text: "Maya stopped before completing this turn. Open the task in Paperclip for details.", }, ]); await addSelectedChatFinal({ agentId: context.fixture.assignedAgentId, body: "The successor's authoritative answer", companyId: context.fixture.companyId, issueId: context.conversation.issueId, runId: successor.runId, }); await db .update(heartbeatRuns) .set({ status: "succeeded", resultJson: { presentationDecision: { chosenSource: "existing_issue_comment", commentAction: "none", }, }, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, successor.runId)); await context.service.processPendingPublications(100); await expect(enqueueChatRunMilestones(db)).resolves.toBe(0); expect(context.providerRuntime.edits).toHaveLength(2); expect(context.providerRuntime.edits[1]).toEqual({ threadId: context.thread.thread.id, messageId: successorWorking!.providerMessageId, text: "The successor's authoritative answer", }); expect( JSON.stringify({ posts: context.providerRuntime.posts, edits: context.providerRuntime.edits, }), ).not.toMatch( /PRIVATE|stdout|tool arguments|server_shutdown_interrupted/, ); await expect( db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `run:${interrupted.runId}:failed:${context.endpoint.id}`, ), ), ).resolves.toEqual([ expect.objectContaining({ state: "published", providerMessageId: originalWorking!.providerMessageId, payload: { progressState: "failed", text: "Maya stopped before completing this turn. Open the task in Paperclip for details.", }, }), ]); await expect( db .select({ status: heartbeatRuns.status }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, interrupted.runId)), ).resolves.toEqual([{ status: "interrupted" }]); } finally { await context.service.shutdown(); } }); it.each(["pending", "published"] as const)( "preserves an interrupted run's selected final when its publication is %s", async (publicationState) => { const context = await safeNativeProgressFixture( "telegram", publicationState === "pending" ? "62" : "63", ); try { const run = await context.createRun("an already answered request"); const final = await addSelectedChatFinal({ agentId: context.fixture.assignedAgentId, body: "The selected final remains authoritative", companyId: context.fixture.companyId, issueId: context.conversation.issueId, runId: run.runId, }); if (publicationState === "published") await context.service.processPendingPublications(100); await db .update(heartbeatRuns) .set({ status: "interrupted", errorCode: "lease_released_before_terminal", finishedAt: new Date(), updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, run.runId)); await expect(enqueueChatRunMilestones(db, { limit: 1 })).resolves.toBe( 0, ); await context.service.processPendingPublications(100); await expect(enqueueChatRunMilestones(db)).resolves.toBe(0); expect(context.providerRuntime.posts).toHaveLength(1); expect(context.providerRuntime.edits).toEqual([ expect.objectContaining({ text: "The selected final remains authoritative", }), ]); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.commentId, final.id)), ).resolves.toEqual([{ state: "published" }]); } finally { await context.service.shutdown(); } }, ); it.each( (["slack", "telegram"] as const).flatMap((provider) => (["failed", "cancelled", "timed_out"] as const).flatMap((status) => (["published_final", "pending_final", "failure_first"] as const).map( (order) => ({ provider, status, order }), ), ), ), )( "preserves selected answer and failure lanes on $provider after $status ($order)", async ({ provider, status, order }) => { const context = await safeNativeProgressFixture(provider, "81"); try { const original = await context.createRun("the original request"); const successor = await context.createRun("the successor request"); const [originalWorking] = await db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `run:${original.runId}:working:${context.endpoint.id}`, ), ); const [successorWorking] = await db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `run:${successor.runId}:working:${context.endpoint.id}`, ), ); const addFinal = () => addSelectedChatFinal({ agentId: context.fixture.assignedAgentId, body: "The original selected answer remains available", companyId: context.fixture.companyId, issueId: context.conversation.issueId, runId: original.runId, }); let final = order === "failure_first" ? null : await addFinal(); if (order === "published_final") await context.service.processPendingPublications(100); await db .update(heartbeatRuns) .set({ status, errorCode: "adapter_failed", error: "PRIVATE internal error token=do-not-publish", finishedAt: new Date(), updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, original.runId)); await enqueueChatRunMilestones(db); if (order === "failure_first") { await context.service.processPendingPublications(100); final = await addFinal(); } await context.service.processPendingPublications(100); const [answerPublication] = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, final!.id)); const [failurePublication] = await db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `run:${original.runId}:failed:${context.endpoint.id}`, ), ); expect(answerPublication?.state).toBe("published"); expect(failurePublication?.state).toBe("published"); expect(answerPublication?.providerMessageId).not.toBe( failurePublication?.providerMessageId, ); expect( order === "failure_first" ? failurePublication?.providerMessageId : answerPublication?.providerMessageId, ).toBe(originalWorking!.providerMessageId); expect(context.providerRuntime.posts).toHaveLength(3); expect(context.providerRuntime.edits).toHaveLength(1); expect(context.providerRuntime.edits[0]?.messageId).toBe( originalWorking!.providerMessageId, ); expect(context.providerRuntime.edits[0]?.messageId).not.toBe( successorWorking!.providerMessageId, ); const links = await db .select({ providerMessageId: chatMessageLinks.providerMessageId, publicationId: chatMessageLinks.publicationId, commentId: chatMessageLinks.commentId, }) .from(chatMessageLinks) .where( and( eq(chatMessageLinks.conversationId, context.conversation.id), eq(chatMessageLinks.direction, "outbound"), ), ); expect(links).toHaveLength(3); expect(links).toEqual( expect.arrayContaining([ { providerMessageId: answerPublication!.providerMessageId, publicationId: answerPublication!.id, commentId: final!.id, }, { providerMessageId: failurePublication!.providerMessageId, publicationId: failurePublication!.id, commentId: null, }, { providerMessageId: successorWorking!.providerMessageId, publicationId: successorWorking!.id, commentId: null, }, ]), ); expect(JSON.stringify(context.providerRuntime.posts)).not.toMatch( /PRIVATE|adapter_failed|do-not-publish/, ); await enqueueChatRunMilestones(db); await context.service.processPendingPublications(100); expect(context.providerRuntime.posts).toHaveLength(3); expect(context.providerRuntime.edits).toHaveLength(1); await expect( db .select({ status: heartbeatRuns.status }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, original.runId)), ).resolves.toEqual([{ status }]); } finally { await context.service.shutdown(); } }, ); it.each([ ["slack", "1", "channel"], ["github", "2", "channel"], ["discord", "3", "channel"], ["telegram", "4", "personal"], ["microsoft-teams", "5", "channel"], ["microsoft-teams", "6", "personal"], ] as const)( "coalesces closed native progress on %s fixture %s (%s) without projecting event content", async (provider, suffix, surface) => { // The collector is global. A provider-filtered run may leave a failed // run's milestone from an earlier fixture that the full suite already // drained. Settle that work before asserting this fixture's exact count. await enqueueChatRunMilestones(db, { since: new Date(0) }); const context = await safeNativeProgressFixture( provider, suffix, surface, ); try { const run = await context.createRun(`${provider} work`); await context.addEvent(run); // A future event name that merely shares an allowlisted prefix must not // become provider authority or replace the latest exact event. await context.addEvent(run, "item.completed.private-extension", 2); await expect( enqueueChatRunMilestones(db, { since: new Date(0) }), ).resolves.toBe(1); await context.service.processPendingPublications(100); expect(context.providerRuntime.posts).toEqual([ { threadId: context.thread.thread.id, text: `Maya is working on ${provider} work…`, }, ]); expect(context.providerRuntime.edits).toEqual([ { threadId: context.thread.thread.id, messageId: "outbound-1", text: "Maya is making progress…", }, ]); expect( JSON.stringify({ edits: context.providerRuntime.edits, posts: context.providerRuntime.posts, }), ).not.toMatch(/PRIVATE|secret_internal_tool/); const progressRows = await db .select() .from(chatPublications) .where( like( chatPublications.idempotencyKey, `run:${run.runId}:working:${context.endpoint.id}:native:%`, ), ); expect(progressRows).toEqual([ expect.objectContaining({ idempotencyKey: `run:${run.runId}:working:${context.endpoint.id}:native:making_progress:1`, payload: { text: "Maya is making progress…", progressState: "working", }, providerMessageId: "outbound-1", state: "published", }), ]); await expect( enqueueChatRunMilestones(db, { since: new Date(0) }), ).resolves.toBe(0); await expect( context.service.processPendingPublications(100), ).resolves.toBe(0); expect(context.providerRuntime.posts).toHaveLength(1); expect(context.providerRuntime.edits).toHaveLength(1); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }, ); it.each([ ["slack", "91", "channel"], ["github", "92", "channel"], ["discord", "93", "channel"], ["telegram", "94", "personal"], ["microsoft-teams", "95", "channel"], ["microsoft-teams", "96", "personal"], ] as const)( "keeps provider startup diagnostics private on %s fixture %s (%s)", async (provider, suffix, surface) => { const context = await safeNativeProgressFixture( provider, suffix, surface, ); try { const run = await context.createRun("startup recovery"); // This is a publication-boundary fixture, not proof that a provider // startup receipt is authentic or that any process has retired. const phases = ["intent", "spawned", "initialization_failed"]; await db.insert(heartbeatRunEvents).values( phases.map((phase, index) => ({ companyId: context.fixture.companyId, agentId: context.fixture.assignedAgentId, runId: run.runId, seq: index + 1, eventType: "harness.diagnostic", message: "PRIVATE provider startup evidence", payload: { code: "provider_startup_ownership", startup: { schema: "paperclip.provider_startup.v1", phase, launchId: "PRIVATE-STARTUP-LAUNCH", requestedThreadId: "PRIVATE-REQUESTED-THREAD", authenticatedThreadId: null, directChildExitObserved: phase === "initialization_failed", }, }, createdAt: new Date(run.baseCreatedAt.getTime() + 30_000 + index), })), ); await expect( enqueueChatRunMilestones(db, { since: new Date(0) }), ).resolves.toBe(0); await expect( context.service.processPendingPublications(100), ).resolves.toBe(0); expect(context.providerRuntime.posts).toHaveLength(1); expect(context.providerRuntime.edits).toHaveLength(0); // Private diagnostics must neither generate progress nor suppress a // later real, allowlisted progress event in the same run. await context.addEvent(run, "item.completed", 4); await expect( enqueueChatRunMilestones(db, { since: new Date(0) }), ).resolves.toBe(1); await context.service.processPendingPublications(100); expect(context.providerRuntime.posts).toHaveLength(1); expect(context.providerRuntime.edits).toEqual([ { threadId: context.thread.thread.id, messageId: "outbound-1", text: "Maya is making progress…", }, ]); const publications = await db .select({ payload: chatPublications.payload }) .from(chatPublications) .where(eq(chatPublications.conversationId, context.conversation.id)); expect(JSON.stringify(publications)).not.toMatch( /PRIVATE|provider_startup|harness\.diagnostic|launchId|requestedThreadId|directChildExitObserved/, ); } finally { await context.service.shutdown(); } }, ); it("enforces the native progress cadence boundary and one publication per phase", async () => { const context = await safeNativeProgressFixture("telegram", "8"); const run = await context.createRun("cadence boundaries"); const at = (offsetMs: number) => new Date(run.baseCreatedAt.getTime() + offsetMs); await context.addEvent(run, "research.progressed", 1, at(19_999)); await expect( enqueueChatRunMilestones(db, { since: new Date(0) }), ).resolves.toBe(0); await context.addEvent(run, "research.completed", 2, at(20_000)); await expect( enqueueChatRunMilestones(db, { since: new Date(0) }), ).resolves.toBe(1); const [researchPublication] = await db .select({ id: chatPublications.id }) .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `run:${run.runId}:working:${context.endpoint.id}:native:researching:2`, ), ); if (!researchPublication) { throw new Error("Expected research progress publication"); } // Model a prior sweep at the event boundary so the next phase can exercise // the same exact cadence without waiting on wall-clock time. await db .update(chatPublications) .set({ createdAt: at(20_000), updatedAt: at(20_000) }) .where(eq(chatPublications.id, researchPublication.id)); await context.addEvent(run, "tool.execution.started", 3, at(39_999)); await expect( enqueueChatRunMilestones(db, { since: new Date(0) }), ).resolves.toBe(0); await context.addEvent(run, "tool.execution.completed", 4, at(40_000)); await expect( enqueueChatRunMilestones(db, { since: new Date(0) }), ).resolves.toBe(1); await context.addEvent(run, "tool.execution.progressed", 5, at(60_000)); await expect( enqueueChatRunMilestones(db, { since: new Date(0) }), ).resolves.toBe(0); await expect( db .select({ id: chatPublications.id }) .from(chatPublications) .where( like( chatPublications.idempotencyKey, `run:${run.runId}:working:${context.endpoint.id}:native:%`, ), ), ).resolves.toHaveLength(2); await context.service.shutdown(); }); it("does not replay stale native progress after a paused endpoint restarts behind an exact final", async () => { const context = await safeNativeProgressFixture("telegram", "7"); await qualifySetupRoundTrip( context.service, context.endpoint.id, context.thread.thread.channelId, ); await context.service.test(context.endpoint.id, "owner-user"); const run = await context.createRun("paused restart"); const workingProviderMessageId = await db .select({ providerMessageId: chatPublications.providerMessageId }) .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `run:${run.runId}:working:${context.endpoint.id}`, ), ) .then((rows) => rows[0]?.providerMessageId ?? null); if (!workingProviderMessageId) { throw new Error("Expected persisted working publication identity"); } await context.addEvent(run, "tool.execution.completed"); await expect( enqueueChatRunMilestones(db, { since: new Date(0) }), ).resolves.toBe(1); await context.service.configure( context.endpoint.id, { action: "pause" }, "owner-user", ); const finalComment = await addSelectedChatFinal({ agentId: context.fixture.assignedAgentId, body: "Authoritative final after restart", companyId: context.fixture.companyId, issueId: context.conversation.issueId, runId: run.runId, }); await db .update(heartbeatRuns) .set({ status: "succeeded", resultJson: { presentationDecision: { chosenSource: "existing_issue_comment", commentAction: "none", }, }, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, run.runId)); const telegramBotId = await db .select({ botExternalId: chatEndpoints.botExternalId }) .from(chatEndpoints) .where(eq(chatEndpoints.id, context.endpoint.id)) .then((rows) => rows[0]?.botExternalId ?? null); if (!telegramBotId) throw new Error("Expected persisted Telegram bot id"); await context.service.shutdown(); const restartedRuntime = new FakeChatSdkRuntime(); const restarted = createService( restartedRuntime, fakeTelegramFetch(Number(telegramBotId)), ); await restarted.service.configure( context.endpoint.id, { action: "resume" }, "owner-user", ); await expect( restarted.service.processPendingPublications(100), ).resolves.toBe(2); const endpointRuntime = restartedRuntime.endpoints.get(context.endpoint.id); expect(endpointRuntime?.posts).toEqual([]); expect(endpointRuntime?.edits).toEqual([ { threadId: context.thread.thread.id, messageId: workingProviderMessageId, text: "Authoritative final after restart", }, ]); const runPublications = await db .select({ idempotencyKey: chatPublications.idempotencyKey, redactedError: chatPublications.redactedError, state: chatPublications.state, }) .from(chatPublications) .where( and( eq(chatPublications.endpointId, context.endpoint.id), like(chatPublications.idempotencyKey, `run:${run.runId}:%`), ), ); expect(runPublications).toEqual( expect.arrayContaining([ expect.objectContaining({ idempotencyKey: `run:${run.runId}:working:${context.endpoint.id}:native:using_tools:1`, redactedError: "Run reached a terminal state before progress delivery", state: "cancelled", }), ]), ); await expect( db .select({ providerMessageId: chatPublications.providerMessageId, state: chatPublications.state, }) .from(chatPublications) .where(eq(chatPublications.commentId, finalComment.id)), ).resolves.toEqual([ { providerMessageId: workingProviderMessageId, state: "published" }, ]); await restarted.service.shutdown(); }); it("suppresses queued and working placeholders that terminalize while paused", async () => { const context = await safeNativeProgressFixture("telegram", "6"); await qualifySetupRoundTrip( context.service, context.endpoint.id, context.thread.thread.channelId, ); await context.service.test(context.endpoint.id, "owner-user"); const runs = await Promise.all( (["queued", "working"] as const).map(async (progressState, index) => { const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: context.fixture.companyId, agentId: context.fixture.assignedAgentId, runtimeMode: "native", status: "running", contextSnapshot: await chatWakeContext({ endpointId: context.endpoint.id, issueId: context.conversation.issueId, provider: "telegram", providerMessageId: context.messageId, }), }); await db.insert(chatPublications).values({ companyId: context.fixture.companyId, endpointId: context.endpoint.id, conversationId: context.conversation.id, issueId: context.conversation.issueId, idempotencyKey: `run:${runId}:${progressState}:${context.endpoint.id}`, payload: { text: `Maya is ${progressState}…`, progressState, }, state: "pending", createdAt: new Date(Date.now() - 60_000 + index), }); return { progressState, runId }; }), ); await context.service.configure( context.endpoint.id, { action: "pause" }, "owner-user", ); const finalCommentIds: string[] = []; for (const [index, run] of runs.entries()) { const finalComment = await addSelectedChatFinal({ agentId: context.fixture.assignedAgentId, body: `Final response ${index + 1}`, companyId: context.fixture.companyId, issueId: context.conversation.issueId, runId: run.runId, }); finalCommentIds.push(finalComment.id); await db .update(heartbeatRuns) .set({ status: "succeeded", resultJson: { presentationDecision: { chosenSource: "existing_issue_comment", commentAction: "none", }, }, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, run.runId)); } const telegramBotId = await db .select({ botExternalId: chatEndpoints.botExternalId }) .from(chatEndpoints) .where(eq(chatEndpoints.id, context.endpoint.id)) .then((rows) => rows[0]?.botExternalId ?? null); if (!telegramBotId) throw new Error("Expected persisted Telegram bot id"); await context.service.shutdown(); const restartedRuntime = new FakeChatSdkRuntime(); const restarted = createService( restartedRuntime, fakeTelegramFetch(Number(telegramBotId)), ); await restarted.service.configure( context.endpoint.id, { action: "resume" }, "owner-user", ); await expect( restarted.service.processPendingPublications(100), ).resolves.toBe(4); const endpointRuntime = restartedRuntime.endpoints.get(context.endpoint.id); expect(endpointRuntime?.posts.map((post) => post.text)).toEqual([ "Final response 1", "Final response 2", ]); expect(endpointRuntime?.edits).toEqual([]); const cancelledProgress = await db .select({ idempotencyKey: chatPublications.idempotencyKey, redactedError: chatPublications.redactedError, state: chatPublications.state, }) .from(chatPublications) .where( inArray( chatPublications.idempotencyKey, runs.map( (run) => `run:${run.runId}:${run.progressState}:${context.endpoint.id}`, ), ), ); expect(cancelledProgress).toHaveLength(runs.length); expect(cancelledProgress).toEqual( expect.arrayContaining( runs.map((run) => ({ idempotencyKey: `run:${run.runId}:${run.progressState}:${context.endpoint.id}`, redactedError: "Run reached a terminal state before progress delivery", state: "cancelled", })), ), ); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(inArray(chatPublications.commentId, finalCommentIds)), ).resolves.toEqual([{ state: "published" }, { state: "published" }]); await restarted.service.shutdown(); }); it.each([ ["telegram", "9", "personal"], ["microsoft-teams", "10", "personal"], ["microsoft-teams", "11", "channel"], ] as const)( "keeps native progress behind question, final, and current reach authority on %s fixture %s (%s)", async (provider, suffix, surface) => { const context = await safeNativeProgressFixture( provider, suffix, surface, ); const setReachEnabled = async (enabled: boolean) => { if (context.conversation.isDirectMessage) { await db .update(chatEndpoints) .set({ allowDirectMessages: enabled, updatedAt: new Date() }) .where(eq(chatEndpoints.id, context.endpoint.id)); } else { expect(context.conversation.resourceId).toBeTruthy(); await db .update(chatEndpointResources) .set({ enabled, updatedAt: new Date() }) .where( and( eq(chatEndpointResources.endpointId, context.endpoint.id), eq(chatEndpointResources.id, context.conversation.resourceId!), ), ); } }; const questionRun = await context.createRun("a question"); await context.addEvent(questionRun); await db.insert(issueThreadInteractions).values({ companyId: context.fixture.companyId, issueId: context.conversation.issueId, kind: "ask_user_questions", continuationPolicy: "wake_assignee", createdByAgentId: context.fixture.assignedAgentId, sourceRunId: questionRun.runId, status: "pending", payload: { version: 1, prompt: "Choose one", questions: [ { id: "choice", prompt: "Choose one", options: [ { id: "a", label: "A" }, { id: "b", label: "B" }, ], }, ], }, }); await expect( enqueueChatRunMilestones(db, { since: new Date(0) }), ).resolves.toBe(0); const revokedRun = await context.createRun("revoked reach"); await context.addEvent(revokedRun); await expect( enqueueChatRunMilestones(db, { since: new Date(0) }), ).resolves.toBe(1); await setReachEnabled(false); const postsBeforeRevocation = context.providerRuntime.posts.length; const editsBeforeRevocation = context.providerRuntime.edits.length; await context.service.processPendingPublications(100); expect(context.providerRuntime.posts).toHaveLength(postsBeforeRevocation); expect(context.providerRuntime.edits).toHaveLength(editsBeforeRevocation); const [revokedProgress] = await db .select() .from(chatPublications) .where( like( chatPublications.idempotencyKey, `run:${revokedRun.runId}:working:${context.endpoint.id}:native:%`, ), ); expect(revokedProgress).toMatchObject({ state: "cancelled", attempts: 1, }); await setReachEnabled(true); const finalRun = await context.createRun("a final answer"); await context.addEvent(finalRun); await expect( enqueueChatRunMilestones(db, { since: new Date(0) }), ).resolves.toBe(1); const finalComment = await addSelectedChatFinal({ agentId: context.fixture.assignedAgentId, body: "Authoritative final answer", companyId: context.fixture.companyId, issueId: context.conversation.issueId, runId: finalRun.runId, }); await db .update(heartbeatRuns) .set({ status: "succeeded", resultJson: { presentationDecision: { chosenSource: "existing_issue_comment", commentAction: "none", }, }, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, finalRun.runId)); await context.service.processPendingPublications(100); expect(context.providerRuntime.edits.at(-1)).toEqual({ threadId: context.thread.thread.id, messageId: expect.any(String), text: "Authoritative final answer", }); expect(JSON.stringify(context.providerRuntime.edits)).not.toContain( "PRIVATE native event prose", ); const finalPublications = await db .select({ idempotencyKey: chatPublications.idempotencyKey, state: chatPublications.state, }) .from(chatPublications) .where( and( eq(chatPublications.endpointId, context.endpoint.id), like(chatPublications.idempotencyKey, `run:${finalRun.runId}:%`), ), ); expect(finalPublications).toContainEqual( expect.objectContaining({ idempotencyKey: expect.stringMatching( new RegExp( `^run:${finalRun.runId}:working:${context.endpoint.id}:native:`, ), ), state: "cancelled", }), ); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.commentId, finalComment.id)), ).resolves.toEqual([{ state: "published" }]); await context.service.shutdown(); }, ); it("coalesces one Telegram run into one provider message", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture); const chatId = "77112236"; const dm = makeThread({ channelId: `telegram:${chatId}`, id: `telegram:${chatId}`, isDM: true, name: "Telegram direct message", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: `${chatId}:71`, text: "Produce one quiet Telegram response", userId: chatId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, chatId); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); if (!conversation) throw new Error("Expected Telegram conversation"); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "telegram", providerMessageId: `${chatId}:71`, }), }); for (const progressState of ["queued", "working"] as const) { await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${runId}:${progressState}:${endpoint.id}`, payload: { text: progressState === "queued" ? "Maya is queued." : "Maya is working…", progressState, }, state: "pending", }); await service.processPendingPublications(); } await addSelectedChatFinal({ agentId: fixture.assignedAgentId, body: "Final Telegram result", companyId: fixture.companyId, issueId: conversation.issueId, runId, }); await service.processPendingPublications(); const providerRuntime = runtime.endpoints.get(endpoint.id); expect(providerRuntime?.posts).toEqual([ { threadId: dm.thread.id, text: "Maya is queued." }, ]); expect(providerRuntime?.edits).toEqual([ { threadId: dm.thread.id, messageId: "outbound-2", text: "Maya is working…", }, { threadId: dm.thread.id, messageId: "outbound-2", text: "Final Telegram result", }, ]); }); it("audits Telegram reactions on completed DM generations idempotently without comments or runs", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service, wakeup } = await configuredTelegramEndpoint(fixture); const chatId = "77112237"; const dm = makeThread({ channelId: `telegram:${chatId}`, id: `telegram:${chatId}`, isDM: true, name: "Telegram direct message", }); const original = makeMessage({ id: `${chatId}:81`, text: "Observe Telegram reactions", userId: chatId, }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: original, trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, chatId); await service.test(endpoint.id, "owner-user"); if (!callbacks.onReaction) throw new Error("Telegram reaction callback was not registered"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); await db .update(issues) .set({ status: "done", completedAt: new Date(), updatedAt: new Date() }) .where(eq(issues.id, conversation.issueId)); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: `${chatId}:82`, text: "Start the next Telegram DM task", userId: chatId, }), trigger: "direct_message", }); const conversations = await service.listConversations(endpoint.id); expect(conversations).toHaveLength(2); expect( conversations.find((row) => row.id === conversation.id), ).toMatchObject({ state: "completed", sessionGeneration: 1 }); const newerConversation = conversations.find( (row) => row.id !== conversation.id, )!; expect(newerConversation).toMatchObject({ state: "active", sessionGeneration: 2, }); const commentCount = await db .select() .from(issueComments) .where(eq(issueComments.companyId, fixture.companyId)) .then((rows) => rows.length); const wakeupCount = wakeup.mock.calls.length; const runCount = await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.companyId, fixture.companyId)) .then((rows) => rows.length); const emoji = { name: "thumbs_up", toJSON: () => "👍", toString: () => "👍", }; const reaction = (added: boolean, updateId: number) => ({ endpointId: endpoint.id, provider: "telegram" as const, event: { adapter: {} as never, added, emoji, message: original, messageId: original.id, raw: { update_id: updateId }, rawEmoji: "👍", thread: dm.thread, threadId: dm.thread.id, user: original.author, }, }); await callbacks.onReaction(reaction(true, 8_001)); await callbacks.onReaction(reaction(true, 8_001)); await callbacks.onReaction(reaction(false, 8_002)); await callbacks.onReaction(reaction(true, 8_003)); await callbacks.onReaction(reaction(false, 8_004)); const foreignThreadReaction = reaction(true, 8_005); foreignThreadReaction.event.threadId = "telegram:77112238"; await callbacks.onReaction(foreignThreadReaction); await service.update( endpoint.id, { allowDirectMessages: false }, "owner-user", ); await callbacks.onReaction(reaction(true, 8_006)); const reactions = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.conversationId, conversation.id)) .then((rows) => rows.filter((row) => row.eventKind.startsWith("reaction_")), ); expect(reactions).toHaveLength(4); expect(reactions.map((row) => row.eventKind).sort()).toEqual([ "reaction_added", "reaction_added", "reaction_removed", "reaction_removed", ]); expect(reactions.every((row) => row.state === "processed")).toBe(true); await expect( db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.conversationId, newerConversation.id), inArray(chatDeliveries.eventKind, [ "reaction_added", "reaction_removed", ]), ), ), ).resolves.toHaveLength(0); expect( await db .select() .from(issueComments) .where(eq(issueComments.companyId, fixture.companyId)) .then((rows) => rows.length), ).toBe(commentCount); expect(wakeup).toHaveBeenCalledTimes(wakeupCount); expect( await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.companyId, fixture.companyId)) .then((rows) => rows.length), ).toBe(runCount); }); it("audits reactions on linked messages without treating them as task instructions", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service, wakeup } = await configuredSlackEndpoint(fixture); const channel = makeThread({ channelId: "C-REACTIONS", id: "slack:C-REACTIONS:7100.1", name: "reactions", }); const original = makeMessage({ id: "7100.1", text: "@maya observe reactions", mentioned: true, }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: original, trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); if (!callbacks.onReaction) throw new Error("Slack reaction callback was not registered"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const commentCountBefore = await db .select() .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)) .then((rows) => rows.length); const wakeupCountBefore = wakeup.mock.calls.length; const emoji = { name: "thumbs_up", toJSON: () => ":thumbs_up:", toString: () => ":thumbs_up:", }; const reaction = (added: boolean, eventTs: string) => ({ endpointId: endpoint.id, provider: "slack" as const, event: { adapter: {} as never, added, emoji, message: original, messageId: original.id, raw: { event_ts: eventTs }, rawEmoji: "+1", thread: channel.thread, threadId: channel.thread.id, user: original.author, }, }); await callbacks.onReaction(reaction(true, "7101.1")); await callbacks.onReaction(reaction(true, "7101.1")); await callbacks.onReaction(reaction(false, "7102.1")); const reactions = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.conversationId, conversation.id)) .then((rows) => rows.filter((row) => row.eventKind.startsWith("reaction_")), ); expect(reactions).toHaveLength(2); expect(reactions.map((row) => row.eventKind).sort()).toEqual([ "reaction_added", "reaction_removed", ]); expect(reactions[0]?.normalizedEvent).toMatchObject({ reaction: { emoji: "thumbs_up", rawEmoji: "+1" }, }); expect( await db .select() .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)) .then((rows) => rows.length), ).toBe(commentCountBefore); expect(wakeup.mock.calls).toHaveLength(wakeupCountBefore); expect( (await service.listActivity(endpoint.id)).filter((item) => item.summary.startsWith("reaction "), ), ).toHaveLength(2); }); it("rechecks the outbound link when publication commits between reaction preflight reads", async () => { const fixture = await seedCompany(); let releasePreflight!: () => void; const preflightRelease = new Promise((resolve) => { releasePreflight = resolve; }); let reachedPreflight!: () => void; const preflightReached = new Promise((resolve) => { reachedPreflight = resolve; }); const { callbacks, endpoint, runtime, service, wakeup } = await configuredSlackEndpoint(fixture, { reactionLinkPreflightBarrier: async () => { reachedPreflight(); await preflightRelease; }, }); const channel = makeThread({ channelId: "C-REACTION-PREFLIGHT", id: "slack:C-REACTION-PREFLIGHT:7125.1", name: "reaction-preflight", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "7125.1", text: "@maya send a reply for the preflight race", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); if (!callbacks.onReaction) throw new Error("Slack reaction callback was not registered"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Slack conversation"); const wakeupCount = wakeup.mock.calls.length; const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack endpoint runtime"); const targetMessageId = "7125.2"; const targetMessage = makeMessage({ id: targetMessageId, text: "" }); const reaction = { endpointId: endpoint.id, provider: "slack" as const, event: { adapter: {} as never, added: true, emoji: { name: "thumbs_up", toJSON: () => ":thumbs_up:", toString: () => ":thumbs_up:", }, message: targetMessage, messageId: targetMessageId, raw: { event_ts: "7125.25" }, rawEmoji: "+1", thread: channel.thread, threadId: channel.thread.id, user: targetMessage.author, }, }; let reactionPromise: Promise | null = null; providerRuntime.postResultIds.push(targetMessageId); providerRuntime.postHook = async () => { if (reactionPromise) return; reactionPromise = callbacks.onReaction!(reaction); await preflightReached; // Return to the provider send while the callback is between its first // exact-link lookup and its streaming lookup. }; await service.publishBoardMessage( endpoint.id, conversation.id, "The preflight race reply", "reaction-preflight-link-commit", "owner-user", ); providerRuntime.postHook = undefined; releasePreflight(); await reactionPromise; await expect( db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "reaction_added"), ), ), ).resolves.toEqual([ expect.objectContaining({ conversationId: conversation.id, state: "processed", }), ]); expect(wakeup).toHaveBeenCalledTimes(wakeupCount); }); it("durably replays a reaction that arrives before its outbound link without waking the task", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredSlackEndpoint(fixture); const channel = makeThread({ channelId: "C-EARLY-REACTION", id: "slack:C-EARLY-REACTION:7130.1", name: "early-reaction", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "7130.1", text: "@maya prepare an outbound reply", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); if (!callbacks.onReaction) throw new Error("Slack reaction callback was not registered"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Slack conversation"); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack endpoint runtime"); const targetMessageId = "7130.2"; const targetMessage = makeMessage({ id: targetMessageId, text: "", mentioned: false, }); const reaction = (messageId: string, eventTs: string) => ({ endpointId: endpoint.id, provider: "slack" as const, event: { adapter: {} as never, added: true, emoji: { name: "thumbs_up", toJSON: () => ":thumbs_up:", toString: () => ":thumbs_up:", }, message: { ...targetMessage, id: messageId }, messageId, raw: { event_ts: eventTs }, rawEmoji: "+1", thread: channel.thread, threadId: channel.thread.id, user: targetMessage.author, }, }); let stagedId: string | null = null; let intercepted = false; providerRuntime.postResultIds.push(targetMessageId); providerRuntime.postHook = async () => { if (intercepted) return; intercepted = true; const event = reaction(targetMessageId, "7130.25"); await callbacks.onReaction!(event); await callbacks.onReaction!(event); const staged = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "reaction_added"), ), ); expect(staged).toHaveLength(1); expect(staged[0]).toMatchObject({ conversationId: null, state: "received", attempts: 0, normalizedEvent: expect.objectContaining({ conversation: { externalThreadId: channel.thread.id }, message: { providerMessageId: targetMessageId }, runtimeContext: { generation: expect.any(Number), credentialFingerprint: expect.any(String), }, }), }); stagedId = staged[0]!.id; }; await service.publishBoardMessage( endpoint.id, conversation.id, "A provider-visible reply", "early-reaction-before-link", "owner-user", ); providerRuntime.postHook = undefined; if (!stagedId) throw new Error("Expected a staged reaction delivery"); // An arbitrary message in the same thread is not plausible once no send // is in flight, so it does not even create a durable reaction row. await callbacks.onReaction(reaction("7130.unknown", "7130.3")); await expect( db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "reaction_added"), ), ), ).resolves.toHaveLength(1); const commentCount = await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)) .then((rows) => rows.length); const issueCount = await db .select({ id: issues.id }) .from(issues) .where(eq(issues.companyId, fixture.companyId)) .then((rows) => rows.length); const runCount = await db .select({ id: heartbeatRuns.id }) .from(heartbeatRuns) .where(eq(heartbeatRuns.companyId, fixture.companyId)) .then((rows) => rows.length); const wakeupCount = wakeup.mock.calls.length; await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, stagedId)); const restarted = createService(); await restarted.service.processPendingDeliveries(1, stagedId); const [replayed] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, stagedId)); expect(replayed).toMatchObject({ conversationId: conversation.id, state: "processed", attempts: 1, nextAttemptAt: null, redactedError: null, }); // The provider's exact retry after the link commit is the same immutable // event and therefore neither duplicates nor replays task work. await callbacks.onReaction(reaction(targetMessageId, "7130.25")); await expect( db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "reaction_added"), ), ), ).resolves.toHaveLength(1); await expect( db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)) .then((rows) => rows.length), ).resolves.toBe(commentCount); await expect( db .select({ id: issues.id }) .from(issues) .where(eq(issues.companyId, fixture.companyId)) .then((rows) => rows.length), ).resolves.toBe(issueCount); await expect( db .select({ id: heartbeatRuns.id }) .from(heartbeatRuns) .where(eq(heartbeatRuns.companyId, fixture.companyId)) .then((rows) => rows.length), ).resolves.toBe(runCount); expect(wakeup).toHaveBeenCalledTimes(wakeupCount); expect(restarted.wakeup).not.toHaveBeenCalled(); await restarted.service.shutdown(); }); it("replays one pre-link Discord reaction after service reconstruction without waking the task", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredDiscordEndpoint(fixture); const guildId = "1457808928258658549"; const channelId = "333333333333333391"; const rootMessageId = "555555555555555691"; const channel = makeThread({ channelId, id: `discord:${guildId}:${channelId}:${rootMessageId}`, name: "discord-reaction-restart", }); const rootMessage = makeMessage({ id: rootMessageId, mentioned: true, text: "@maya prepare a Discord reply before service reconstruction", userId: "444444444444444491", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "discord", thread: channel.thread, message: rootMessage, trigger: "mention", }); await qualifySetupRoundTrip( service, endpoint.id, rootMessage.author.userId, ); await service.test(endpoint.id, "owner-user"); if (!callbacks.onReaction) throw new Error("Discord reaction callback was not registered"); const [conversation] = await service.listConversations(endpoint.id); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!conversation || !providerRuntime) throw new Error("Expected an active Discord conversation"); const targetMessageId = "555555555555555692"; const targetMessage = makeMessage({ id: targetMessageId, text: "", userId: rootMessage.author.userId, }); const reaction = { endpointId: endpoint.id, provider: "discord" as const, event: { adapter: {} as never, added: true, emoji: { name: "thumbsup", toJSON: () => "👍", toString: () => "👍", }, message: targetMessage, messageId: targetMessageId, raw: { channel_id: channelId, emoji: { id: null, name: "👍" }, gateway_dispatch: { eventType: "MESSAGE_REACTION_ADD", sequence: 891, sessionFingerprint: "f".repeat(24), shardId: 0, }, guild_id: guildId, message_id: targetMessageId, user_id: targetMessage.author.userId, }, rawEmoji: "👍", thread: channel.thread, threadId: channel.thread.id, user: targetMessage.author, }, }; let stagedId: string | null = null; providerRuntime.postResultIds.push(targetMessageId); providerRuntime.postHook = async () => { if (stagedId) return; await callbacks.onReaction!(reaction); await callbacks.onReaction!(reaction); const staged = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "reaction_added"), ), ); expect(staged).toHaveLength(1); expect(staged[0]).toMatchObject({ conversationId: null, state: "received", attempts: 0, normalizedEvent: expect.objectContaining({ conversation: { externalThreadId: channel.thread.id }, message: { providerMessageId: targetMessageId }, runtimeContext: { generation: expect.any(Number), credentialFingerprint: expect.any(String), }, }), }); stagedId = staged[0]!.id; }; await service.publishBoardMessage( endpoint.id, conversation.id, "A Discord reply whose reaction must survive reconstruction", "discord-reaction-before-restart", "owner-user", ); providerRuntime.postHook = undefined; if (!stagedId) throw new Error("Expected a staged Discord reaction"); const commentCount = await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)) .then((rows) => rows.length); const runCount = await db .select({ id: heartbeatRuns.id }) .from(heartbeatRuns) .where(eq(heartbeatRuns.companyId, fixture.companyId)) .then((rows) => rows.length); const wakeupCount = wakeup.mock.calls.length; await service.shutdown(); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, stagedId)); const restarted = createService(); try { await restarted.service.processPendingDeliveries(1, stagedId); await expect( db.select().from(chatDeliveries).where(eq(chatDeliveries.id, stagedId)), ).resolves.toEqual([ expect.objectContaining({ conversationId: conversation.id, state: "processed", attempts: 1, nextAttemptAt: null, redactedError: null, normalizedEvent: expect.objectContaining({ conversation: { externalThreadId: channel.thread.id }, message: { providerMessageId: targetMessageId }, reaction: { added: true, emoji: "thumbsup", rawEmoji: "👍", }, }), }), ]); const reactionActivity = () => restarted.service .listActivity(endpoint.id) .then((items) => items.filter((item) => item.id === stagedId)); await expect(reactionActivity()).resolves.toEqual([ expect.objectContaining({ kind: "delivery", status: "processed", summary: "reaction added processed", }), ]); await restarted.service.processPendingDeliveries(1, stagedId); await expect(reactionActivity()).resolves.toHaveLength(1); await expect( db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)) .then((rows) => rows.length), ).resolves.toBe(commentCount); await expect( db .select({ id: heartbeatRuns.id }) .from(heartbeatRuns) .where(eq(heartbeatRuns.companyId, fixture.companyId)) .then((rows) => rows.length), ).resolves.toBe(runCount); expect(wakeup).toHaveBeenCalledTimes(wakeupCount); expect(restarted.wakeup).not.toHaveBeenCalled(); } finally { await restarted.service.shutdown(); } }); it.each(["action_first", "reaction_first"] as const)( "joins in-flight recovery before rethrowing an ordinary drain failure (%s)", async (releaseOrder) => { const gate = () => { let release!: () => void; const promise = new Promise((resolve) => { release = resolve; }); return { promise, release }; }; const actionGate = gate(); const reactionGate = gate(); let recoveryArmed = false; let actionEntered = false; let reactionEntered = false; const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredSlackEndpoint(fixture, { reactionReplayEndpointLockBarrier: async () => { if (!recoveryArmed) return; reactionEntered = true; await reactionGate.promise; }, }); const restoreMocks: Array<() => void> = []; let ordinaryDeliveryId: string | null = null; let sweep: Promise<{ ok: true } | { ok: false; error: unknown }> | null = null; try { // Recovery scans the shared fixture database. Earlier tests can leave // authorized retry intents deliberately staged across service shutdown. // Settle those before this fixture arms its controlled work and records // the strict global no-new-wakeup/post/row baseline below. await service.processPendingDeliveries(); const channel = makeThread({ channelId: "C-RECOVERY-JOIN", id: "slack:C-RECOVERY-JOIN:7147.1", name: "recovery-join", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "7147.1", text: "@maya prepare the recovery join fixture", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!conversation || !providerRuntime || !callbacks.onReaction) throw new Error("Expected the active Slack reaction fixture"); // Obtain a current-runtime inbound receipt and its real normalized // delivery, then retain only the durable retry state needed here. const inboundMessageId = "7147.3"; await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: inboundMessageId, text: "Ready to retry", }), trigger: "subscribed_message", }); const [inbound] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), sql`${chatDeliveries.normalizedEvent}->'message'->>'providerMessageId' = ${inboundMessageId}`, ), ); const [receipt] = inbound ? await db .select() .from(chatActions) .where( and( eq(chatActions.deliveryId, inbound.id), eq(chatActions.kind, "receipt_reaction"), sql`${chatActions.payload}->>'operation' = 'add'`, ), ) : []; if (!inbound || !receipt) throw new Error("Expected an inbound delivery and receipt action"); const targetMessageId = "7147.4"; providerRuntime.postResultIds.push(targetMessageId); await service.publishBoardMessage( endpoint.id, conversation.id, "The linked reaction target", `recovery-join-${releaseOrder}`, "owner-user", ); const target = makeMessage({ id: targetMessageId, text: "" }); await callbacks.onReaction({ endpointId: endpoint.id, provider: "slack", event: { adapter: {} as never, added: true, emoji: { name: "thumbs_up", toJSON: () => ":thumbs_up:", toString: () => ":thumbs_up:", }, message: target, messageId: targetMessageId, raw: { event_ts: "7147.45" }, rawEmoji: "+1", thread: channel.thread, threadId: channel.thread.id, user: target.author, }, }); const [reaction] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "reaction_added"), ), ); if (!reaction) throw new Error("Expected the normalized reaction"); await db .update(chatDeliveries) .set({ conversationId: null, state: "received", attempts: 0, processedAt: null, nextAttemptAt: new Date(0), }) .where(eq(chatDeliveries.id, reaction.id)); await db .update(chatActions) .set({ status: "received", result: null, createdAt: new Date(0) }) .where(eq(chatActions.id, receipt.id)); ordinaryDeliveryId = randomUUID(); const ordinaryEventId = `recovery-join-ordinary-${randomUUID()}`; await db.insert(chatDeliveries).values({ ...inbound, id: ordinaryDeliveryId, providerEventId: ordinaryEventId, deduplicationKey: ordinaryEventId, normalizedEvent: { ...inbound.normalizedEvent, providerEventId: ordinaryEventId, message: { ...(inbound.normalizedEvent.message as Record), providerMessageId: "7147.5", }, }, state: "received", attempts: 0, receivedAt: new Date(0), processedAt: null, nextAttemptAt: null, }); const originalThread = providerRuntime.thread.bind(providerRuntime); const recoveredReceipt = vi.fn(async () => { actionEntered = true; await actionGate.promise; }); const threadSpy = vi .spyOn(providerRuntime, "thread") .mockImplementation((threadId) => { const thread = originalThread(threadId); const originalAddReaction = thread.adapter.addReaction; thread.adapter.addReaction = async (...args) => { if ( threadId === channel.thread.id && args[1] === inboundMessageId ) await recoveredReceipt(); return originalAddReaction(...args); }; return thread; }); restoreMocks.push(() => threadSpy.mockRestore()); // Fail only the ordinary drain's lease acquisition, outside its // per-message catch. Recovery's credential leases must keep working. const sentinel = new Error("injected ordinary drain lease failure"); let ordinaryFailed = false; const originalInsert = db.insert.bind(db); const insertSpy = vi.spyOn(db, "insert").mockImplementation((table) => { const builder = originalInsert(table); if (table === chatEndpointLeases) { const originalValues = builder.values.bind(builder); builder.values = ((values: { endpointId?: string; leaseKey?: string; }) => { if ( !ordinaryFailed && values.endpointId === endpoint.id && values.leaseKey?.startsWith("inbound:") ) { ordinaryFailed = true; throw sentinel; } return originalValues(values); }) as typeof builder.values; } return builder; }); restoreMocks.push(() => insertSpy.mockRestore()); const unchangedRows = async () => Promise.all([ db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.companyId, fixture.companyId)), db .select({ id: issues.id }) .from(issues) .where(eq(issues.companyId, fixture.companyId)), db .select({ id: heartbeatRuns.id }) .from(heartbeatRuns) .where(eq(heartbeatRuns.companyId, fixture.companyId)), db .select({ id: chatPublications.id }) .from(chatPublications) .where(eq(chatPublications.endpointId, endpoint.id)), ]); const baseline = await unchangedRows(); const wakeupCount = wakeup.mock.calls.length; const postCount = providerRuntime.posts.length; let settled = false; recoveryArmed = true; sweep = service.processPendingDeliveries().then( () => { settled = true; return { ok: true as const }; }, (error: unknown) => { settled = true; return { ok: false as const, error }; }, ); await expect .poll(() => ordinaryFailed && actionEntered && reactionEntered) .toBe(true); expect(settled).toBe(false); const actionState = () => db .select({ state: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, receipt.id)) .then((rows) => rows[0]?.state); const reactionState = () => db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where(eq(chatDeliveries.id, reaction.id)) .then((rows) => rows[0]?.state); if (releaseOrder === "action_first") { actionGate.release(); await expect.poll(actionState).toBe("processed"); } else { reactionGate.release(); await expect.poll(reactionState).toBe("processed"); } expect(settled).toBe(false); actionGate.release(); reactionGate.release(); const outcome = await sweep; expect(outcome.ok).toBe(false); if (outcome.ok) throw new Error("Expected the ordinary drain failure"); expect(outcome.error).toBe(sentinel); expect(await actionState()).toBe("processed"); await expect( db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, reaction.id)), ).resolves.toEqual([ expect.objectContaining({ conversationId: conversation.id, state: "processed", attempts: 1, }), ]); await expect( db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where(eq(chatDeliveries.id, ordinaryDeliveryId)), ).resolves.toEqual([{ state: "received" }]); expect(recoveredReceipt).toHaveBeenCalledTimes(1); expect(wakeup).toHaveBeenCalledTimes(wakeupCount); expect(providerRuntime.posts).toHaveLength(postCount); expect(await unchangedRows()).toEqual(baseline); await expect( db .select({ id: chatEndpointLeases.id }) .from(chatEndpointLeases) .where(eq(chatEndpointLeases.endpointId, endpoint.id)), ).resolves.toEqual([]); } finally { actionGate.release(); reactionGate.release(); await sweep; for (const restore of restoreMocks.reverse()) restore(); if (ordinaryDeliveryId) await db .delete(chatDeliveries) .where(eq(chatDeliveries.id, ordinaryDeliveryId)); await service.shutdown(); } }, 20_000, ); it("does not deadlock reaction replay against publication link settlement", async () => { const fixture = await seedCompany(); let beginContention!: () => void; const contention = new Promise((resolve) => { beginContention = resolve; }); let endpointLocked!: () => void; const endpointLockReached = new Promise((resolve) => { endpointLocked = resolve; }); let conversationLocked!: () => void; const conversationLockReached = new Promise((resolve) => { conversationLocked = resolve; }); const { callbacks, endpoint, runtime, service, wakeup } = await configuredSlackEndpoint(fixture, { reactionReplayEndpointLockBarrier: async () => { endpointLocked(); await contention; }, reactionReplayConversationLockBarrier: async () => { conversationLocked(); }, }); const channel = makeThread({ channelId: "C-REACTION-LINK-LOCK", id: "slack:C-REACTION-LINK-LOCK:7135.1", name: "reaction-link-lock", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "7135.1", text: "@maya send a reply for the link lock race", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); if (!callbacks.onReaction) throw new Error("Slack reaction callback was not registered"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Slack conversation"); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack endpoint runtime"); const targetMessageId = "7135.2"; providerRuntime.postResultIds.push(targetMessageId); const publication = await service.publishBoardMessage( endpoint.id, conversation.id, "The link-lock race reply", "reaction-link-lock-settlement", "owner-user", ); const [link] = await db .select() .from(chatMessageLinks) .where( and( eq(chatMessageLinks.publicationId, publication.id), eq(chatMessageLinks.providerMessageId, targetMessageId), ), ); if (!link) throw new Error("Expected the initial outbound link"); await db.delete(chatMessageLinks).where(eq(chatMessageLinks.id, link.id)); await db .update(chatPublications) .set({ state: "streaming", providerMessageId: null, updatedAt: new Date(), }) .where(eq(chatPublications.id, publication.id)); const targetMessage = makeMessage({ id: targetMessageId, text: "" }); await callbacks.onReaction({ endpointId: endpoint.id, provider: "slack", event: { adapter: {} as never, added: true, emoji: { name: "thumbs_up", toJSON: () => ":thumbs_up:", toString: () => ":thumbs_up:", }, message: targetMessage, messageId: targetMessageId, raw: { event_ts: "7135.25" }, rawEmoji: "+1", thread: channel.thread, threadId: channel.thread.id, user: targetMessage.author, }, }); const [staged] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "reaction_added"), ), ); if (!staged) throw new Error("Expected a staged reaction delivery"); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, staged.id)); let linkInserted!: () => void; const linkInsertReached = new Promise((resolve) => { linkInserted = resolve; }); const settlement = db.transaction(async (tx) => { await tx .update(chatPublications) .set({ state: "published", providerMessageId: targetMessageId, publishedAt: new Date(), updatedAt: new Date(), }) .where(eq(chatPublications.id, publication.id)); await tx.insert(chatMessageLinks).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, publicationId: publication.id, commentId: publication.commentId, providerMessageId: targetMessageId, direction: "outbound", }); linkInserted(); await contention; await tx .update(chatEndpoints) .set({ updatedAt: new Date() }) .where(eq(chatEndpoints.id, endpoint.id)); }); await linkInsertReached; const wakeupCount = wakeup.mock.calls.length; const replay = service.processPendingDeliveries(1, staged.id); await endpointLockReached; beginContention(); await conversationLockReached; await Promise.all([settlement, replay]); const [waitingForLink] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, staged.id)); expect(waitingForLink).toMatchObject({ conversationId: null, state: "retry", attempts: 1, redactedError: "Waiting for the sent message link", }); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, staged.id)); await service.processPendingDeliveries(1, staged.id); const [processed] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, staged.id)); expect(processed).toMatchObject({ conversationId: conversation.id, state: "processed", attempts: 2, }); expect(wakeup).toHaveBeenCalledTimes(wakeupCount); }); it("filters a late duplicate instead of bypassing the bounded reaction-link lifetime", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture); const channel = makeThread({ channelId: "C-EXPIRED-REACTION", id: "slack:C-EXPIRED-REACTION:7140.1", name: "expired-reaction", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "7140.1", text: "@maya prepare another reply", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); if (!callbacks.onReaction) throw new Error("Slack reaction callback was not registered"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Slack conversation"); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack endpoint runtime"); const targetMessageId = "7140.2"; const targetMessage = makeMessage({ id: targetMessageId, text: "" }); const reaction = { endpointId: endpoint.id, provider: "slack" as const, event: { adapter: {} as never, added: true, emoji: { name: "eyes", toJSON: () => ":eyes:", toString: () => ":eyes:", }, message: targetMessage, messageId: targetMessageId, raw: { event_ts: "7140.25" }, rawEmoji: "eyes", thread: channel.thread, threadId: channel.thread.id, user: targetMessage.author, }, }; let stagedId: string | null = null; providerRuntime.postResultIds.push(targetMessageId); providerRuntime.postHook = async () => { if (stagedId) return; await callbacks.onReaction!(reaction); stagedId = await db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "reaction_added"), ), ) .then((rows) => rows[0]?.id ?? null); }; await service.publishBoardMessage( endpoint.id, conversation.id, "Another provider-visible reply", "expired-reaction-before-link", "owner-user", ); providerRuntime.postHook = undefined; if (!stagedId) throw new Error("Expected a staged reaction delivery"); await db .update(chatDeliveries) .set({ receivedAt: new Date(Date.now() - 2 * 60_000 - 1), nextAttemptAt: new Date(0), }) .where(eq(chatDeliveries.id, stagedId)); await callbacks.onReaction(reaction); const [expired] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, stagedId)); expect(expired).toMatchObject({ conversationId: null, state: "filtered", redactedError: "Reaction target did not become linked before replay expiry", }); }); it("resamples reaction expiry after replay waits on authorization locks", async () => { const fixture = await seedCompany(); const baseTime = new Date("2026-09-07T20:30:00.000Z"); let advanceReplayClock = false; const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture, { reactionReplayEndpointLockBarrier: async () => { if (advanceReplayClock) { vi.setSystemTime(new Date(baseTime.getTime() + 2_000)); } }, }); const channel = makeThread({ channelId: "C-REACTION-EXPIRY-LOCK", id: "slack:C-REACTION-EXPIRY-LOCK:7142.1", name: "reaction-expiry-lock", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "7142.1", text: "@maya prepare the expiry-lock reply", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); if (!callbacks.onReaction) throw new Error("Slack reaction callback was not registered"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Slack conversation"); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack endpoint runtime"); const targetMessageId = "7142.2"; const targetMessage = makeMessage({ id: targetMessageId, text: "" }); let stagedId: string | null = null; providerRuntime.postResultIds.push(targetMessageId); providerRuntime.postHook = async () => { if (stagedId) return; await callbacks.onReaction!({ endpointId: endpoint.id, provider: "slack", event: { adapter: {} as never, added: true, emoji: { name: "eyes", toJSON: () => ":eyes:", toString: () => ":eyes:", }, message: targetMessage, messageId: targetMessageId, raw: { event_ts: "7142.25" }, rawEmoji: "eyes", thread: channel.thread, threadId: channel.thread.id, user: targetMessage.author, }, }); stagedId = await db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "reaction_added"), ), ) .then((rows) => rows[0]?.id ?? null); }; await service.publishBoardMessage( endpoint.id, conversation.id, "The expiry-lock reply", "reaction-expiry-after-lock", "owner-user", ); providerRuntime.postHook = undefined; if (!stagedId) throw new Error("Expected a staged reaction delivery"); await db .update(chatDeliveries) .set({ receivedAt: new Date(baseTime.getTime() - 2 * 60_000 + 1_000), nextAttemptAt: new Date(0), }) .where(eq(chatDeliveries.id, stagedId)); vi.useFakeTimers({ toFake: ["Date"] }); try { vi.setSystemTime(baseTime); advanceReplayClock = true; await service.processPendingDeliveries(1, stagedId); } finally { vi.useRealTimers(); } const [filtered] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, stagedId)); expect(filtered).toMatchObject({ conversationId: null, state: "filtered", redactedError: "Reaction target did not become linked before replay expiry", }); }); it("filters a staged reaction when destination reach is revoked before replay", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredSlackEndpoint(fixture); const channel = makeThread({ channelId: "C-REVOKED-EARLY-REACTION", id: "slack:C-REVOKED-EARLY-REACTION:7145.1", name: "revoked-early-reaction", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: "7145.1", text: "@maya prepare a reply before reach changes", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); if (!callbacks.onReaction) throw new Error("Slack reaction callback was not registered"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation?.resourceId) throw new Error("Expected a resource-backed Slack conversation"); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack endpoint runtime"); const targetMessageId = "7145.2"; const targetMessage = makeMessage({ id: targetMessageId, text: "" }); const reaction = { endpointId: endpoint.id, provider: "slack" as const, event: { adapter: {} as never, added: true, emoji: { name: "thumbs_up", toJSON: () => ":thumbs_up:", toString: () => ":thumbs_up:", }, message: targetMessage, messageId: targetMessageId, raw: { event_ts: "7145.25" }, rawEmoji: "+1", thread: channel.thread, threadId: channel.thread.id, user: targetMessage.author, }, }; let stagedId: string | null = null; providerRuntime.postResultIds.push(targetMessageId); providerRuntime.postHook = async () => { if (stagedId) return; await callbacks.onReaction!(reaction); stagedId = await db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "reaction_added"), ), ) .then((rows) => rows[0]?.id ?? null); }; await service.publishBoardMessage( endpoint.id, conversation.id, "A reply whose reach will be revoked", "revoked-reaction-before-link", "owner-user", ); providerRuntime.postHook = undefined; if (!stagedId) throw new Error("Expected a staged reaction delivery"); const commentCount = await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)) .then((rows) => rows.length); const wakeupCount = wakeup.mock.calls.length; await db .update(chatEndpointResources) .set({ enabled: false, updatedAt: new Date() }) .where(eq(chatEndpointResources.id, conversation.resourceId)); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, stagedId)); await service.processPendingDeliveries(1, stagedId); const [filtered] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, stagedId)); expect(filtered).toMatchObject({ conversationId: null, state: "filtered", redactedError: "Reaction destination or principal is no longer authorized", }); await callbacks.onReaction(reaction); await expect( db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "reaction_added"), ), ), ).resolves.toHaveLength(1); await expect( db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)) .then((rows) => rows.length), ).resolves.toBe(commentCount); expect(wakeup).toHaveBeenCalledTimes(wakeupCount); }); it("replays a pre-link reaction into the completed DM generation that sent the message", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredSlackEndpoint(fixture); const channelId = "D-EARLY-REACTION"; const dm = makeThread({ channelId, id: `slack:${channelId}:`, isDM: true, name: "Slack early-reaction DM", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: dm.thread, message: makeMessage({ id: "7150.1", text: "Prepare the first DM generation reply", userId: "U-EARLY-DM", }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); if (!callbacks.onReaction) throw new Error("Slack reaction callback was not registered"); const [firstConversation] = await service.listConversations(endpoint.id); if (!firstConversation) throw new Error("Expected Slack DM conversation"); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack endpoint runtime"); const targetMessageId = "7150.2"; const targetMessage = makeMessage({ id: targetMessageId, text: "", userId: "U-EARLY-DM", }); const reaction = { endpointId: endpoint.id, provider: "slack" as const, event: { adapter: {} as never, added: true, emoji: { name: "thumbs_up", toJSON: () => ":thumbs_up:", toString: () => ":thumbs_up:", }, message: targetMessage, messageId: targetMessageId, raw: { event_ts: "7150.25" }, rawEmoji: "+1", thread: dm.thread, threadId: dm.thread.id, user: targetMessage.author, }, }; let stagedId: string | null = null; providerRuntime.postResultIds.push(targetMessageId); providerRuntime.postHook = async () => { if (stagedId) return; const completedAt = new Date(); await db .update(issues) .set({ status: "done", completedAt, updatedAt: completedAt }) .where(eq(issues.id, firstConversation.issueId)); await db .update(chatConversations) .set({ state: "completed", updatedAt: completedAt }) .where(eq(chatConversations.id, firstConversation.id)); await callbacks.onReaction!(reaction); stagedId = await db .select({ id: chatDeliveries.id }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "reaction_added"), ), ) .then((rows) => rows[0]?.id ?? null); }; await service.publishBoardMessage( endpoint.id, firstConversation.id, "The first DM generation reply", "completed-dm-reaction-before-link", "owner-user", ); providerRuntime.postHook = undefined; if (!stagedId) throw new Error("Expected a staged DM reaction"); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: dm.thread, message: makeMessage({ id: "7150.3", text: "Start the next DM generation", userId: "U-EARLY-DM", }), trigger: "direct_message", }); const conversations = await service.listConversations(endpoint.id); expect(conversations).toHaveLength(2); const nextConversation = conversations.find( (candidate) => candidate.id !== firstConversation.id, ); expect(nextConversation).toMatchObject({ state: "active", sessionGeneration: 2, }); const wakeupCount = wakeup.mock.calls.length; const runCount = await db .select({ id: heartbeatRuns.id }) .from(heartbeatRuns) .where(eq(heartbeatRuns.companyId, fixture.companyId)) .then((rows) => rows.length); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, stagedId)); await service.processPendingDeliveries(1, stagedId); const [replayed] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, stagedId)); expect(replayed).toMatchObject({ conversationId: firstConversation.id, state: "processed", }); expect(replayed.conversationId).not.toBe(nextConversation?.id); expect(wakeup).toHaveBeenCalledTimes(wakeupCount); await expect( db .select({ id: heartbeatRuns.id }) .from(heartbeatRuns) .where(eq(heartbeatRuns.companyId, fixture.companyId)) .then((rows) => rows.length), ).resolves.toBe(runCount); }); it("maps a timestamp-bearing Slack DM reaction to the completed generation that owns its linked message", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service, wakeup } = await configuredSlackEndpoint(fixture); const channelId = "D-TOP-LEVEL-REACTION"; const dm = makeThread({ channelId, id: `slack:${channelId}:`, isDM: true, name: "Slack direct message", }); const original = makeMessage({ id: "7120.1", text: "Observe this top-level DM reaction", userId: "U-DM-REACTION", }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: dm.thread, message: original, trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); if (!callbacks.onReaction) throw new Error("Slack reaction callback was not registered"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); expect(conversation).toMatchObject({ externalThreadId: dm.thread.id, isDirectMessage: true, }); await db .update(issues) .set({ status: "done", completedAt: new Date(), updatedAt: new Date() }) .where(eq(issues.id, conversation!.issueId)); const newerMessage = makeMessage({ id: "7120.2", text: "Start the newer DM task generation", userId: original.author.userId, }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: dm.thread, message: newerMessage, trigger: "direct_message", }); const conversations = await service.listConversations(endpoint.id); expect(conversations).toHaveLength(2); expect( conversations.find((candidate) => candidate.id === conversation!.id), ).toMatchObject({ state: "completed", sessionGeneration: 1 }); expect( conversations.find((candidate) => candidate.id !== conversation!.id), ).toMatchObject({ state: "active", sessionGeneration: 2 }); const commentCountBefore = await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation!.issueId)) .then((rows) => rows.length); const wakeupCountBefore = wakeup.mock.calls.length; const runCountBefore = await db .select({ id: heartbeatRuns.id }) .from(heartbeatRuns) .where(eq(heartbeatRuns.companyId, fixture.companyId)) .then((rows) => rows.length); const issueCountBefore = await db .select({ id: issues.id }) .from(issues) .where(eq(issues.companyId, fixture.companyId)) .then((rows) => rows.length); const emoji = { name: "thumbs_up", toJSON: () => ":thumbs_up:", toString: () => ":thumbs_up:", }; await callbacks.onReaction({ endpointId: endpoint.id, provider: "slack", event: { adapter: {} as never, added: true, emoji, message: original, messageId: original.id, raw: { event_ts: "7121.1" }, rawEmoji: "+1", thread: dm.thread, threadId: `slack:${channelId}:${original.id}`, user: original.author, }, }); await expect( db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.conversationId, conversation!.id), eq(chatDeliveries.eventKind, "reaction_added"), ), ), ).resolves.toEqual([ expect.objectContaining({ state: "processed", normalizedEvent: expect.objectContaining({ conversation: { externalThreadId: `slack:${channelId}:${original.id}`, }, message: { providerMessageId: original.id }, reaction: { added: true, emoji: "thumbs_up", rawEmoji: "+1" }, }), }), ]); await expect( db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation!.issueId)) .then((rows) => rows.length), ).resolves.toBe(commentCountBefore); expect(wakeup).toHaveBeenCalledTimes(wakeupCountBefore); await expect( db .select({ id: heartbeatRuns.id }) .from(heartbeatRuns) .where(eq(heartbeatRuns.companyId, fixture.companyId)) .then((rows) => rows.length), ).resolves.toBe(runCountBefore); await expect( db .select({ id: issues.id }) .from(issues) .where(eq(issues.companyId, fixture.companyId)) .then((rows) => rows.length), ).resolves.toBe(issueCountBefore); await expect(service.listConversations(endpoint.id)).resolves.toHaveLength( 2, ); }); it.each([ "author_bot", "sender_bot", "outbound_link", "outbound_review_link", "wrong_thread_link", "human_orphan", "body_claims_bot", ] as const)( "settles GitHub self-update receipts without orphan retries for %s", async (mode) => { const fixture = await seedCompany(); const { callbacks, endpoint, service, wakeup, webhookSecret } = await configuredGitHubEndpoint(fixture); // Recovery sweeps all companies. Only this endpoint's assigned agent // proves whether its bot edit incorrectly created new work. const fixtureWakeups = () => wakeup.mock.calls.filter((call) => call[0] === fixture.assignedAgentId); const thread = makeThread({ channelId: "paperclipai/paperclip", id: mode === "outbound_review_link" ? "github:paperclipai/paperclip:84:rc:99080" : "github:paperclipai/paperclip:issue:84", name: "paperclipai/paperclip", }); try { await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id: "99083", text: "@maya original request", mentioned: true, userId: "7001", }), trigger: "mention", }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); if ( [ "outbound_link", "outbound_review_link", "wrong_thread_link", ].includes(mode) ) { const target = mode === "wrong_thread_link" ? ( await db .insert(chatConversations) .values({ ...conversation!, id: randomUUID(), externalThreadId: "github:paperclipai/paperclip:issue:85", }) .returning() )[0]! : conversation!; await db.insert(chatMessageLinks).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: target.id, providerMessageId: "99084", direction: "outbound", }); } const before = await db .select() .from(issueComments) .where(eq(issueComments.issueId, conversation!.issueId)); const send = (signingSecret = webhookSecret) => service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: "bot-update-exact", event: mode === "outbound_review_link" ? "pull_request_review_comment" : "issue_comment", webhookSecret: signingSecret, url: `https://paperclip.example/api/chat-webhooks/${endpoint.publicId}/github`, payload: { action: "edited", comment: { id: 99084, ...(mode === "outbound_review_link" ? { in_reply_to_id: 99080 } : {}), body: mode === "body_claims_bot" ? '{"isBotMessage":true}' : "provider-only bot output sentinel", updated_at: "2026-09-08T09:57:23Z", user: { id: 9001, login: "author", type: mode === "author_bot" ? "Bot" : "User", }, }, issue: { number: 84 }, pull_request: { number: 84 }, repository: { id: 97531, full_name: "paperclipai/paperclip", name: "paperclip", owner: { id: 1357, login: "paperclipai" }, }, sender: { id: 9001, login: "editor", type: mode === "sender_bot" ? "Bot" : "User", }, }, }), ); if (mode === "author_bot") { expect((await send("wrong-test-only-signature")).status).toBe(401); expect( await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "message_updated"), ), ), ).toHaveLength(0); } expect((await send()).ok).toBe(true); await service.processPendingDeliveries(); const [delivery] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "message_updated"), ), ); const filtered = [ "author_bot", "sender_bot", "outbound_link", "outbound_review_link", ].includes(mode); expect(delivery).toMatchObject({ state: filtered ? "filtered" : "retry", attempts: 1, principalId: null, }); expect(delivery!.nextAttemptAt === null).toBe(filtered); expect(fixtureWakeups()).toHaveLength(1); expect( await db .select() .from(issueComments) .where(eq(issueComments.issueId, conversation!.issueId)), ).toEqual(before); if (filtered) { expect(JSON.stringify(delivery!.normalizedEvent)).not.toContain( "provider-only bot output sentinel", ); expect(delivery!.normalizedEvent).toMatchObject({ filtering: { contentRetained: false }, }); expect((await send()).ok).toBe(true); await service.processPendingDeliveries(); expect( await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "message_updated"), ), ), ).toEqual([delivery]); expect(fixtureWakeups()).toHaveLength(1); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id: "99085", text: "Follow-up after bot edit", userId: "7001", }), trigger: "subscribed_message", }); expect(fixtureWakeups()).toHaveLength(2); } } finally { await service.shutdown(); } }, ); it("orders reversed GitHub edit and delete callbacks behind their durable root", async () => { const fixture = await seedCompany(); const deferred: Array<() => void | Promise> = []; const { callbacks, endpoint, service, wakeup, webhookSecret } = await configuredGitHubEndpoint(fixture, { deferWebhookProcessing: true, scheduleDeferredWork: (task) => deferred.push(task), }); const drainDeferred = async () => { let processed = 0; while (deferred.length > 0) { if (processed++ >= 20) throw new Error("GitHub deferred test work did not quiesce"); await deferred.shift()?.(); } }; const thread = makeThread({ channelId: "paperclipai/chat-lifecycle-order", id: "github:paperclipai/chat-lifecycle-order:issue:84", name: "paperclipai/chat-lifecycle-order", }); await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "repository", providerResourceId: "paperclipai/chat-lifecycle-order", label: "paperclipai/chat-lifecycle-order", availability: "available", enabled: true, }); const original = { ...makeMessage({ id: "99084", text: "@maya original ordered GitHub request", mentioned: true, userId: "7001", }), metadata: { dateSent: new Date("2026-09-05T11:59:59Z"), edited: false, }, } as Message; await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: original, trigger: "mention", }); const sendLifecycle = ( action: "edited" | "deleted", body: string, deliverySuffix = action, updatedAt = "2026-09-05T12:00:00Z", ) => service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: `delivery-84-${deliverySuffix}`, event: "issue_comment", payload: { action, comment: { id: 99084, body, updated_at: updatedAt, }, issue: { number: 84 }, repository: { id: 98400, full_name: "paperclipai/chat-lifecycle-order", name: "chat-lifecycle-order", owner: { id: 1357, login: "paperclipai" }, }, sender: { id: 7001, login: "alex-e2e" }, }, webhookSecret, url: `https://paperclip.example/api/chat-webhooks/${endpoint.publicId}/github`, }), ); // GitHub may deliver these as independent requests in the opposite order. await expect( sendLifecycle("deleted", "deleted payload"), ).resolves.toMatchObject({ ok: true }); await expect( sendLifecycle("edited", "@maya corrected ordered GitHub request"), ).resolves.toMatchObject({ ok: true, }); // An exact GitHub redelivery id must remain a single durable transition. await expect( sendLifecycle("edited", "@maya corrected ordered GitHub request"), ).resolves.toMatchObject({ ok: true, }); // One root-conversation drain plus one durable-ingress callback per HTTP // request is queued. The duplicate delivery callback becomes a no-op. expect(deferred).toHaveLength(4); await drainDeferred(); await vi.waitFor(async () => { const deliveries = await db .select({ eventKind: chatDeliveries.eventKind, state: chatDeliveries.state, }) .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(deliveries).toHaveLength(3); expect( deliveries.every((delivery) => delivery.state === "processed"), ).toBe(true); }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); const comments = await db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)) .orderBy(asc(issueComments.createdAt), asc(issueComments.id)); expect(comments.map((comment) => comment.body)).toEqual([ "@maya original ordered GitHub request", "An external message was edited:\n\n@maya corrected ordered GitHub request", "An external message in this conversation was deleted.", ]); expect(wakeup).toHaveBeenCalledTimes(1); await expect( sendLifecycle( "edited", "@maya a delayed edit must not resurrect this message", "edited-after-delete", "2026-09-05T12:01:00Z", ), ).resolves.toMatchObject({ ok: true }); expect(deferred).toHaveLength(1); await drainDeferred(); await vi.waitFor(() => expect(deferred).toHaveLength(1)); await drainDeferred(); await vi.waitFor(async () => { await expect( db .select({ eventKind: chatDeliveries.eventKind, redactedError: chatDeliveries.redactedError, state: chatDeliveries.state, }) .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "message_updated"), eq(chatDeliveries.state, "filtered"), ), ), ).resolves.toEqual([ { eventKind: "message_updated", redactedError: "Message edit arrived after the provider message was deleted", state: "filtered", }, ]); }); await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)), ).resolves.toHaveLength(3); expect(wakeup).toHaveBeenCalledTimes(1); await service.shutdown(); }); it("supplements Chat SDK with verified GitHub comment edit and delete lifecycle events", async () => { const fixture = await seedCompany(); const privateKey = generateKeyPairSync("rsa", { modulusLength: 2048 }) .privateKey.export({ type: "pkcs8", format: "pem" }) .toString(); const runtime = new FakeChatSdkRuntime(); const { service, wakeup } = createService(runtime, (async ( input: string | URL | Request, ) => { const url = String(input); if (url === "https://api.github.com/app") { return new Response( JSON.stringify({ id: 790, slug: "maya-paperclip-lifecycle", name: "Maya Paperclip", owner: { login: "paperclipai" }, permissions: { issues: "write", metadata: "read", pull_requests: "write", }, events: [ "github_app_authorization", "installation", "installation_repositories", "issue_comment", "pull_request_review_comment", ], }), { status: 200, headers: { "content-type": "application/json" } }, ); } if (url === "https://api.github.com/app/installations?per_page=100") { return new Response( JSON.stringify([ { id: 8642, account: { id: 1, login: "paperclipai" }, permissions: { issues: "write", metadata: "read", pull_requests: "write", }, suspended_at: null, }, ]), { status: 200, headers: { "content-type": "application/json" } }, ); } if ( url === "https://api.github.com/app/installations/8642/access_tokens" ) { return new Response(JSON.stringify({ token: "installation-token" }), { status: 201, headers: { "content-type": "application/json" }, }); } if ( url === "https://api.github.com/installation/repositories?per_page=100&page=1" ) { return new Response(JSON.stringify({ repositories: [] }), { status: 200, headers: { "content-type": "application/json" }, }); } throw new Error(`Unexpected provider request: ${url}`); }) as typeof globalThis.fetch); const endpoint = await service.create( fixture.companyId, { provider: "github", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); const { webhookSecret } = await service.generateSetupSecret( endpoint.id, "owner-user", ); await recordGitHubWebhookVerification( service, endpoint.publicId, webhookSecret, ); await service.configure( endpoint.id, { action: "configure", credentials: { appId: "123456", privateKey, }, }, "owner-user", ); const callbacks = runtime.configurations.get(endpoint.id)?.callbacks; if (!callbacks) throw new Error("Expected GitHub callbacks"); const thread = makeThread({ channelId: "paperclipai/chat-e2e", id: "github:paperclipai/chat-e2e:issue:42", name: "paperclipai/chat-e2e", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id: "77001", text: "@maya original GitHub request", mentioned: true, userId: "7001", }), trigger: "mention", }); for (const item of [ { id: "github:paperclipai/chat-e2e:43", messageId: "77002", }, { id: "github:paperclipai/chat-e2e:43:rc:88001", messageId: "88001", }, ]) { const nativeThread = makeThread({ channelId: "paperclipai/chat-e2e", id: item.id, name: "paperclipai/chat-e2e", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: nativeThread.thread, message: makeMessage({ id: item.messageId, text: "@maya original GitHub PR request", mentioned: true, userId: "7001", }), trigger: "mention", }); } await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const sendLifecycle = async (input: { action: "edited" | "deleted"; event: "issue_comment" | "pull_request_review_comment"; messageId: number; number: number; updatedAt: string; body?: string; inReplyToId?: number; issueIsPullRequest?: boolean; }) => { await service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: `github-comment-${input.event}-${input.messageId}-${input.action}-${input.updatedAt}-${createHash( "sha256", ) .update(input.body ?? "@maya corrected GitHub request") .digest("hex") .slice(0, 12)}`, event: input.event, payload: { action: input.action, comment: { id: input.messageId, in_reply_to_id: input.inReplyToId, body: input.body ?? "@maya corrected GitHub request", updated_at: input.updatedAt, }, issue: { number: input.number, pull_request: input.issueIsPullRequest ? {} : undefined, }, pull_request: { number: input.number }, repository: { id: 97531, full_name: "paperclipai/chat-e2e", name: "chat-e2e", owner: { id: 1357, login: "paperclipai" }, }, sender: { id: 7001, login: "alex-e2e" }, }, webhookSecret, url: `https://paperclip.example/api/chat-webhooks/${endpoint.publicId}/github`, }), ); }; for (const input of [ { event: "issue_comment" as const, messageId: 77001, number: 42, }, { event: "issue_comment" as const, issueIsPullRequest: true, messageId: 77002, number: 43, }, { event: "pull_request_review_comment" as const, inReplyToId: 88001, messageId: 88001, number: 43, }, ].entries()) { const [index, event] = input; await sendLifecycle({ ...event, action: "edited", updatedAt: `2026-09-05T12:0${index}:00Z`, }); await sendLifecycle({ ...event, action: "deleted", updatedAt: `2026-09-05T12:1${index}:00Z`, }); } const sameSecondEdit = { action: "edited" as const, body: "@maya corrected GitHub request again in the same second", event: "issue_comment" as const, messageId: 77001, number: 42, updatedAt: "2026-09-05T12:00:00Z", }; await sendLifecycle(sameSecondEdit); await sendLifecycle(sameSecondEdit); await vi.waitFor(async () => { const lifecycle = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect( lifecycle.filter((row) => ["message_updated", "message_deleted"].includes(row.eventKind), ), ).toHaveLength(7); expect( lifecycle.filter( (row) => row.eventKind === "message_updated" && row.providerEventId.includes("-77001-"), ), ).toHaveLength(2); }); expect(wakeup).toHaveBeenCalledTimes(4); await service.shutdown(); }); it("retains content-free GitHub source invalidation after actor revocation and denies stale retry after regrant", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service, webhookSecret, wakeup, runtime } = await configuredGitHubEndpoint(fixture); try { const thread = makeThread({ channelId: "paperclipai/paperclip", id: "github:paperclipai/paperclip:issue:91", name: "paperclipai/paperclip", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id: "99101", text: "@maya establish a GitHub task before access changes", mentioned: true, userId: "7001", }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id, "7001"); await service.test(endpoint.id, "owner-user"); // The retry source must be the newest admitted request, not the older // root preceding the setup qualification's follow-up. await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "github", thread: thread.thread, message: makeMessage({ id: "99102", text: "The exact request that fails before actor access changes", userId: "7001", }), trigger: "subscribed_message", }); const [originalAction] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "inbound_wakeup"), ), ) .orderBy(desc(chatActions.createdAt)) .limit(1); expect(originalAction).toBeDefined(); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected GitHub conversation"); const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "failed", errorCode: "adapter_failed", finishedAt: new Date(), wakeupRequestId: originalAction!.id, contextSnapshot: { ...(await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "github", providerMessageId: "99102", })), taskKey: conversation.issueId, }, }); await db .update(agentWakeupRequests) .set({ status: "failed", runId }) .where(eq(agentWakeupRequests.id, originalAction!.id)); const retryInput = { companyId: fixture.companyId, issueId: conversation.issueId, agentId: fixture.assignedAgentId, failedRunId: runId, initiatedByUserId: "owner-user", }; const rollbackProbe = new Error("rollback authorized GitHub retry probe"); await expect( db.transaction(async (tx) => { await expect( service.prepareFailedChatRunRetry(tx, retryInput), ).resolves.toMatchObject({ issueId: conversation.issueId }); throw rollbackProbe; }), ).rejects.toBe(rollbackProbe); const commentCount = await db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)) .then((rows) => rows.length); await service.update( endpoint.id, { allowUnlinkedPeople: false }, "owner-user", ); const wakeCount = wakeup.mock.calls.length; const providerRuntime = runtime.endpoints.get(endpoint.id)!; const postCount = providerRuntime.posts.length; const reactionCount = providerRuntime.reactions.length; const revokedEdit = "@maya do not retain this revoked GitHub edit"; const response = await service.handleWebhook( endpoint.publicId, "github", signedGitHubWebhookRequest({ delivery: "github-revoked-edit-99102", event: "issue_comment", payload: { action: "edited", comment: { id: 99102, body: revokedEdit, updated_at: "2026-09-05T12:20:00Z", }, issue: { number: 91 }, repository: { id: 97531, full_name: "paperclipai/paperclip", name: "paperclip", owner: { id: 1357, login: "paperclipai" }, }, sender: { id: 7001, login: "alex-e2e" }, }, webhookSecret, }), ); expect(response.ok).toBe(true); await expect( db .select({ id: issueComments.id }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)), ).resolves.toHaveLength(commentCount); const lifecycle = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq(chatDeliveries.eventKind, "message_updated"), ), ) .then((rows) => rows[0]); expect(lifecycle).toMatchObject({ state: "processed", conversationId: conversation.id, redactedError: "Provider edit invalidation retained without admitting content from an unauthorized actor", normalizedEvent: { runtimeContext: { generation: expect.any(Number), credentialFingerprint: expect.any(String), }, filtering: { contentRetained: false }, message: { providerMessageId: "99102", targetProviderEventId: `${thread.thread.id}:99102`, providerSentAt: "2026-09-05T12:20:00.000Z", }, }, }); expect(JSON.stringify(lifecycle?.normalizedEvent)).not.toContain( revokedEdit, ); expect(lifecycle!.normalizedEvent.message).not.toHaveProperty("text"); await service.update( endpoint.id, { allowUnlinkedPeople: true }, "owner-user", ); await expect( db.transaction((tx) => service.prepareFailedChatRunRetry(tx, retryInput), ), ).rejects.toMatchObject({ details: { code: "chat_failed_run_retry_not_authorized" }, }); await expect( db .select({ id: chatActions.id }) .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "failed_run_retry"), ), ), ).resolves.toEqual([]); expect(wakeup.mock.calls.length).toBe(wakeCount); expect(providerRuntime.posts.length).toBe(postCount); expect(providerRuntime.reactions.length).toBe(reactionCount); } finally { await db .update(chatConversations) .set({ state: "completed" }) .where( and( eq(chatConversations.companyId, fixture.companyId), eq(chatConversations.endpointId, endpoint.id), ), ); await retirePublicationFixture(service, endpoint.id); } }); it("migrates Telegram basic-group reach and topic tasks to the replacement supergroup", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, service, wakeup } = await configuredTelegramEndpoint(fixture); await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, endpoint.id)); const previousChatId = "-5546433913"; const migratedChatId = "-1004415501660"; const topicId = 77; const [previousResource] = await db .insert(chatEndpointResources) .values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "chat", providerResourceId: previousChatId, label: "Telegram migration group", availability: "available", enabled: true, }) .returning(); const previousTopic = makeThread({ channelId: `telegram:${previousChatId}`, id: `telegram:${previousChatId}:${topicId}`, name: "Telegram migration group", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: previousTopic.thread, message: makeMessage({ id: `${previousChatId}:40`, raw: { message_id: 40, message_thread_id: topicId }, text: "Start before the group migration", mentioned: true, userId: "telegram-migration-user", }), trigger: "mention", }); const [beforeMigration] = await service.listConversations(endpoint.id); if (!beforeMigration) throw new Error("Expected pre-migration Telegram topic task"); const response = await service.handleWebhook( endpoint.publicId, "telegram", new Request("https://paperclip.example/telegram", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ update_id: 46, message: { message_id: 41, date: 1_788_700_041, chat: { id: Number(previousChatId), type: "group", title: "Telegram migration group", }, migrate_to_chat_id: Number(migratedChatId), }, }), }), ); expect(response.status).toBe(202); const resources = await service.listResources(endpoint.id); expect(resources).toEqual( expect.arrayContaining([ expect.objectContaining({ id: previousResource!.id, providerResourceId: previousChatId, availability: "unavailable", enabled: false, metadata: expect.objectContaining({ migratedTo: migratedChatId }), }), expect.objectContaining({ providerResourceId: migratedChatId, label: "Telegram migration group", availability: "available", enabled: true, metadata: expect.objectContaining({ migratedFrom: previousChatId }), }), ]), ); const [migratedConversation] = await service.listConversations(endpoint.id); expect(migratedConversation).toMatchObject({ id: beforeMigration.id, issueId: beforeMigration.issueId, externalConversationId: `telegram:${migratedChatId}`, externalThreadId: `telegram:${migratedChatId}:${topicId}`, externalLabel: "Telegram migration group", state: "active", }); const migratedTopic = makeThread({ channelId: `telegram:${migratedChatId}`, id: `telegram:${migratedChatId}:${topicId}`, name: `telegram:${migratedChatId}`, }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: migratedTopic.thread, message: makeMessage({ id: `${migratedChatId}:42`, raw: { message_id: 42, message_thread_id: topicId }, text: "Continue by replying after the group migration", mentioned: true, userId: "telegram-migration-user", }), trigger: "subscribed_message", }); await expect(service.listConversations(endpoint.id)).resolves.toEqual([ expect.objectContaining({ id: beforeMigration.id, issueId: beforeMigration.issueId, externalThreadId: `telegram:${migratedChatId}:${topicId}`, }), ]); await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, beforeMigration.issueId)) .orderBy(asc(issueComments.createdAt), asc(issueComments.id)), ).resolves.toEqual([ { body: "Start before the group migration" }, { body: "Continue by replying after the group migration" }, ]); expect(wakeup).toHaveBeenCalledTimes(2); }); it("enforces Telegram group reach and privacy-mode addressing before retaining message content", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredTelegramEndpoint(fixture); await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, endpoint.id)); const chatId = "-10077114455"; const [resource] = await db .insert(chatEndpointResources) .values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "chat", providerResourceId: chatId, label: "Telegram production group", availability: "available", enabled: false, }) .returning(); const group = makeThread({ channelId: chatId, id: `telegram:${chatId}`, name: "Telegram production group", }); if (!callbacks.onSlashCommand) throw new Error("Expected Telegram slash-command callbacks"); const invokeTaskCommand = async (input: { document?: { fileId: string; fileName: string; fileSize: number; mimeType: string; }; messageId: number; prompt: string; threadId?: string; topicId?: number; }) => { await callbacks.onSlashCommand!({ endpointId: endpoint.id, provider: "telegram", event: { channel: { id: input.threadId ?? group.thread.id, name: "Telegram production group", isDM: false, } as never, command: "/task@paperclip_test_bot", text: input.prompt, user: { userId: "telegram-group-user", userName: "telegram-group-user", fullName: "Telegram Group User", isBot: false, isMe: false, isSystem: false, }, raw: { message_id: input.messageId, ...(input.topicId === undefined ? {} : { message_thread_id: input.topicId }), date: 1_788_700_000 + input.messageId, chat: { id: Number(chatId), type: "supergroup", title: "Telegram production group", }, from: { id: 77112233, is_bot: false }, text: `/task@paperclip_test_bot${input.prompt ? ` ${input.prompt}` : ""}`, entities: [ { offset: 0, length: "/task@paperclip_test_bot".length, type: "bot_command", }, ], ...(input.document ? { document: { file_id: input.document.fileId, file_unique_id: `${input.document.fileId}-unique`, file_name: input.document.fileName, file_size: input.document.fileSize, mime_type: input.document.mimeType, }, } : {}), }, adapter: {} as never, openModal: async () => undefined, }, }); }; await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: group.thread, message: makeMessage({ id: `${chatId}:10`, text: "@maya disabled-group-private-marker", mentioned: true, userId: "telegram-disabled-user", }), trigger: "mention", }); const [disabledDelivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(disabledDelivery).toMatchObject({ state: "filtered", attempts: 0, principalId: null, redactedError: "Destination is not enabled in Paperclip", normalizedEvent: { filtering: { contentRetained: false }, message: { providerMessageId: `${chatId}:10` }, }, }); expect(JSON.stringify(disabledDelivery.normalizedEvent)).not.toContain( "disabled-group-private-marker", ); expect(JSON.stringify(disabledDelivery.normalizedEvent)).not.toContain( "telegram-disabled-user", ); expect(await service.listConversations(endpoint.id)).toEqual([]); expect(wakeup).not.toHaveBeenCalled(); await expect( db .select({ id: chatExternalPrincipals.id }) .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, fixture.companyId), eq(chatExternalPrincipals.provider, "telegram"), ), ), ).resolves.toHaveLength(0); await service.replaceResources(endpoint.id, [ { id: resource!.id, enabled: true }, ]); await invokeTaskCommand({ messageId: 11, prompt: "" }); expect(await service.listConversations(endpoint.id)).toEqual([]); expect(wakeup).not.toHaveBeenCalled(); expect( runtime.endpoints.get(endpoint.id)?.posts.map((post) => post.text), ).toEqual([ expect.stringMatching( /^Please include a request after \/task@paperclip_\d+_bot\.$/, ), ]); runtime.endpoints.get(endpoint.id)!.posts.length = 0; await invokeTaskCommand({ messageId: 12, prompt: "create the enabled group task", document: { fileId: "telegram-captioned-command-file", fileName: "captioned-command.txt", fileSize: 41, mimeType: "text/plain", }, }); // Exact provider redelivery of the same command remains one task turn. await invokeTaskCommand({ messageId: 12, prompt: "create the enabled group task", document: { fileId: "telegram-captioned-command-file", fileName: "captioned-command.txt", fileSize: 41, mimeType: "text/plain", }, }); const [groupConversation] = await service.listConversations(endpoint.id); if (!groupConversation) throw new Error("Expected enabled Telegram group conversation"); expect(wakeup).toHaveBeenCalledTimes(1); const captionedDelivery = await db .select() .from(chatDeliveries) .where( eq(chatDeliveries.providerEventId, `${group.thread.id}:${chatId}:12`), ) .then((rows) => rows[0]); expect(captionedDelivery?.normalizedEvent).toMatchObject({ message: { attachments: [ { name: "captioned-command.txt", mimeType: "text/plain", size: 41, recovery: { locator: { kind: "test_attachment", recoveryKey: "telegram-captioned-command-file", }, }, }, ], }, }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: group.thread, message: makeMessage({ id: `${chatId}:13`, text: "unrelated-subscribed-private-marker", userId: "telegram-bystander", }), trigger: "subscribed_message", }); const unaddressedDelivery = await db .select() .from(chatDeliveries) .where( eq(chatDeliveries.providerEventId, `${group.thread.id}:${chatId}:13`), ) .then((rows) => rows[0]); expect(unaddressedDelivery).toMatchObject({ state: "filtered", attempts: 0, principalId: null, redactedError: "Message did not address the agent", normalizedEvent: { filtering: { contentRetained: false } }, }); expect(JSON.stringify(unaddressedDelivery.normalizedEvent)).not.toContain( "unrelated-subscribed-private-marker", ); expect(JSON.stringify(unaddressedDelivery.normalizedEvent)).not.toContain( "telegram-bystander", ); expect(wakeup).toHaveBeenCalledTimes(1); // The pinned Telegram adapter marks a direct reply to the bot as a // mention. Simulate that normalized contract on the subscribed path. await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: group.thread, message: makeMessage({ id: `${chatId}:14`, text: "continue by replying directly to Maya", mentioned: true, userId: "telegram-group-user", }), trigger: "subscribed_message", }); expect(wakeup).toHaveBeenCalledTimes(2); const topicId = 91; const topic = makeThread({ channelId: chatId, id: `telegram:${chatId}:${topicId}`, name: "Telegram production group", }); await invokeTaskCommand({ messageId: 20, prompt: "create a topic-bound task", threadId: topic.thread.id, topicId, }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: topic.thread, message: makeMessage({ id: `${chatId}:21`, raw: { message_id: 21, message_thread_id: topicId }, text: "unrelated-topic-private-marker", userId: "telegram-topic-bystander", }), trigger: "subscribed_message", }); const conversations = await service.listConversations(endpoint.id); expect(conversations).toEqual( expect.arrayContaining([ expect.objectContaining({ id: groupConversation.id, issueId: groupConversation.issueId, }), expect.objectContaining({ externalThreadId: topic.thread.id, sessionGeneration: 1, }), ]), ); expect(conversations).toHaveLength(2); expect(wakeup).toHaveBeenCalledTimes(3); await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, groupConversation.issueId)) .orderBy(asc(issueComments.createdAt), asc(issueComments.id)), ).resolves.toEqual([ { body: "create the enabled group task" }, { body: "continue by replying directly to Maya" }, ]); const topicConversation = conversations.find( (candidate) => candidate.externalThreadId === topic.thread.id, )!; await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, topicConversation.issueId)), ).resolves.toEqual([{ body: "create a topic-bound task" }]); await expect( db .select({ id: chatExternalPrincipals.id }) .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, fixture.companyId), eq(chatExternalPrincipals.provider, "telegram"), inArray(chatExternalPrincipals.externalId, [ "telegram-bystander", "telegram-topic-bystander", ]), ), ), ).resolves.toHaveLength(0); }); it.each([ "current", "restart", "unknown_document", "malformed_note", "oversize", "revoked_after_receipt", ] as const)( "ingests a pinned Telegram video-note without optional MIME metadata (%s)", async (mode) => { const fixture = await seedCompany(); const storage = createStorageService(); const deferred = mode === "restart" || mode === "revoked_after_receipt"; const { callbacks, endpoint, runtime, service, wakeup } = await configuredTelegramEndpoint(fixture, { storage: storage.storage, deferWebhookProcessing: deferred, scheduleDeferredWork: () => undefined, }); const configuration = runtime.configurations.get(endpoint.id)!; const pinned = createChatSdkEndpointRuntime({ ...configuration, logger: "silent", }); const providerRequests: string[] = []; let restarted: ReturnType | undefined; let recoveredParser: ReturnType | undefined; const fetchSpy = vi .spyOn(globalThis, "fetch") .mockImplementation(async (input, init) => { const url = new URL( input instanceof Request ? input.url : String(input), ); if (url.hostname !== "api.telegram.org") throw new Error("Unexpected fixture host"); if (url.pathname.endsWith("/getFile")) { expect(JSON.parse(String(init?.body))).toEqual({ file_id: "telegram-video-note-fixture", }); providerRequests.push("getFile"); return Response.json({ ok: true, result: { file_path: "video-notes/fixture.mp4" }, }); } if (url.pathname.endsWith("/video-notes/fixture.mp4")) { providerRequests.push("download"); return new Response(TELEGRAM_VIDEO_NOTE_MP4, { headers: { "content-type": "video/mp4" }, }); } throw new Error("Unexpected fixture provider method"); }); try { // Only provider I/O and scheduling are simulated. Parse with the real // pinned adapter, then use real durable descriptors and service policy. Object.assign(runtime.endpoints.get(endpoint.id)!, { attachmentRecoveryDescriptor: pinned.attachmentRecoveryDescriptor.bind(pinned), rehydrateAttachment: pinned.rehydrateAttachment.bind(pinned), }); const raw = { message_id: 41, date: Math.floor(Date.now() / 1_000), chat: { id: 77115569, type: "private" }, from: { id: 77115569, is_bot: false, first_name: "Video fixture" }, video_note: { file_id: "telegram-video-note-fixture", file_unique_id: "telegram-video-note-unique", length: 16, duration: 1, file_size: TELEGRAM_VIDEO_NOTE_MP4.length, }, }; if (mode === "malformed_note") raw.video_note.duration = -1; if (mode === "oversize") raw.video_note.file_size = MAX_ATTACHMENT_BYTES + 1; const { video_note: _note, ...withoutNote } = raw; const message = pinned.parseTelegramCommandMessage( mode === "unknown_document" ? { ...withoutNote, document: { file_id: "unknown-document", file_unique_id: "unknown-document-unique", file_name: "unknown.bin", file_size: TELEGRAM_VIDEO_NOTE_MP4.length, }, } : raw, )!; const dm = makeThread({ channelId: "77115569", id: "telegram:77115569", isDM: true, }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message, trigger: "direct_message", }); if (deferred) { const [received] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(received).toMatchObject({ state: "received", attempts: 0 }); expect(providerRequests).toEqual([]); expect(storage.putFile).not.toHaveBeenCalled(); expect(wakeup).not.toHaveBeenCalled(); // Stop the original service and throw away all live fetch closures. // The replacement parser must reconstruct the exact durable file ID. message.attachments[0]!.fetchData = vi.fn(async () => { throw new Error("Original live closure must not run after restart"); }); await service.shutdown(); recoveredParser = createChatSdkEndpointRuntime({ ...configuration, logger: "silent", }); const nextRuntime = new FakeChatSdkRuntime(); const replace = nextRuntime.replaceEndpoint.bind(nextRuntime); vi.spyOn(nextRuntime, "replaceEndpoint").mockImplementation( async (options) => { const next = await replace(options); Object.assign(next, { attachmentRecoveryDescriptor: recoveredParser!.attachmentRecoveryDescriptor.bind( recoveredParser, ), rehydrateAttachment: recoveredParser!.rehydrateAttachment.bind(recoveredParser), }); const thread = next.thread.bind(next); vi.spyOn(next, "thread").mockImplementation((threadId) => ({ ...thread(threadId), isDM: recoveredParser!.getProviderAdapter().isDM!(threadId), })); return next; }, ); if (mode === "revoked_after_receipt") { await db .update(chatEndpoints) .set({ allowDirectMessages: false, updatedAt: new Date() }) .where(eq(chatEndpoints.id, endpoint.id)); } await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, received.id)); restarted = createService( nextRuntime, fakeTelegramFetch() as typeof globalThis.fetch, { storage: storage.storage, scheduleDeferredWork: () => undefined }, ); await restarted.service.processPendingDeliveries(25, received.id); expect(message.attachments[0]!.fetchData).not.toHaveBeenCalled(); } if (!["current", "restart"].includes(mode)) { expect(storage.putFile).not.toHaveBeenCalled(); expect(providerRequests).toEqual([]); expect(wakeup).not.toHaveBeenCalled(); expect(restarted?.wakeup.mock.calls.length ?? 0).toBe(0); const [denied] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); if (mode === "revoked_after_receipt") expect(denied.state).toBe("filtered"); else expect(denied.redactedError).toContain( mode === "oversize" ? "declared too large" : "unsupported type", ); await expect( db .select({ id: issueAttachments.id }) .from(issueAttachments) .where(eq(issueAttachments.companyId, fixture.companyId)), ).resolves.toEqual([]); return; } expect(storage.putFile).toHaveBeenCalledOnce(); expect(storage.putFile.mock.calls[0]![0]).toMatchObject({ body: TELEGRAM_VIDEO_NOTE_MP4, contentType: "video/mp4", }); expect(providerRequests).toEqual(["getFile", "download"]); expect(restarted?.wakeup ?? wakeup).toHaveBeenCalledOnce(); const [conversation] = await ( restarted?.service ?? service ).listConversations(endpoint.id); const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(delivery).toMatchObject({ state: "processed", redactedError: null, conversationId: conversation.id, }); expect(delivery.normalizedEvent).toMatchObject({ message: { providerMessageId: "77115569:41", attachments: [ { mimeType: "video/mp4", recovery: { version: 1, provider: "telegram", attachment: { type: "video", mimeType: "video/mp4" }, locator: { kind: "telegram_file_id", fileId: raw.video_note.file_id, fileUniqueId: raw.video_note.file_unique_id, }, }, }, ], }, }); await expect( db .select({ sha256: assets.sha256, byteSize: assets.byteSize, contentType: assets.contentType, }) .from(issueAttachments) .innerJoin(assets, eq(assets.id, issueAttachments.assetId)) .where(eq(issueAttachments.issueId, conversation.issueId)), ).resolves.toEqual([ { sha256: createHash("sha256") .update(TELEGRAM_VIDEO_NOTE_MP4) .digest("hex"), byteSize: TELEGRAM_VIDEO_NOTE_MP4.length, contentType: "video/mp4", }, ]); } finally { try { await pinned.shutdown(); } finally { try { await recoveredParser?.shutdown(); } finally { try { await retirePublicationFixture(service, endpoint.id); } finally { try { await restarted?.service.shutdown(); } finally { fetchSpy.mockRestore(); } } } } } }, ); it("ingests bounded Telegram photo, audio, video, and document attachments without stranding text", async () => { const fixture = await seedCompany(); const storage = createStorageService(); const runtime = new FakeChatSdkRuntime(); const { service, wakeup } = createService( runtime, fakeTelegramFetch() as typeof globalThis.fetch, { storage: storage.storage, }, ); const endpoint = await service.create( fixture.companyId, { provider: "telegram", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await service.configure( endpoint.id, { action: "configure", credentials: { botToken: "123456:telegram-media-test" }, }, "owner-user", ); const callbacks = runtime.configurations.get(endpoint.id)?.callbacks; if (!callbacks) throw new Error("Expected Telegram media callbacks"); const accepted = [ { type: "image", name: "photo.jpg", mimeType: "image/jpeg", body: Buffer.from("photo"), }, { type: "audio", name: "voice.ogg", mimeType: "audio/ogg", body: Buffer.from("voice"), }, { type: "video", name: "clip.mp4", mimeType: "video/mp4", body: Buffer.from("video"), }, { type: "file", name: "notes.txt", mimeType: "text/plain", body: Buffer.from("notes"), }, ] as const; const acceptedFetches = accepted.map(({ body }) => vi.fn(async () => body)); const oversizedDeclaredFetch = vi.fn(async () => Buffer.from("must-not-download"), ); const rejectedTypeFetch = vi.fn(async () => Buffer.from("must-not-download"), ); const oversizedDownloadedFetch = vi.fn(async () => Buffer.alloc(MAX_ATTACHMENT_BYTES + 1), ); const emptyFetch = vi.fn(async () => Buffer.alloc(0)); const failedDownload = vi.fn(async () => { throw new Error("injected Telegram download failure"); }); const attachments: Attachment[] = [ ...accepted.map( (item, index) => ({ type: item.type, name: item.name, mimeType: item.mimeType, size: item.body.length, fetchData: acceptedFetches[index], fetchMetadata: { testRecoveryKey: `telegram-media-${index}` }, }) as Attachment, ), { type: "file", name: "declared-too-large.txt", mimeType: "text/plain", size: MAX_ATTACHMENT_BYTES + 1, fetchData: oversizedDeclaredFetch, }, { type: "file", name: "payload.exe", mimeType: "application/x-msdownload", size: 4, fetchData: rejectedTypeFetch, }, { type: "file", name: "downloaded-too-large.txt", mimeType: "text/plain", fetchData: oversizedDownloadedFetch, }, { type: "file", name: "empty.txt", mimeType: "text/plain", fetchData: emptyFetch, }, { type: "file", name: "unavailable.txt", mimeType: "text/plain", fetchData: failedDownload, }, ]; const dm = makeThread({ channelId: "77115566", id: "telegram:77115566", isDM: true, name: "Telegram media DM", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ attachments, id: "77115566:40", text: "", userId: "77115566", }), trigger: "direct_message", }); expect( acceptedFetches.every((fetchData) => fetchData.mock.calls.length === 1), ).toBe(true); expect(oversizedDeclaredFetch).not.toHaveBeenCalled(); expect(rejectedTypeFetch).not.toHaveBeenCalled(); expect(oversizedDownloadedFetch).toHaveBeenCalledTimes(1); expect(emptyFetch).toHaveBeenCalledTimes(1); expect(failedDownload).toHaveBeenCalledTimes(1); expect(storage.putFile).toHaveBeenCalledTimes(4); const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(delivery).toMatchObject({ state: "processed", attempts: 1, redactedError: expect.stringContaining( "5 external attachments were omitted", ), }); expect(delivery.redactedError).toContain("declared too large: 1"); expect(delivery.redactedError).toContain("unsupported type: 1"); expect(delivery.redactedError).toContain("downloaded too large: 1"); expect(delivery.redactedError).toContain("empty download: 1"); expect(delivery.redactedError).toContain("processing failed: 1"); expect(delivery.redactedError).not.toContain("unavailable.txt"); await expect(service.listActivity(endpoint.id)).resolves.toEqual( expect.arrayContaining([ expect.objectContaining({ id: delivery.id, status: "processed", detail: expect.stringContaining( "5 external attachments were omitted", ), }), ]), ); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Telegram media conversation"); await expect( db .select({ contentType: assets.contentType, originalFilename: assets.originalFilename, }) .from(issueAttachments) .innerJoin(assets, eq(assets.id, issueAttachments.assetId)) .where(eq(issueAttachments.issueId, conversation.issueId)), ).resolves.toEqual( expect.arrayContaining( accepted.map((item) => ({ contentType: item.mimeType, originalFilename: item.name, })), ), ); expect(wakeup).toHaveBeenCalledTimes(1); }); it("surfaces an attachment-only Telegram ingestion failure without waking the agent", async () => { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, service, wakeup } = await configuredTelegramEndpoint(fixture, { storage: storage.storage }); const dm = makeThread({ channelId: "77115567", id: "telegram:77115567", isDM: true, name: "Telegram failed media DM", }); const failedDownload = vi.fn(async () => { throw new Error("injected Telegram attachment outage"); }); try { await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ attachments: [ { type: "file", name: "request.txt", mimeType: "text/plain", size: 42, fetchData: failedDownload, fetchMetadata: { testRecoveryKey: "telegram-failed-media" }, } as Attachment, ], id: "77115567:41", text: "", userId: "77115567", }), trigger: "direct_message", }); expect(failedDownload).toHaveBeenCalledTimes(1); expect(storage.putFile).not.toHaveBeenCalled(); expect(wakeup).not.toHaveBeenCalled(); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Telegram attachment failure conversation"); const visibleFailure = `Paperclip could not safely import the attached Telegram file. Please resend it as a supported file under ${formatAttachmentSize(MAX_ATTACHMENT_BYTES)} or include text describing the request.`; await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, conversation.issueId)), ).resolves.toEqual([{ body: visibleFailure }]); const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(delivery).toMatchObject({ state: "processed", redactedError: expect.stringContaining( "1 external attachment was omitted", ), }); await vi.waitFor(() => expect(dm.post).toHaveBeenCalledWith(visibleFailure), ); await expect( db .select({ kind: chatActions.kind, status: chatActions.status }) .from(chatActions) .where(eq(chatActions.deliveryId, delivery.id)), ).resolves.toEqual( expect.arrayContaining([ { kind: "inbound_wakeup", status: "failed" }, { kind: "provider_effect", status: "processed" }, ]), ); } finally { await retirePublicationFixture(service, endpoint.id); } }); it.each([ { command: "/new", bound: true, denied: false }, { command: "/status", bound: true, denied: false }, { command: "/close", bound: true, denied: false }, { command: "/help", bound: true, denied: false }, { command: "/new", bound: false, denied: false }, { command: "/start", bound: false, denied: false }, { command: "/task", bound: false, denied: false }, { command: "/status", bound: false, denied: true }, ])( "acknowledges Telegram control $command without a processing receipt (bound=$bound denied=$denied)", async ({ command, bound, denied }) => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredTelegramEndpoint(fixture, { scheduleDeferredWork: () => undefined, }); const dm = makeThread({ channelId: "77115570", id: "telegram:77115570", isDM: true, name: "Telegram control receipt", }); try { await db .update(chatEndpoints) .set({ status: "active", ...(denied ? { allowUnlinkedPeople: false } : {}), }) .where(eq(chatEndpoints.id, endpoint.id)); if (bound) { await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: "77115570:50", text: "Ordinary work keeps its processing receipt", userId: "77115570", }), trigger: "direct_message", }); } const providerRuntime = runtime.endpoints.get(endpoint.id)!; const originalReceipts = [...providerRuntime.reactions]; expect(originalReceipts).toHaveLength(bound ? 1 : 0); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: "77115570:51", text: command, userId: "77115570", }), trigger: "direct_message", }); await service.processPendingPublications(); const [delivery] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), like(chatDeliveries.providerEventId, "%77115570:51"), ), ); expect(delivery).toMatchObject({ state: denied ? "filtered" : "processed", }); expect(providerRuntime.reactions).toEqual(originalReceipts); await expect( db .select({ id: chatActions.id }) .from(chatActions) .where( and( eq(chatActions.deliveryId, delivery.id), eq(chatActions.kind, "receipt_reaction"), ), ), ).resolves.toEqual([]); expect(wakeup).toHaveBeenCalledTimes(bound ? 1 : 0); if (!denied) { expect( providerRuntime.posts.length + dm.post.mock.calls.length, ).toBeGreaterThan(0); } else { expect(providerRuntime.posts).toEqual([]); expect(dm.post).not.toHaveBeenCalled(); } } finally { await retirePublicationFixture(service, endpoint.id); } }, ); it.each([ "ordinary", "final_before_add", "newer_followup", "restart", "superseded", "remove_retry", ] as const)( "retires only the exact Telegram final receipt through the pinned adapter (%s)", async (mode) => { const fixture = await seedCompany(); let earlyFinal: | (( agentId: string, opts: Parameters< ChatChannelServiceOptions["heartbeat"]["wakeup"] >[1], ) => Promise) | undefined; const context = await configuredTelegramEndpoint(fixture, { scheduleDeferredWork: () => undefined, ...(mode === "final_before_add" ? { wakeup: async (agentId, opts) => earlyFinal!(agentId, opts) } : {}), }); const { callbacks, endpoint, runtime, service, wakeup } = context; let activeService = service; const pinned = createChatSdkEndpointRuntime({ ...runtime.configurations.get(endpoint.id)!, logger: "silent", }); const adapter = pinned.getProviderAdapter(); const reactionRequests: Array> = []; let failRemoval = mode === "remove_retry"; const fetchSpy = vi .spyOn(globalThis, "fetch") .mockImplementation(async (input, init) => { if (!String(input).endsWith("/setMessageReaction")) throw new Error("Unexpected Telegram fixture HTTP"); const body = JSON.parse(String(init?.body)) as Record< string, unknown >; reactionRequests.push(body); if ( failRemoval && Array.isArray(body.reaction) && body.reaction.length === 0 ) { failRemoval = false; return Response.json( { ok: false, error_code: 429, description: "Too Many Requests", parameters: { retry_after: 60 }, }, { status: 429 }, ); } return Response.json({ ok: true, result: true }); }); const attachPinnedReactions = (providerRuntime: FakeEndpointRuntime) => { const thread = providerRuntime.thread.bind(providerRuntime); providerRuntime.thread = (id) => { const value = thread(id); value.adapter.addReaction = (...args) => adapter.addReaction(...args); value.adapter.removeReaction = (...args) => adapter.removeReaction(...args); return value; }; }; const originalRuntime = runtime.endpoints.get(endpoint.id)!; attachPinnedReactions(originalRuntime); const dm = makeThread({ channelId: "77115569", id: "telegram:77115569", isDM: true, name: "Telegram terminal receipt", }); const runId = randomUUID(); const publishFinal = async () => { const [conversation] = await service.listConversations(endpoint.id); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "succeeded", contextSnapshot: await chatWakeContext({ endpointId: endpoint.id, issueId: conversation.issueId, provider: "telegram", providerMessageId: "77115569:43", }), }); await addSelectedChatFinal({ agentId: fixture.assignedAgentId, body: "TELEGRAM-RECEIPT-FINAL", companyId: fixture.companyId, issueId: conversation.issueId, runId, }); await service.processPendingPublications(); }; earlyFinal = async (agentId, opts) => { const request = opts.durableChatRequest!; await db.transaction(async (tx) => { await request.authorize(tx as never); await tx.insert(agentWakeupRequests).values({ id: request.id, companyId: request.companyId, agentId, source: opts.source!, triggerDetail: opts.triggerDetail, reason: opts.reason, payload: opts.payload, requestedByActorType: opts.requestedByActorType, requestedByActorId: opts.requestedByActorId, idempotencyKey: request.idempotencyKey, requestedAt: request.requestedAt, status: "queued", }); }); expect(reactionRequests).toEqual([]); await publishFinal(); return { runId }; }; try { await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, endpoint.id)); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: "77115569:43", text: "Reply once, then retire this receipt", userId: "77115569", }), trigger: "direct_message", }); if (mode !== "final_before_add") await publishFinal(); const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); const [removal] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq( chatActions.providerActionId, `receipt_reaction_remove:${delivery.id}`, ), ), ); expect(removal).toBeDefined(); if (mode === "newer_followup") { await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: "77115569:44", text: "A separate turn must retain its own receipt", userId: "77115569", }), trigger: "direct_message", }); } if (mode === "restart") { await service.shutdown(); const next = createService( new FakeChatSdkRuntime(), fakeTelegramFetch() as typeof globalThis.fetch, { scheduleDeferredWork: () => undefined }, ); activeService = next.service; next.runtime.initializeHook = async (id) => { if (id === endpoint.id) attachPinnedReactions(next.runtime.endpoints.get(id)!); }; } if (mode === "superseded") { const [current] = await db .select() .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)); await db .update(chatEndpoints) .set({ setup: { ...current.setup, runtimeGeneration: Number(current.setup.runtimeGeneration ?? 0) + 1, }, }) .where(eq(chatEndpoints.id, endpoint.id)); } await activeService.processPendingReceiptReactions(1, removal!.id); if (mode === "remove_retry") { const [failed] = await db .select() .from(chatActions) .where(eq(chatActions.id, removal!.id)); expect(failed).toMatchObject({ status: "failed", result: { retryable: true, attempts: 1 }, }); await db .update(chatActions) .set({ result: { ...failed.result, retryAt: new Date(0).toISOString() }, }) .where(eq(chatActions.id, failed.id)); } await activeService.processPendingReceiptReactions(1, removal!.id); await activeService.processPendingReceiptReactions(1, removal!.id); const adds = reactionRequests.filter( (value) => (value.reaction as unknown[]).length > 0, ); const removals = reactionRequests.filter( (value) => (value.reaction as unknown[]).length === 0, ); expect(adds.map((value) => value.message_id)).toEqual( mode === "final_before_add" ? [] : mode === "newer_followup" ? [43, 44] : [43], ); expect(removals).toEqual( Array.from( { length: mode === "superseded" ? 0 : mode === "remove_retry" ? 2 : 1, }, () => ({ chat_id: "77115569", message_id: 43, reaction: [] }), ), ); const [add] = await db .select() .from(chatActions) .where( eq(chatActions.providerActionId, `receipt_reaction:${delivery.id}`), ); await db .update(chatActions) .set({ status: "failed", result: { retryable: true, retryAt: new Date(0).toISOString() }, }) .where(eq(chatActions.id, add.id)); await activeService.processPendingReceiptReactions(1, add.id); expect( reactionRequests.filter( (value) => (value.reaction as unknown[]).length > 0, ), ).toEqual(adds); expect( [...originalRuntime.posts, ...originalRuntime.edits].filter( (entry) => entry.text === "TELEGRAM-RECEIPT-FINAL", ), ).toHaveLength(1); expect(wakeup).toHaveBeenCalledTimes(mode === "newer_followup" ? 2 : 1); await expect( db .select({ status: heartbeatRuns.status }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, runId)), ).resolves.toEqual([{ status: "succeeded" }]); } finally { fetchSpy.mockRestore(); await pinned.shutdown(); await retirePublicationFixture(activeService, endpoint.id); if (activeService !== service) await service.shutdown(); } }, ); it("defers a rate-limited Telegram receipt reaction for the full provider interval without blocking the inbound turn", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service, wakeup } = await configuredTelegramEndpoint(fixture); const dm = makeThread({ channelId: "77115568", id: "telegram:77115568", isDM: true, name: "Telegram receipt retry DM", }); const endpointRuntime = runtime.endpoints.get(endpoint.id); if (!endpointRuntime) throw new Error("Expected Telegram runtime"); endpointRuntime.reactionErrors.push( Object.assign(new Error("Telegram flood control"), { adapter: "telegram", code: "RATE_LIMITED", name: "AdapterRateLimitError", retryAfter: 60 * 60, }), ); const startedAt = Date.now(); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: "77115568:42", text: "Do the work while the receipt is rate limited", userId: "77115568", }), trigger: "direct_message", }); expect(Date.now() - startedAt).toBeLessThan(5_000); expect(wakeup).toHaveBeenCalledTimes(1); const [reactionAction] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "receipt_reaction"), ), ); expect(reactionAction).toMatchObject({ status: "failed", result: { attempts: 1, retryable: true, code: "receipt_reaction_retry", }, }); expect( Date.parse(String(reactionAction.result?.retryAt)) - startedAt, ).toBeGreaterThanOrEqual(60 * 60 * 1000 - 1_000); await db .update(chatActions) .set({ result: { ...reactionAction.result, retryAt: new Date(0).toISOString(), }, }) .where(eq(chatActions.id, reactionAction.id)); await service.processPendingDeliveries(); await expect( db .select({ status: chatActions.status, result: chatActions.result }) .from(chatActions) .where(eq(chatActions.id, reactionAction.id)), ).resolves.toEqual([{ status: "processed", result: { attempts: 2 } }]); expect(runtime.endpoints.get(endpoint.id)?.reactions).toContainEqual({ threadId: dm.thread.id, messageId: "77115568:42", emoji: "eyes", }); }); it("honors Telegram flood-control timing and publishes one final message after retry", async () => { const fixture = await seedCompany(); const { callbacks, endpoint, runtime, service } = await configuredTelegramEndpoint(fixture); const dm = makeThread({ channelId: "77116677", id: "telegram:77116677", isDM: true, name: "Telegram flood-control DM", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: "77116677:50", text: "Exercise Telegram flood control", userId: "77116677", }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, "77116677"); await service.test(endpoint.id, "owner-user"); const [conversation] = await service.listConversations(endpoint.id); if (!conversation) throw new Error("Expected Telegram flood-control conversation"); const comment = await issueService(db).addComment( conversation.issueId, "Telegram flood-control final", { userId: "owner-user" }, { authorType: "user" }, ); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Telegram provider runtime"); providerRuntime.postError = Object.assign( new Error("Telegram flood control"), { name: "AdapterRateLimitError", adapter: "telegram", code: "RATE_LIMITED", retryAfter: 7, }, ); const beforeAttempt = Date.now(); await service.publishComment(endpoint.id, conversation.id, comment.id); const [retrying] = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, comment.id)); expect(retrying).toMatchObject({ state: "retry", attempts: 1 }); expect(retrying.nextAttemptAt?.getTime()).toBeGreaterThanOrEqual( beforeAttempt + 6_500, ); expect(providerRuntime.posts).toEqual([]); await db .update(chatPublications) .set({ attempts: 4, nextAttemptAt: new Date(0) }) .where(eq(chatPublications.id, retrying.id)); await service.processPendingPublications(); const [stillRetrying] = await db .select() .from(chatPublications) .where(eq(chatPublications.id, retrying.id)); expect(stillRetrying).toMatchObject({ state: "retry", attempts: 5 }); providerRuntime.postError = null; await db .update(chatPublications) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatPublications.id, retrying.id)); await service.processPendingPublications(); await expect( db .select({ attempts: chatPublications.attempts, state: chatPublications.state, }) .from(chatPublications) .where(eq(chatPublications.id, retrying.id)), ).resolves.toEqual([{ attempts: 6, state: "published" }]); expect(providerRuntime.posts).toEqual([ { threadId: dm.thread.id, text: "Telegram flood-control final" }, ]); }); it("recovers the same Telegram bot and historical task after token rotation", async () => { const fixture = await seedCompany(); const firstToken = "445500:telegram-rotation-first"; const replacementToken = "445500:telegram-rotation-replacement"; const registeredTokens: string[] = []; const providerFetch = vi.fn(async (input: string | URL | Request) => { const url = String(input); const token = url.includes(encodeURIComponent(replacementToken)) ? replacementToken : firstToken; if (url.endsWith("/getMe")) { return new Response( JSON.stringify({ ok: true, result: { id: 445500, username: "maya_rotation_bot", first_name: "Maya", }, }), { status: 200, headers: { "content-type": "application/json" } }, ); } if (url.endsWith("/getWebhookInfo")) { return new Response(JSON.stringify({ ok: true, result: { url: "" } }), { status: 200, headers: { "content-type": "application/json" }, }); } if (url.endsWith("/setWebhook")) { registeredTokens.push(token); return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } if (url.endsWith("/setMyCommands")) { return new Response(JSON.stringify({ ok: true, result: true }), { status: 200, headers: { "content-type": "application/json" }, }); } throw new Error("Unexpected Telegram rotation fixture request"); }) as typeof globalThis.fetch; const runtime = new FakeChatSdkRuntime(); const { service } = createService(runtime, providerFetch); const endpoint = await service.create( fixture.companyId, { provider: "telegram", assignedAgentId: fixture.assignedAgentId }, "owner-user", ); await service.configure( endpoint.id, { action: "configure", credentials: { botToken: firstToken } }, "owner-user", ); const firstCallbacks = runtime.configurations.get(endpoint.id)?.callbacks; if (!firstCallbacks) throw new Error("Expected initial Telegram callbacks"); const dm = makeThread({ channelId: "77117788", id: "telegram:77117788", isDM: true, name: "Telegram rotation DM", }); await deliverMessage({ callbacks: firstCallbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: "77117788:60", text: "Create the durable task before rotation", userId: "77117788", }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, "77117788"); await service.test(endpoint.id, "owner-user"); const [beforeRotation] = await service.listConversations(endpoint.id); if (!beforeRotation) throw new Error("Expected pre-rotation Telegram conversation"); const failureComment = await issueService(db).addComment( beforeRotation.issueId, "Detect revoked Telegram credentials", { userId: "owner-user" }, { authorType: "user" }, ); const activeRuntime = runtime.endpoints.get(endpoint.id); if (!activeRuntime) throw new Error("Expected active Telegram runtime"); activeRuntime.postError = Object.assign(new Error("Unauthorized"), { name: "AuthenticationError", adapter: "telegram", code: "AUTH_FAILED", }); await service.publishComment( endpoint.id, beforeRotation.id, failureComment.id, ); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "attention", healthMessage: "Provider credentials or permissions need attention", }); expect(runtime.endpoints.has(endpoint.id)).toBe(false); const reconnected = await service.configure( endpoint.id, { action: "reconnect", credentials: { botToken: replacementToken } }, "owner-user", ); expect(reconnected).toMatchObject({ status: "verifying", botExternalId: "445500", botUsername: "maya_rotation_bot", setup: { step: "test" }, }); expect(registeredTokens).toEqual([firstToken, replacementToken]); const replacementCallbacks = runtime.configurations.get( endpoint.id, )?.callbacks; if (!replacementCallbacks) throw new Error("Expected replacement Telegram callbacks"); await deliverMessage({ callbacks: replacementCallbacks, endpointId: endpoint.id, provider: "telegram", thread: dm.thread, message: makeMessage({ id: "77117788:61", text: "Continue the same task after rotation", userId: "77117788", }), trigger: "direct_message", }); await qualifySetupRoundTrip(service, endpoint.id, "77117788"); await service.test(endpoint.id, "owner-user"); const [afterRotation] = await service.listConversations(endpoint.id); expect(afterRotation).toMatchObject({ id: beforeRotation.id, issueId: beforeRotation.issueId, state: "active", }); await expect( db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.issueId, beforeRotation.issueId)), ).resolves.toEqual( expect.arrayContaining([ { body: "Create the durable task before rotation" }, { body: "Continue the same task after rotation" }, ]), ); const safeEndpoint = await service.get(endpoint.id); expect(safeEndpoint.status).toBe("active"); expect(JSON.stringify(safeEndpoint)).not.toContain(firstToken); expect(JSON.stringify(safeEndpoint)).not.toContain(replacementToken); }); async function committedChatResponseRecoveryFixture( provider: "slack" | "telegram" | "github", format: "coordinator" | "port" = "coordinator", githubUnavailableFile = false, beforeAcceptance?: ( context: Awaited>, ) => Promise, summary?: (issueId: string) => string, ) { const context = await failedChatRetryFixture( provider, provider === "github" ? "42" : undefined, false, githubUnavailableFile, ); await beforeAcceptance?.(context); const contractId = randomUUID(); const sessionId = randomUUID(); const runnerId = randomUUID(); const contractSha256 = `recovered-response-${randomUUID()}`; await db.insert(completionContracts).values({ id: contractId, companyId: context.fixture.companyId, issueId: context.issue.id, revision: 1, schemaVersion: "paperclip.completion-contract.v1", policyVersion: "phase6-v3", risk: "low", completionAuthority: "agent_claim_policy", incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: { revision: "recovered-response-v1", objective: "Answer the exact message", criteria: [ { id: "response", requirement: "Return the requested answer" }, ], }, canonicalSha256: contractSha256, createdByActorType: "system", createdByActorId: "test", }); await db .update(heartbeatRuns) .set({ runtimeMode: "native", nativeIssueId: context.issue.id, nativeSessionId: sessionId, runnerInstanceId: runnerId, completionContractId: contractId, completionContractSha256: contractSha256, status: "failed", errorCode: "adapter_failed", error: "provider_transport_failed: checkpoint is quarantined", finishedAt: new Date(), processPid: 987654, runnerProfileJson: { sessionCheckpoint: { retainedEvidence: "do not alter" }, }, }) .where(eq(heartbeatRuns.id, context.runId)); await db .update(issues) .set({ status: "in_review", executionRunId: null }) .where(eq(issues.id, context.issue.id)); const result: PrpStructuredRunResult = { schema: "paperclip.run_result.v1", reportedWorkDisposition: "yielded", summary: summary?.(context.issue.id) ?? "This is the exact accepted answer. I will wait for your next message.", completionClaim: { contractRevision: "recovered-response-v1", objectiveSatisfied: true, criteria: [ { criterionId: "response", status: "satisfied", evidenceRefs: [] }, ], remainingWork: [], }, evidence: [], verification: [], attentionRequests: [], artifacts: [], continuation: { kind: "response_wake", summary: "Wait for the next authorized message.", idempotencyKey: `wait:${context.runId}`, }, }; const terminal: PrpTerminalState = { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "yielded", workAssessmentId: randomUUID(), statusDecisionId: randomUUID(), }; const binding = { companyId: context.fixture.companyId, issueId: context.issue.id, runId: context.runId, agentId: context.fixture.assignedAgentId, normalizedSessionId: sessionId, runnerSourceInstanceId: runnerId, completionContractId: contractId, completionContractSha256: contractSha256, completionContractRevision: "recovered-response-v1", completionContractCriterionIds: ["response"], }; if (format === "coordinator") { await new NativeRunCoordinatorStore(db, binding).completeRun({ result, terminal, turnId: `turn-${context.runId}`, }); } else { const controlPlaneSourceInstanceId = `control-${context.runId}`; const port = new PaperclipControlPlanePort(db, { companyId: binding.companyId, issueId: binding.issueId, runId: binding.runId, agentId: binding.agentId, sessionId, completionContractId: contractId, completionContractSha256: contractSha256, sourceInstanceId: runnerId, controlPlaneSourceInstanceId, }); await port.openRun({ identity: { companyId: binding.companyId, issueId: binding.issueId, runId: binding.runId, agentId: binding.agentId, sessionId, }, backendKind: "mock", sourceInstanceId: runnerId, }); await port.completeRun({ result, terminal, turnId: `turn-${context.runId}`, }); await db.insert(heartbeatRunEvents).values({ companyId: binding.companyId, agentId: binding.agentId, runId: binding.runId, seq: 1, eventType: "run.result.accepted", sourceInstanceId: controlPlaneSourceInstanceId, payload: { prpEvent: { sourceKind: "control_plane" } }, }); } const [acceptedRow] = await db .select() .from(nativeRunResults) .where(eq(nativeRunResults.runId, context.runId)); const accepted = { resultId: acceptedRow.id }; await finalizeNativeRun({ db, runId: context.runId, workspaceFinalizeStatus: "succeeded", }); const [coordinator] = await db .select() .from(nativeRunFinalizations) .where(eq(nativeRunFinalizations.runId, context.runId)); expect(coordinator.phase).toBe("committed"); await db .update(heartbeatRuns) .set({ resultJson: { finalResponse: "PRIVATE provider narration must not be recovered", privateTrace: "PRIVATE tool arguments", }, }) .where(eq(heartbeatRuns.id, context.runId)); const [beforeIssue] = await db .select() .from(issues) .where(eq(issues.id, context.issue.id)); const repair = () => repairCommittedNativeChatResponse(db, { companyId: context.fixture.companyId, issueId: context.issue.id, runId: context.runId, }); return { ...context, accepted, result, terminal, binding, coordinator, beforeIssue, repair, }; } it.each(["slack", "telegram"] as const)( "recovers a committed %s answer once without rerunning or changing review/quarantine evidence", async (provider) => { const context = await committedChatResponseRecoveryFixture(provider); try { expect(await Promise.all([context.repair(), context.repair()])).toEqual( expect.arrayContaining([true, false]), ); await expect(context.repair()).resolves.toBe(false); const comments = await db .select() .from(issueComments) .where(eq(issueComments.createdByRunId, context.runId)); expect(comments).toHaveLength(1); expect(comments[0].body).toBe(context.result.summary); const publications = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, comments[0].id)); expect(publications).toHaveLength(1); expect(publications[0]).toMatchObject({ state: "pending", endpointId: context.endpoint.id, conversationId: context.conversation.id, }); const [run] = await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, context.runId)); expect(run).toMatchObject({ status: "succeeded", errorCode: null, error: null, processPid: 987654, runnerProfileJson: { sessionCheckpoint: { retainedEvidence: "do not alter" }, }, resultJson: { recoveredExecutionFailure: { errorCode: "adapter_failed", error: "provider_transport_failed: checkpoint is quarantined", }, nativeCommittedChatResponse: { resultId: context.accepted.resultId, }, presentationDecision: { commentId: comments[0].id }, }, }); const [afterIssue] = await db .select() .from(issues) .where(eq(issues.id, context.issue.id)); expect(afterIssue).toMatchObject({ status: context.beforeIssue.status, statusVersion: context.beforeIssue.statusVersion, lastStatusDecisionId: context.beforeIssue.lastStatusDecisionId, executionRunId: context.beforeIssue.executionRunId, }); await context.service.processPendingPublications(100); expect( JSON.stringify({ posts: context.providerRuntime.posts, edits: context.providerRuntime.edits, }), ).not.toContain("PRIVATE"); expect( [ ...context.providerRuntime.posts, ...context.providerRuntime.edits, ].filter((entry) => entry.text === context.result.summary), ).toHaveLength(1); } finally { await context.service.shutdown(); } }, ); it.each([ "source_edited", "source_deleted", "principal_revoked", "endpoint_paused", "generation_changed", "digest_changed", "private_disposition", "missing_authority", "cross_company", "cross_run", "ambiguous_delivery", "live_heartbeat", ] as const)( "does not recover a committed answer after %s", async (mutation) => { const context = await committedChatResponseRecoveryFixture("slack"); try { if (mutation === "source_edited") await db .update(issueComments) .set({ body: "Changed request", updatedAt: new Date(Date.now() + 1), }) .where( eq(issueComments.id, String(context.action.payload.commentId)), ); if (mutation === "source_deleted") await db .update(issueComments) .set({ deletedAt: new Date() }) .where( eq(issueComments.id, String(context.action.payload.commentId)), ); if (mutation === "principal_revoked") await db .update(chatEndpoints) .set({ allowUnlinkedPeople: false }) .where(eq(chatEndpoints.id, context.endpoint.id)); if (mutation === "endpoint_paused") await db .update(chatEndpoints) .set({ status: "paused" }) .where(eq(chatEndpoints.id, context.endpoint.id)); if (mutation === "generation_changed") await db .update(chatEndpoints) .set({ setup: sql`jsonb_set(${chatEndpoints.setup}, '{runtimeGeneration}', to_jsonb(coalesce((${chatEndpoints.setup}->>'runtimeGeneration')::int, 0) + 1))`, }) .where(eq(chatEndpoints.id, context.endpoint.id)); if (mutation === "ambiguous_delivery") await db.insert(chatPublications).values({ companyId: context.fixture.companyId, endpointId: context.endpoint.id, conversationId: context.conversation.id, issueId: context.issue.id, idempotencyKey: `run:${context.runId}:working:${context.endpoint.id}`, payload: { text: "Maya is working…" }, state: "delivery_unknown", }); if (mutation === "digest_changed") await db .update(nativeRunResults) .set({ canonicalSha256: "sha256:changed" }) .where(eq(nativeRunResults.id, context.accepted.resultId)); if (mutation === "private_disposition") await db .update(nativeRunResults) .set({ resultJson: { result: { ...context.result, reportedWorkDisposition: "needs_review", }, terminal: {}, }, }) .where(eq(nativeRunResults.id, context.accepted.resultId)); if (mutation === "missing_authority") await context.service.shutdown(); if (mutation === "live_heartbeat") await db .update(heartbeatRuns) .set({ status: "running", finishedAt: null }) .where(eq(heartbeatRuns.id, context.runId)); await expect( mutation === "cross_company" ? repairCommittedNativeChatResponse(db, { companyId: randomUUID(), issueId: context.issue.id, runId: context.runId, }) : mutation === "cross_run" ? repairCommittedNativeChatResponse(db, { companyId: context.fixture.companyId, issueId: context.issue.id, runId: randomUUID(), }) : context.repair(), ).resolves.toBe(false); await expect( db .select() .from(issueComments) .where(eq(issueComments.createdByRunId, context.runId)), ).resolves.toHaveLength(0); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.endpointId, context.endpoint.id)), ).resolves.toHaveLength(mutation === "ambiguous_delivery" ? 1 : 0); } finally { await context.service.shutdown(); } }, ); it("recovers the exact port-bound canonical response before skipping a superseded native decision", async () => { const context = await committedChatResponseRecoveryFixture( "telegram", "port", ); try { const [originalRun] = await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, context.runId)); const laterId = randomUUID(); await db.insert(heartbeatRuns).values({ ...originalRun, id: laterId, wakeupRequestId: null, nativeSessionId: randomUUID(), runnerInstanceId: randomUUID(), contextSnapshot: {}, status: "running", resultJson: null, }); const laterResult = { ...context.result, reportedWorkDisposition: "needs_review" as const, }; delete laterResult.continuation; await new NativeRunCoordinatorStore(db, { ...context.binding, runId: laterId, }).completeRun({ result: laterResult, terminal: { ...context.terminal, reportedWorkDisposition: "needs_review", }, }); await finalizeNativeRun({ db, runId: laterId, workspaceFinalizeStatus: "succeeded", }); const [laterIssue] = await db .select() .from(issues) .where(eq(issues.id, context.issue.id)); expect(laterIssue.lastStatusDecisionId).not.toBe( context.coordinator.decisionId, ); await reconcileNativeFinalizations(db, [context.runId]); await reconcileNativeFinalizations(db, [context.runId]); const comments = await db .select() .from(issueComments) .where(eq(issueComments.createdByRunId, context.runId)); expect(comments).toHaveLength(1); expect(comments[0].body).toBe(context.result.summary); const [after] = await db .select() .from(issues) .where(eq(issues.id, context.issue.id)); expect(after).toMatchObject({ status: laterIssue.status, statusVersion: laterIssue.statusVersion, lastStatusDecisionId: laterIssue.lastStatusDecisionId, }); } finally { await context.service.shutdown(); } }); async function githubOmissionFixture( unavailable = true, beforeAcceptance?: Parameters< typeof committedChatResponseRecoveryFixture >[3], summary?: Parameters[4], ) { const egress = vi .spyOn(attachmentEgress, "guardedRemoteHttpFetch") .mockResolvedValue( new Response("private file unavailable", { status: 404 }), ); try { const context = await committedChatResponseRecoveryFixture( "github", "coordinator", unavailable, beforeAcceptance, summary, ); return { ...context, egress, cleanup: async () => { try { await context.service.shutdown(); } finally { egress.mockRestore(); } }, }; } catch (error) { egress.mockRestore(); throw error; } } it("adds a durable safe task link to the exact GitHub native omission final", async () => { const context = await githubOmissionFixture(); try { expect(context.action.payload.attachmentOmissionReasons).toEqual({ download_unavailable: 1, }); expect(context.egress).toHaveBeenCalledTimes(1); await expect(context.repair()).resolves.toBe(true); await context.service.processPendingPublications(100); const taskUrl = `https://paperclip.example/issues/${context.issue.id}`; const expected = `${context.result.summary}\n\n[Open this Paperclip task](${taskUrl})`; expect(context.providerRuntime.posts.map((post) => post.text)).toEqual([ expected, ]); const [publication] = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, context.endpoint.id), isNotNull(chatPublications.commentId), ), ); expect(publication).toMatchObject({ state: "published", attempts: 1, payload: { text: expected }, }); await context.service.processPendingPublications(100); expect(context.providerRuntime.posts).toHaveLength(1); await expect( db .select({ id: issueAttachments.id }) .from(issueAttachments) .where(eq(issueAttachments.issueId, context.issue.id)), ).resolves.toEqual([]); } finally { await context.cleanup(); } }); it.each([ { base: null, expected: null }, { base: "http://127.0.0.1:3137", expected: null }, { base: "https://user:secret@board.example", expected: null }, { base: "https://10.0.0.1", expected: null }, { base: "https://board.example/prefix?token=PRIVATE#fragment", expected: "https://board.example", }, ])( "uses only the configured safe Board origin for GitHub omission navigation: $base", async ({ base, expected }) => { const context = await githubOmissionFixture(); let restarted: ReturnType | undefined; try { await expect(context.repair()).resolves.toBe(true); await context.service.shutdown(); restarted = createService( new FakeChatSdkRuntime(), context.providerFetch!, { publicBaseUrl: base, webhookPublicBaseUrl: "https://ingress.example:8443", }, ); await restarted.service.processPendingPublications(100); const text = restarted.runtime.endpoints.get(context.endpoint.id)! .posts[0]!.text; expect(text).toBe( expected ? `${context.result.summary}\n\n[Open this Paperclip task](${expected}/issues/${context.issue.id})` : context.result.summary, ); for (const secret of [ "PRIVATE", "secret@", "127.0.0.1", "10.0.0.1", "ingress.example", "#fragment", ]) expect(text).not.toContain(secret); } finally { await restarted?.service.shutdown(); await context.cleanup(); } }, ); it.each(["append_link", "existing_link"] as const)( "keeps GitHub omission navigation byte-stable across a retry and Board-origin change: %s", async (mode) => { const context = await githubOmissionFixture( true, undefined, mode === "existing_link" ? (issueId) => `Attach directly: [Open this Paperclip task](https://paperclip.example/issues/${issueId})` : undefined, ); let restarted: ReturnType | undefined; try { await expect(context.repair()).resolves.toBe(true); context.providerRuntime.postError = Object.assign( new Error("Rate limited"), { name: "RateLimitError", retryAfter: 60 }, ); await context.service.processPendingPublications(100); const [before] = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, context.endpoint.id), isNotNull(chatPublications.commentId), ), ); expect(before).toMatchObject({ state: "retry", attempts: 1 }); expect( before.payload.text.match(/Open this Paperclip task/g), ).toHaveLength(1); const [preparation] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, context.endpoint.id), eq(chatActions.kind, "github_omission_navigation"), ), ); expect(preparation.payload).toMatchObject({ publicationId: before.id, runId: context.runId, resultId: context.accepted.resultId, preparedTextSha256: createHash("sha256") .update(before.payload.text) .digest("hex"), }); await context.service.shutdown(); restarted = createService( new FakeChatSdkRuntime(), context.providerFetch!, { publicBaseUrl: "https://changed.example" }, ); await db .update(chatPublications) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatPublications.id, before.id)); await restarted.service.processPendingPublications(100); await restarted.service.processPendingPublications(100); expect( restarted.runtime.endpoints .get(context.endpoint.id)! .posts.map((post) => post.text), ).toEqual([before.payload.text]); await expect( db .select({ state: chatPublications.state, attempts: chatPublications.attempts, payload: chatPublications.payload, }) .from(chatPublications) .where(eq(chatPublications.id, before.id)), ).resolves.toEqual([ { state: "published", attempts: 2, payload: before.payload }, ]); } finally { await restarted?.service.shutdown(); await context.cleanup(); } }, ); it.each([ "no_omission", "forged_hint", "explicit_board", "progress", "source_edited", "principal_revoked", "generation_changed", ] as const)( "does not grant GitHub omission navigation from $0", async (mode) => { const context = await githubOmissionFixture( !["no_omission", "forged_hint"].includes(mode), ); try { await expect(context.repair()).resolves.toBe(true); const [publication] = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, context.endpoint.id), isNotNull(chatPublications.commentId), ), ); if (mode === "forged_hint") await db .update(chatPublications) .set({ payload: { ...publication.payload, attachmentOmissionReasons: { download_unavailable: 1 }, } as typeof publication.payload, }) .where(eq(chatPublications.id, publication.id)); if (mode === "explicit_board") await db .update(chatPublications) .set({ idempotencyKey: `explicit-board:${publication.id}` }) .where(eq(chatPublications.id, publication.id)); if (mode === "progress") await db .update(chatPublications) .set({ payload: { ...publication.payload, progressState: "completed" }, }) .where(eq(chatPublications.id, publication.id)); if (mode === "source_edited") await db .update(issueComments) .set({ body: "Source changed", updatedAt: new Date(Date.now() + 1), }) .where( eq(issueComments.id, String(context.action.payload.commentId)), ); if (mode === "principal_revoked") await db .update(chatEndpoints) .set({ allowUnlinkedPeople: false }) .where(eq(chatEndpoints.id, context.endpoint.id)); if (mode === "generation_changed") await db .update(chatEndpoints) .set({ setup: sql`jsonb_set(${chatEndpoints.setup}, '{runtimeGeneration}', to_jsonb(coalesce((${chatEndpoints.setup}->>'runtimeGeneration')::int, 0) + 1))`, }) .where(eq(chatEndpoints.id, context.endpoint.id)); await context.service.processPendingPublications(100); if ( ["source_edited", "principal_revoked", "generation_changed"].includes( mode, ) ) { expect(context.providerRuntime.posts).toEqual([]); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.id, publication.id)), ).resolves.toEqual([{ state: "cancelled" }]); } else expect( context.providerRuntime.posts.map((post) => post.text), ).toEqual([context.result.summary]); await expect( db .select({ id: chatActions.id }) .from(chatActions) .where( and( eq(chatActions.endpointId, context.endpoint.id), eq(chatActions.kind, "github_omission_navigation"), ), ), ).resolves.toEqual([]); } finally { await context.cleanup(); } }, ); it.each(["complete_batch", "dropped_sibling", "old_omission"] as const)( "binds GitHub omission navigation to the complete current batch: %s", async (mode) => { let currentCommentId = ""; const context = await githubOmissionFixture(true, async (source) => { const callbacks = source.runtime.configurations.get( source.endpoint.id, )!.callbacks; await deliverMessage({ callbacks, endpointId: source.endpoint.id, provider: "github", thread: source.thread.thread, message: makeMessage({ id: "990099", text: "A plain current follow-up", userId: "42", }), trigger: "subscribed_message", }); const [second] = await db .select() .from(chatActions) .where( and( eq(chatActions.conversationId, source.conversation.id), eq(chatActions.kind, "inbound_wakeup"), sql`${chatActions.id} <> ${source.action.id}::uuid`, ), ); expect(second).toBeDefined(); expect(second.payload.attachmentOmissionReasons ?? {}).toEqual({}); currentCommentId = String(second.payload.commentId); if (mode === "old_omission") { const historicalRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: historicalRunId, companyId: source.fixture.companyId, agentId: source.fixture.assignedAgentId, status: "failed", wakeupRequestId: source.receipt.id, finishedAt: new Date(), contextSnapshot: { issueId: source.issue.id, source: "chat:github", wakeCommentIds: [source.action.payload.commentId], }, }); await db .update(agentWakeupRequests) .set({ runId: historicalRunId }) .where(eq(agentWakeupRequests.id, source.receipt.id)); await db .update(agentWakeupRequests) .set({ status: "failed", runId: source.runId }) .where(eq(agentWakeupRequests.id, second.id)); await db .update(heartbeatRuns) .set({ wakeupRequestId: second.id }) .where(eq(heartbeatRuns.id, source.runId)); } else { const [receipt] = await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, second.id)); await db .update(agentWakeupRequests) .set({ status: "coalesced", runId: null, payload: { ...receipt.payload, coalescedIntoWakeupRequestId: source.receipt.id, }, }) .where(eq(agentWakeupRequests.id, second.id)); } await db .update(heartbeatRuns) .set({ contextSnapshot: { issueId: source.issue.id, taskKey: source.issue.identifier, source: "chat:github", wakeCommentId: currentCommentId, wakeCommentIds: mode === "old_omission" ? [currentCommentId] : [source.action.payload.commentId, currentCommentId], }, }) .where(eq(heartbeatRuns.id, source.runId)); }); try { expect(context.action.payload.attachmentOmissionReasons).toEqual({ download_unavailable: 1, }); await expect(context.repair()).resolves.toBe(true); if (mode === "dropped_sibling") { await db .update(heartbeatRuns) .set({ contextSnapshot: { issueId: context.issue.id, taskKey: context.issue.identifier, source: "chat:github", wakeCommentId: currentCommentId, wakeCommentIds: [currentCommentId], }, }) .where(eq(heartbeatRuns.id, context.runId)); } await context.service.processPendingPublications(100); expect(context.providerRuntime.posts.map((post) => post.text)).toEqual( mode === "dropped_sibling" ? [] : [ mode === "complete_batch" ? `${context.result.summary}\n\n[Open this Paperclip task](https://paperclip.example/issues/${context.issue.id})` : context.result.summary, ], ); const preparations = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, context.endpoint.id), eq(chatActions.kind, "github_omission_navigation"), ), ); expect(preparations).toHaveLength(mode === "complete_batch" ? 1 : 0); } finally { await context.cleanup(); } }, ); it.each([ "text", "progress", "card", "interaction", "explicit", "no_comment", "source_revoked", "control", ] as const)( "refuses changed prepared GitHub omission navigation on retry: %s", async (mode) => { const context = await githubOmissionFixture(); try { await expect(context.repair()).resolves.toBe(true); context.providerRuntime.postError = Object.assign( new Error("Rate limited"), { name: "RateLimitError", retryAfter: 60 }, ); await context.service.processPendingPublications(100); const [publication] = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, context.endpoint.id), isNotNull(chatPublications.commentId), ), ); expect(publication).toMatchObject({ state: "retry", attempts: 1 }); expect(publication.payload.text).toContain("Open this Paperclip task"); const payload = { ...publication.payload }; if (mode === "text") payload.text += "\nChanged after preparation"; if (mode === "progress") payload.progressState = "completed"; if (mode === "card") payload.card = { title: "Changed presentation", children: [], } as never; if (mode === "interaction") payload.interactionId = randomUUID(); if (mode === "source_revoked") await db .update(chatEndpoints) .set({ allowUnlinkedPeople: false }) .where(eq(chatEndpoints.id, context.endpoint.id)); if (mode === "control") await db.insert(chatActions).values({ companyId: context.fixture.companyId, endpointId: context.endpoint.id, conversationId: context.conversation.id, principalId: context.action.principalId, kind: "task_control_authorization", status: "issued", providerActionId: `task-control-authorization:${publication.id}`, payload: {}, }); await db .update(chatPublications) .set({ payload, nextAttemptAt: new Date(0), ...(mode === "explicit" ? { idempotencyKey: `explicit-board:${publication.id}` } : {}), ...(mode === "control" ? { idempotencyKey: `control:probe:${publication.id}` } : {}), ...(mode === "no_comment" ? { commentId: null } : {}), }) .where(eq(chatPublications.id, publication.id)); context.providerRuntime.postError = undefined; await context.service.processPendingPublications(100); expect(context.providerRuntime.posts).toEqual([]); expect(context.providerRuntime.edits).toEqual([]); const [after] = await db .select() .from(chatPublications) .where(eq(chatPublications.id, publication.id)); expect(after.state).toBe("cancelled"); const preparations = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, context.endpoint.id), eq(chatActions.kind, "github_omission_navigation"), ), ); expect(preparations).toHaveLength(1); expect(preparations[0].status).toBe("processed"); expect(preparations[0].payload.preparedTextSha256).toBe( createHash("sha256").update(publication.payload.text).digest("hex"), ); if (mode === "control") { const [authorization] = await db .select() .from(chatActions) .where( eq( chatActions.providerActionId, `task-control-authorization:${publication.id}`, ), ); // Refusal leaves the claim issued, allowing the existing no-send // settlement to cancel it instead of stranding it in processing. expect(authorization.status).toBe("cancelled"); expect(authorization.result).toEqual({ code: "task_control_authorization_changed", }); } } finally { await context.cleanup(); } }, ); async function linkedCommittedResponseFailure( context: Awaited>, runId = context.runId, messageId = "same-run-message", ) { const [failure] = await db .insert(chatPublications) .values({ companyId: context.fixture.companyId, endpointId: context.endpoint.id, conversationId: context.conversation.id, issueId: context.issue.id, idempotencyKey: `run:${runId}:failed:${context.endpoint.id}`, payload: { text: "Maya stopped before completing this turn.", progressState: "failed", }, state: "published", providerMessageId: messageId, publishedAt: new Date(), attempts: 1, }) .returning(); await db.insert(chatMessageLinks).values({ companyId: context.fixture.companyId, endpointId: context.endpoint.id, conversationId: context.conversation.id, publicationId: failure.id, providerMessageId: messageId, direction: "outbound", }); return failure; } it.each(["slack", "telegram"] as const)( "updates only the linked same-run %s failure notice when recovering an accepted response", async (provider) => { const context = await committedChatResponseRecoveryFixture(provider); try { const unrelatedRunId = randomUUID(); for (const [runId, messageId] of [ [context.runId, "same-run-message"], [unrelatedRunId, "later-turn-message"], ]) { await linkedCommittedResponseFailure(context, runId, messageId); } await expect(context.repair()).resolves.toBe(true); await context.service.processPendingPublications(100); expect(context.providerRuntime.posts).toEqual([]); expect(context.providerRuntime.edits).toEqual([ { threadId: context.thread.thread.id, messageId: "same-run-message", text: context.result.summary, }, ]); await context.service.processPendingPublications(100); expect(context.providerRuntime.edits).toHaveLength(1); expect(context.providerRuntime.posts).toEqual([]); const [answer] = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, context.endpoint.id), isNotNull(chatPublications.commentId), ), ); expect(answer).toMatchObject({ state: "published", providerMessageId: "same-run-message", attempts: 1, }); await expect( db .select({ publicationId: chatMessageLinks.publicationId }) .from(chatMessageLinks) .where( and( eq(chatMessageLinks.endpointId, context.endpoint.id), eq(chatMessageLinks.providerMessageId, "same-run-message"), ), ), ).resolves.toEqual([{ publicationId: answer.id }]); await expect( db .select({ id: heartbeatRuns.id }) .from(heartbeatRuns) .where(eq(heartbeatRuns.nativeIssueId, context.issue.id)), ).resolves.toEqual([{ id: context.runId }]); } finally { await context.service.shutdown(); } }, ); it.each([ "source_edited", "principal_revoked", "generation_changed", "marker_changed", "delivery_unknown", ] as const)( "rechecks a recovered response at provider dispatch after %s", async (mutation) => { const context = await committedChatResponseRecoveryFixture("slack"); try { const failure = await linkedCommittedResponseFailure(context); await expect(context.repair()).resolves.toBe(true); if (mutation === "source_edited") await db .update(issueComments) .set({ updatedAt: new Date(Date.now() + 1), body: "Edited original request", }) .where( eq(issueComments.id, String(context.action.payload.commentId)), ); if (mutation === "principal_revoked") await db .update(chatEndpoints) .set({ allowUnlinkedPeople: false }) .where(eq(chatEndpoints.id, context.endpoint.id)); if (mutation === "generation_changed") await db .update(chatEndpoints) .set({ setup: sql`jsonb_set(${chatEndpoints.setup}, '{runtimeGeneration}', to_jsonb(coalesce((${chatEndpoints.setup}->>'runtimeGeneration')::int, 0) + 1))`, }) .where(eq(chatEndpoints.id, context.endpoint.id)); if (mutation === "marker_changed") await db .update(heartbeatRuns) .set({ resultJson: sql`jsonb_set(${heartbeatRuns.resultJson}, '{nativeCommittedChatResponse,resultId}', to_jsonb(${randomUUID()}::text))`, }) .where(eq(heartbeatRuns.id, context.runId)); if (mutation === "delivery_unknown") await db .update(chatPublications) .set({ state: "delivery_unknown" }) .where(eq(chatPublications.id, failure.id)); await context.service.processPendingPublications(100); expect(context.providerRuntime.posts).toEqual([]); expect(context.providerRuntime.edits).toEqual([]); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where( and( eq(chatPublications.endpointId, context.endpoint.id), isNotNull(chatPublications.commentId), ), ), ).resolves.toEqual([ { state: mutation === "delivery_unknown" ? "pending" : "cancelled" }, ]); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.id, failure.id)), ).resolves.toEqual([ { ...failure, state: mutation === "delivery_unknown" ? "delivery_unknown" : "published", }, ]); } finally { await context.service.shutdown(); } }, ); it.each([ "other_run_failure", "authored_answer", "payload_claim_only", ] as const)( "does not lend a consumed failure lane to a recovered answer with %s", async (mutation) => { const context = await committedChatResponseRecoveryFixture("telegram"); try { const failure = await linkedCommittedResponseFailure(context); await expect(context.repair()).resolves.toBe(true); const [answer] = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, context.endpoint.id), isNotNull(chatPublications.commentId), ), ); let currentLinkPublicationId = failure.id; if (mutation === "payload_claim_only") { const [run] = await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, context.runId)); const callerPayload = { ...answer.payload, nativeCommittedChatResponse: run.resultJson!.nativeCommittedChatResponse, }; await db .update(chatPublications) .set({ payload: callerPayload }) .where(eq(chatPublications.id, answer.id)); await db .update(heartbeatRuns) .set({ resultJson: sql`${heartbeatRuns.resultJson} - 'nativeCommittedChatResponse'`, }) .where(eq(heartbeatRuns.id, context.runId)); } else { // The old same-run failure retains its provider ID, but the current // outbound link is now owned by a different failure or authored text. const [otherComment] = mutation === "authored_answer" ? await db .insert(issueComments) .values({ companyId: context.fixture.companyId, issueId: context.issue.id, authorAgentId: context.fixture.assignedAgentId, body: "An already published answer", }) .returning() : []; const [current] = await db .insert(chatPublications) .values({ companyId: context.fixture.companyId, endpointId: context.endpoint.id, conversationId: context.conversation.id, issueId: context.issue.id, idempotencyKey: `run:${randomUUID()}:failed:${context.endpoint.id}`, ...(otherComment ? { commentId: otherComment.id } : {}), payload: mutation === "authored_answer" ? { text: "An already published answer" } : { text: "A later run failed", progressState: "failed" }, state: "published", providerMessageId: failure.providerMessageId, publishedAt: new Date(), attempts: 1, }) .returning(); currentLinkPublicationId = current.id; await db .update(chatMessageLinks) .set({ publicationId: current.id, commentId: current.commentId }) .where( and( eq(chatMessageLinks.endpointId, context.endpoint.id), eq( chatMessageLinks.providerMessageId, failure.providerMessageId!, ), ), ); } await context.service.processPendingPublications(100); expect(context.providerRuntime.edits).toEqual([]); // A denied replacement is not a denied, otherwise-authorized answer. expect(context.providerRuntime.posts).toEqual([ { threadId: context.thread.thread.id, text: context.result.summary }, ]); await expect( db .select({ publicationId: chatMessageLinks.publicationId }) .from(chatMessageLinks) .where( and( eq(chatMessageLinks.endpointId, context.endpoint.id), eq( chatMessageLinks.providerMessageId, failure.providerMessageId!, ), ), ), ).resolves.toEqual([{ publicationId: currentLinkPublicationId }]); await context.service.processPendingPublications(100); expect(context.providerRuntime.posts).toHaveLength(1); expect(context.providerRuntime.edits).toEqual([]); } finally { await context.service.shutdown(); } }, ); it.each(["streaming", "delivery_unknown"] as const)( "does not replace a failure while another exact-run authored publication is %s", async (state) => { const context = await committedChatResponseRecoveryFixture("slack"); try { const failure = await linkedCommittedResponseFailure(context); await expect(context.repair()).resolves.toBe(true); const [answer] = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, context.endpoint.id), isNotNull(chatPublications.commentId), ), ); // Later-created unresolved output deliberately does not block the // normal FIFO head. The replacement check must see it independently. const [unresolved] = await db .insert(chatPublications) .values({ companyId: context.fixture.companyId, endpointId: context.endpoint.id, conversationId: context.conversation.id, issueId: context.issue.id, commentId: answer.commentId, idempotencyKey: `uncertain-answer:${randomUUID()}`, payload: { text: "Another unresolved authored output" }, state, attempts: 1, createdAt: new Date(answer.createdAt.getTime() + 1), }) .returning(); await context.service.processPendingPublications(100); expect(context.providerRuntime.edits).toEqual([]); expect(context.providerRuntime.posts).toEqual( state === "delivery_unknown" ? [] : [ { threadId: context.thread.thread.id, text: context.result.summary, }, ], ); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.id, unresolved.id)), ).resolves.toEqual([unresolved]); await expect( db .select({ publicationId: chatMessageLinks.publicationId }) .from(chatMessageLinks) .where( and( eq(chatMessageLinks.endpointId, context.endpoint.id), eq( chatMessageLinks.providerMessageId, failure.providerMessageId!, ), ), ), ).resolves.toEqual([{ publicationId: failure.id }]); } finally { await context.service.shutdown(); } }, ); it("rechecks the recovered failure link after waiting for the provider mutation lane", async () => { const context = await committedChatResponseRecoveryFixture("telegram"); const blockerToken = `recovery-link-owner:${randomUUID()}`; let worker: Promise | undefined; let restore = () => {}; const releaseBlocker = () => db .delete(chatEndpointLeases) .where( and( eq(chatEndpointLeases.endpointId, context.endpoint.id), eq(chatEndpointLeases.leaseKey, "credentials"), eq(chatEndpointLeases.token, blockerToken), ), ); try { const failure = await linkedCommittedResponseFailure(context); await expect(context.repair()).resolves.toBe(true); await db.insert(chatEndpointLeases).values({ companyId: context.fixture.companyId, endpointId: context.endpoint.id, leaseKey: "credentials", token: blockerToken, expiresAt: new Date(Date.now() + 60_000), }); let attempted = false; const originalInsert = db.insert.bind(db); const insertSpy = vi.spyOn(db, "insert").mockImplementation((table) => { const builder = originalInsert(table); if (table === chatEndpointLeases) { const originalValues = builder.values.bind(builder); builder.values = ((values: { endpointId?: string; leaseKey?: string; token?: string; }) => { if ( values.endpointId === context.endpoint.id && values.leaseKey === "credentials" && values.token !== blockerToken ) attempted = true; return originalValues(values); }) as typeof builder.values; } return builder; }); restore = () => insertSpy.mockRestore(); worker = context.service.processPendingPublications(100); await expect.poll(() => attempted).toBe(true); expect(context.providerRuntime.posts).toEqual([]); expect(context.providerRuntime.edits).toEqual([]); // Simulate the current lease owner's durable publication commit before // releasing the endpoint. The earlier same-run row keeps its old ID. const [successor] = await db .insert(chatPublications) .values({ companyId: context.fixture.companyId, endpointId: context.endpoint.id, conversationId: context.conversation.id, issueId: context.issue.id, idempotencyKey: `run:${randomUUID()}:failed:${context.endpoint.id}`, payload: { text: "A successor's failure must not be rewritten", progressState: "failed", }, state: "published", providerMessageId: failure.providerMessageId, attempts: 1, publishedAt: new Date(), }) .returning(); await db .update(chatMessageLinks) .set({ publicationId: successor.id }) .where( and( eq(chatMessageLinks.endpointId, context.endpoint.id), eq(chatMessageLinks.providerMessageId, failure.providerMessageId!), ), ); await releaseBlocker(); await worker; expect(context.providerRuntime.edits).toEqual([]); expect(context.providerRuntime.posts).toEqual([ { threadId: context.thread.thread.id, text: context.result.summary }, ]); await expect( db .select({ publicationId: chatMessageLinks.publicationId }) .from(chatMessageLinks) .where( and( eq(chatMessageLinks.endpointId, context.endpoint.id), eq( chatMessageLinks.providerMessageId, failure.providerMessageId!, ), ), ), ).resolves.toEqual([{ publicationId: successor.id }]); } finally { await releaseBlocker(); try { await worker; } finally { restore(); await context.service.shutdown(); } } }); it.each([false, true])( "never replaces or resurrects an already selected answer (deleted=%s)", async (deleted) => { const context = await committedChatResponseRecoveryFixture("slack"); try { const comment = await issueService(db).addComment( context.issue.id, "The real provider final answer is already selected.", { agentId: context.fixture.assignedAgentId, runId: context.runId }, { authorizationReason: "internal_agent_write" }, ); await db .update(heartbeatRuns) .set({ resultJson: { presentationDecision: { commentId: comment.id } }, }) .where(eq(heartbeatRuns.id, context.runId)); if (deleted) await db .update(issueComments) .set({ deletedAt: new Date(), body: "Deleted" }) .where(eq(issueComments.id, comment.id)); await expect(context.repair()).resolves.toBe(false); await expect( db .select() .from(issueComments) .where(eq(issueComments.createdByRunId, context.runId)), ).resolves.toHaveLength(1); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.endpointId, context.endpoint.id)), ).resolves.toHaveLength(0); } finally { await context.service.shutdown(); } }, ); it("leaves a contended accepted response retryable without reclassifying or partially publishing it", async () => { const context = await committedChatResponseRecoveryFixture("telegram"); let release!: () => void; let observed!: () => void; const hold = new Promise((resolve) => { release = resolve; }); const locked = new Promise((resolve) => { observed = resolve; }); const holder = db.transaction(async (tx) => { await tx .select() .from(chatActions) .where(eq(chatActions.id, context.action.id)) .for("update"); observed(); await hold; }); try { await locked; await expect(context.repair()).resolves.toBe(false); await expect( db .select() .from(issueComments) .where(eq(issueComments.createdByRunId, context.runId)), ).resolves.toHaveLength(0); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.endpointId, context.endpoint.id)), ).resolves.toHaveLength(0); await expect( db .select({ phase: nativeRunFinalizations.phase }) .from(nativeRunFinalizations) .where(eq(nativeRunFinalizations.runId, context.runId)), ).resolves.toEqual([{ phase: "committed" }]); release(); await holder; await expect(context.repair()).resolves.toBe(true); } finally { release(); await holder; await context.service.shutdown(); } }); async function committedNativeReviewPublicationFixture(label: string) { const fixture = await seedCompany(); const storage = createStorageService(); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture, { storage: storage.storage }); const providerMessageId = `review-transport-${label}-${randomUUID()}`; const channel = makeThread({ channelId: `C-REVIEW-${label.toUpperCase()}`, id: `slack:C-REVIEW-${label.toUpperCase()}:${Date.now()}.1`, name: `review-${label}`, }); await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread, message: makeMessage({ id: providerMessageId, text: "@maya return the exact response and files, then wait", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(service, endpoint.id); await service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, endpoint.id)); if (!conversation) throw new Error("Expected review transport conversation"); const inbound = await db .select({ commentId: chatMessageLinks.commentId, principalId: chatDeliveries.principalId, }) .from(chatMessageLinks) .innerJoin( chatDeliveries, eq(chatDeliveries.id, chatMessageLinks.deliveryId), ) .where( and( eq(chatMessageLinks.endpointId, endpoint.id), eq(chatMessageLinks.conversationId, conversation.id), eq(chatMessageLinks.providerMessageId, providerMessageId), eq(chatMessageLinks.direction, "inbound"), ), ) .then((rows) => rows[0] ?? null); if (!inbound?.commentId || !inbound.principalId) { throw new Error("Expected exact inbound requester lineage"); } await db .insert(chatIdentityLinks) .values({ companyId: fixture.companyId, endpointId: endpoint.id, principalId: inbound.principalId, paperclipUserId: "owner-user", status: "linked", confirmedAt: new Date(), }) .onConflictDoUpdate({ target: [chatIdentityLinks.endpointId, chatIdentityLinks.principalId], set: { paperclipUserId: "owner-user", status: "linked", confirmedAt: new Date(), revokedAt: null, updatedAt: new Date(), }, }); const issueId = conversation.issueId; const runId = randomUUID(); const sessionId = randomUUID(); const runnerInstanceId = randomUUID(); const contractId = randomUUID(); const contractSha256 = `review-transport-${randomUUID()}`; const contextSnapshot = { issueId, source: "chat:slack", wakeCommentId: inbound.commentId, wakeCommentIds: [inbound.commentId], paperclipHarnessCheckedOut: true, paperclipWake: { reason: "External chat message received", externalChatProvider: "slack", checkedOutByHarness: true, issue: { id: issueId, workMode: "standard" }, commentIds: [inbound.commentId], }, }; await db.insert(completionContracts).values({ id: contractId, companyId: fixture.companyId, issueId, revision: 1, schemaVersion: "paperclip.completion-contract.v1", policyVersion: "phase6-v3", risk: "low", completionAuthority: "agent_claim_policy", incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: { revision: "review-transport-v1", objective: "Return the exact requested response and files", criteria: [{ id: "response", requirement: "Return the response" }], }, canonicalSha256: contractSha256, createdByActorType: "system", createdByActorId: "test", }); await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", runtimeMode: "native", nativeIssueId: issueId, nativeSessionId: sessionId, runnerInstanceId, completionContractId: contractId, completionContractSha256: contractSha256, contextSnapshot, }); await db .update(issues) .set({ status: "in_progress", assigneeAgentId: fixture.assignedAgentId, executionRunId: runId, }) .where(eq(issues.id, issueId)); const port = new PaperclipControlPlanePort(db, { companyId: fixture.companyId, issueId, runId, agentId: fixture.assignedAgentId, sessionId, completionContractId: contractId, completionContractSha256: contractSha256, sourceInstanceId: runnerInstanceId, controlPlaneSourceInstanceId: `review-transport-${runId}`, }); await port.openRun({ identity: { companyId: fixture.companyId, issueId, runId, agentId: fixture.assignedAgentId, sessionId, }, backendKind: "mock", sourceInstanceId: runnerInstanceId, }); const responseResult: PrpStructuredRunResult = { schema: "paperclip.run_result.v1", reportedWorkDisposition: "yielded", summary: "The exact response and files are prepared. I will wait.", completionClaim: { contractRevision: "review-transport-v1", objectiveSatisfied: true, criteria: [ { criterionId: "response", status: "satisfied", evidenceRefs: [], }, ], remainingWork: [], }, evidence: [], verification: [], attentionRequests: [], artifacts: [], continuation: { kind: "response_wake", summary: "Wait for the next authorized Slack message.", idempotencyKey: `review-transport-wait:${conversation.id}`, }, }; const terminal: PrpTerminalState = { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "yielded", workAssessmentId: randomUUID(), statusDecisionId: randomUUID(), }; await port.completeRun({ result: responseResult, terminal, callerResultId: `review-response-${runId}`, }); await db .update(heartbeatRuns) .set({ resultJson: { nativeResult: responseResult } }) .where(eq(heartbeatRuns.id, runId)); const [accepted] = await db .select() .from(nativeRunResults) .where(eq(nativeRunResults.runId, runId)); if (!accepted) throw new Error("Expected accepted native result"); const reviewRunId = randomUUID(); const reviewSessionId = randomUUID(); const reviewRunnerInstanceId = randomUUID(); await db.insert(heartbeatRuns).values({ id: reviewRunId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", runtimeMode: "native", nativeIssueId: issueId, nativeSessionId: reviewSessionId, runnerInstanceId: reviewRunnerInstanceId, completionContractId: contractId, completionContractSha256: contractSha256, contextSnapshot: {}, createdAt: new Date(Date.now() - 60_000), }); await db .update(issues) .set({ executionRunId: reviewRunId }) .where(eq(issues.id, issueId)); const reviewPort = new PaperclipControlPlanePort(db, { companyId: fixture.companyId, issueId, runId: reviewRunId, agentId: fixture.assignedAgentId, sessionId: reviewSessionId, completionContractId: contractId, completionContractSha256: contractSha256, sourceInstanceId: reviewRunnerInstanceId, controlPlaneSourceInstanceId: `review-gate-${reviewRunId}`, }); await reviewPort.openRun({ identity: { companyId: fixture.companyId, issueId, runId: reviewRunId, agentId: fixture.assignedAgentId, sessionId: reviewSessionId, }, backendKind: "mock", sourceInstanceId: reviewRunnerInstanceId, }); const reviewResult = { ...(accepted.resultJson.result as PrpStructuredRunResult), reportedWorkDisposition: "needs_review" as const, attentionRequests: [{ kind: "review" as const, ownerClass: "human" as const, summary: "Approve the prepared response and selected files." }], }; delete reviewResult.continuation; await reviewPort.completeRun({ result: reviewResult, terminal: { ...(accepted.resultJson.terminal as PrpTerminalState), reportedWorkDisposition: "needs_review", }, callerResultId: `review-gate-${reviewRunId}`, }); await finalizeNativeRun({ db, runId: reviewRunId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true, }); const [gate] = await db .select() .from(issueThreadInteractions) .where(eq(issueThreadInteractions.sourceRunId, reviewRunId)); if (!gate) throw new Error("Expected genuine native completion review"); expect(gate).toMatchObject({ kind: "request_confirmation", status: "pending", effectiveResolverPolicy: "human_only", createdByAgentId: null, createdByUserId: null, }); await db .update(issues) .set({ executionRunId: runId }) .where(eq(issues.id, issueId)); await db .update(heartbeatRuns) .set({ contextSnapshot, startedAt: new Date(gate.createdAt.getTime() + 1), }) .where(eq(heartbeatRuns.id, runId)); const selection = await issueService(db).addComment( issueId, "Prepared the exact requested files for this response.", { agentId: fixture.assignedAgentId, runId }, { authorizationReason: "paperclip_runner_protocol" }, ); const attachments = []; for (const [originalFilename, contentType, body] of [ ["review-note.txt", "text/plain", Buffer.from("review note", "utf8")], ["review-image.png", "image/png", Buffer.from("review image", "utf8")], ] as const) { const stored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${issueId}`, originalFilename, contentType, body, }); attachments.push( await issueService(db).createAttachment({ issueId, issueCommentId: selection.id, provider: stored.provider, objectKey: stored.objectKey, contentType: stored.contentType, byteSize: stored.byteSize, sha256: stored.sha256, originalFilename: stored.originalFilename, createdByAgentId: fixture.assignedAgentId, createdByRunId: runId, }), ); } await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true, }); const [committedRun] = await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, runId)); expect(committedRun).toMatchObject({ status: "succeeded", resultJson: { finalizationPhase: "committed", finalizationReasonCode: "governed_response_waiting", externalChatReviewPresentation: { runId, gateId: gate.id, }, }, }); const publications = await db .select() .from(chatPublications) .where( and( eq(chatPublications.issueId, issueId), eq(chatPublications.state, "pending"), ), ); expect(publications).toHaveLength(3); expect(publications).toEqual( expect.arrayContaining( attachments.map((attachment) => expect.objectContaining({ commentId: selection.id, idempotencyKey: `attachment:${attachment.id}:${endpoint.id}`, payload: expect.objectContaining({ attachmentIds: [attachment.id], }), }), ), ), ); const providerRuntime = runtime.endpoints.get(endpoint.id); if (!providerRuntime) throw new Error("Expected Slack provider runtime"); providerRuntime.posts.length = 0; providerRuntime.edits.length = 0; providerRuntime.slackFilePublicationAttempts = 0; providerRuntime.slackFileReceiptLookups.length = 0; return { attachments, conversation, endpoint, fixture, gate, providerRuntime, publications, runId, service, }; } it.each(["principal_revoked", "gate_changed"] as const)( "blocks a committed native review response and selected files at transport after %s", async (kind) => { const committed = await committedNativeReviewPublicationFixture(kind); if (kind === "principal_revoked") { await db .update(companyMemberships) .set({ status: "suspended", updatedAt: new Date() }) .where( and( eq(companyMemberships.companyId, committed.fixture.companyId), eq(companyMemberships.principalId, "owner-user"), ), ); } else { await db .update(issueThreadInteractions) .set({ effectiveResolverPolicy: "anyone", updatedAt: new Date() }) .where(eq(issueThreadInteractions.id, committed.gate.id)); } await expect( committed.service.processPendingPublications(100), ).resolves.toBe(3); expect(committed.providerRuntime.posts).toEqual([]); expect(committed.providerRuntime.edits).toEqual([]); expect(committed.providerRuntime.slackFilePublicationAttempts).toBe(0); expect(committed.providerRuntime.slackFileReceiptLookups).toEqual([]); const publicationIds = committed.publications.map(({ id }) => id); await expect( db .select({ attempts: chatPublications.attempts, providerMessageId: chatPublications.providerMessageId, redactedError: chatPublications.redactedError, state: chatPublications.state, }) .from(chatPublications) .where(inArray(chatPublications.id, publicationIds)), ).resolves.toEqual( expect.arrayContaining( publicationIds.map(() => ({ attempts: 1, providerMessageId: null, redactedError: "Task control requester or destination is no longer authorized", state: "cancelled", })), ), ); await expect( db .select({ id: chatMessageLinks.id }) .from(chatMessageLinks) .where(inArray(chatMessageLinks.publicationId, publicationIds)), ).resolves.toEqual([]); await expect( db .select({ status: issues.status }) .from(issues) .where(eq(issues.id, committed.conversation.issueId)), ).resolves.toEqual([{ status: "in_review" }]); await expect( db .select({ status: issueThreadInteractions.status }) .from(issueThreadInteractions) .where(eq(issueThreadInteractions.id, committed.gate.id)), ).resolves.toEqual([{ status: "pending" }]); }, ); it("retries a contended committed review response before transport, then publishes it and its files once", async () => { const committed = await committedNativeReviewPublicationFixture("busy"); let releaseGate!: () => void; let gateLocked!: () => void; const gateRelease = new Promise((resolve) => { releaseGate = resolve; }); const gateLockObserved = new Promise((resolve) => { gateLocked = resolve; }); const holder = db.transaction(async (tx) => { await tx .select({ id: issueThreadInteractions.id }) .from(issueThreadInteractions) .where(eq(issueThreadInteractions.id, committed.gate.id)) .for("update"); gateLocked(); await gateRelease; }); await gateLockObserved; try { await expect( committed.service.processPendingPublications(100), ).resolves.toBe(1); } finally { releaseGate(); await holder; } expect(committed.providerRuntime.posts).toEqual([]); expect(committed.providerRuntime.edits).toEqual([]); expect(committed.providerRuntime.slackFilePublicationAttempts).toBe(0); expect(committed.providerRuntime.slackFileReceiptLookups).toEqual([]); const [retrying] = await db .select() .from(chatPublications) .where( and( inArray( chatPublications.id, committed.publications.map(({ id }) => id), ), eq(chatPublications.state, "retry"), ), ); expect(retrying).toMatchObject({ attempts: 1, providerMessageId: null, nextAttemptAt: expect.any(Date), redactedError: "Chat response authorization is temporarily busy; no provider delivery was attempted", }); await db .update(chatPublications) .set({ nextAttemptAt: new Date(0), updatedAt: new Date() }) .where(eq(chatPublications.id, retrying!.id)); await expect( committed.service.processPendingPublications(100), ).resolves.toBe(3); expect(committed.providerRuntime.posts).toHaveLength(3); expect(committed.providerRuntime.edits).toEqual([]); expect(committed.providerRuntime.slackFilePublicationAttempts).toBe(2); expect(committed.providerRuntime.slackFileReceiptLookups).toEqual([]); const publicationIds = committed.publications.map(({ id }) => id); await expect( db .select({ attempts: chatPublications.attempts, providerMessageId: chatPublications.providerMessageId, state: chatPublications.state, }) .from(chatPublications) .where(inArray(chatPublications.id, publicationIds)), ).resolves.toEqual( expect.arrayContaining([ expect.objectContaining({ attempts: 2, providerMessageId: expect.any(String), state: "published", }), expect.objectContaining({ attempts: 1, providerMessageId: expect.any(String), state: "published", }), expect.objectContaining({ attempts: 1, providerMessageId: expect.any(String), state: "published", }), ]), ); await expect( db .select({ id: chatMessageLinks.id }) .from(chatMessageLinks) .where(inArray(chatMessageLinks.publicationId, publicationIds)), ).resolves.toHaveLength(3); await expect( committed.service.processPendingPublications(100), ).resolves.toBe(0); expect(committed.providerRuntime.posts).toHaveLength(3); expect(committed.providerRuntime.slackFilePublicationAttempts).toBe(2); }); async function deferredChatQueueFixture(admissionFails = false) { const fixture = await seedCompany(); let ownerId: string | null = null; const configured = await configuredSlackEndpoint(fixture, { wakeup: async (agentId, opts) => { if (admissionFails) throw new Error("PRIVATE admission failure token=do-not-publish"); const request = opts.durableChatRequest!; await db.transaction(async (tx) => { await request.authorize( tx as unknown as Parameters[0], ); const [owner] = ownerId ? await tx .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, ownerId)) : []; const existingContext = owner?.payload?._paperclipWakeContext as Record | undefined; const commentIds = [ ...(Array.isArray(existingContext?.wakeCommentIds) ? existingContext.wakeCommentIds : []), request.commentId, ]; const payload = { ...opts.payload, _paperclipWakeContext: { ...opts.contextSnapshot, wakeCommentIds: commentIds, }, }; if (owner) await tx .update(agentWakeupRequests) .set({ payload, coalescedCount: owner.coalescedCount + 1 }) .where(eq(agentWakeupRequests.id, owner.id)); await tx.insert(agentWakeupRequests).values({ id: request.id, companyId: fixture.companyId, agentId, source: "assignment", reason: "issue_execution_deferred", status: owner ? "coalesced" : "deferred_issue_execution", payload: owner ? { ...opts.payload, coalescedIntoWakeupRequestId: owner.id } : payload, requestedByActorType: request.requestedByActorType, requestedByActorId: request.requestedByActorId, requestedAt: request.requestedAt, idempotencyKey: request.idempotencyKey, }); ownerId ??= request.id; }); return { accepted: true }; }, }); const thread = makeThread({ channelId: "C-DEFERRED-NOTICE", id: "slack:C-DEFERRED-NOTICE:1999000.1", }); const send = async (id = "1999000.1") => deliverMessage({ callbacks: configured.callbacks, endpointId: configured.endpoint.id, thread: thread.thread, message: makeMessage({ id, text: "@maya exact queued request", mentioned: id === "1999000.1", }), trigger: id === "1999000.1" ? "mention" : "subscribed_message", }); await send(); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.endpointId, configured.endpoint.id)); const [action] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, configured.endpoint.id), eq(chatActions.kind, "inbound_wakeup"), ), ); const promote = async () => { const [owner] = await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, ownerId!)); const runId = randomUUID(); const contextSnapshot = owner.payload!._paperclipWakeContext as Record< string, unknown >; await db.transaction(async (tx) => { await tx.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", runtimeMode: "native", wakeupRequestId: owner.id, contextSnapshot, }); await tx .update(agentWakeupRequests) .set({ runId, status: "claimed" }) .where(eq(agentWakeupRequests.id, owner.id)); }); return runId; }; return { ...configured, action, conversation, fixture, promote, send, thread, }; } it("keeps one durable pre-run FIFO notice across coalescing and restart, then reuses only its successor lane", async () => { const context = await deferredChatQueueFixture(); let restarted: ChatChannelService | null = null; try { const { service, endpoint, conversation, fixture, action } = context; await expect( db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.companyId, fixture.companyId)), ).resolves.toEqual([]); await expect(service.enqueueInboundWakeupPublications()).resolves.toBe(1); await service.processPendingPublications(); const [queued] = await db .select() .from(chatPublications) .where(eq(chatPublications.endpointId, endpoint.id)); expect(queued).toMatchObject({ commentId: action.payload.commentId, state: "published", payload: { text: "Your follow-up is queued.", progressState: "queued" }, }); await expect( db .select({ commentId: chatMessageLinks.commentId }) .from(chatMessageLinks) .where(eq(chatMessageLinks.publicationId, queued.id)), ).resolves.toEqual([{ commentId: null }]); await expect( db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slack_session_sync"), ), ), ).resolves.toEqual([]); await context.send("1999000.2"); await expect(service.enqueueInboundWakeupPublications()).resolves.toBe(0); await service.processPendingPublications(); expect(context.runtime.endpoints.get(endpoint.id)?.posts).toHaveLength(1); await service.shutdown(); const fresh = createService(); restarted = fresh.service; await expect(restarted.enqueueInboundWakeupPublications()).resolves.toBe( 0, ); fresh.runtime.initializeHook = async () => { fresh.runtime.endpoints .get(endpoint.id)! .postResultIds.push( "predecessor-provider-message", "second-answer-provider-message", ); }; const runId = await context.promote(); // A predecessor's own lane cannot claim the deferred successor's notice. const predecessor = randomUUID(); await db.insert(heartbeatRuns).values({ id: predecessor, companyId: fixture.companyId, agentId: fixture.assignedAgentId, status: "running", contextSnapshot: { issueId: conversation.issueId }, }); await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${predecessor}:working:${endpoint.id}`, payload: { text: "Predecessor still working", progressState: "working", }, }); await restarted.processPendingPublications(); const runtime = fresh.runtime.endpoints.get(endpoint.id)!; expect(runtime.posts).toHaveLength(1); expect(runtime.edits).toEqual([]); await db.insert(chatPublications).values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, idempotencyKey: `run:${runId}:working:${endpoint.id}`, payload: { text: "Successor working", progressState: "working" }, }); await restarted.processPendingPublications(); expect(runtime.edits).toEqual([ { threadId: context.thread.thread.id, messageId: queued.providerMessageId, text: "Successor working", }, ]); await addSelectedChatFinal({ agentId: fixture.assignedAgentId, body: "The exact successor answer", companyId: fixture.companyId, issueId: conversation.issueId, runId, }); await restarted.processPendingPublications(); expect(runtime.edits.at(-1)).toMatchObject({ messageId: queued.providerMessageId, text: "The exact successor answer", }); await addSelectedChatFinal({ agentId: fixture.assignedAgentId, body: "A separate second answer", companyId: fixture.companyId, issueId: conversation.issueId, runId, }); await restarted.processPendingPublications(); expect(runtime.posts.at(-1)?.text).toBe("A separate second answer"); expect(runtime.edits).toHaveLength(2); await expect(restarted.processPendingPublications()).resolves.toBe(0); } finally { await restarted?.shutdown(); await context.service.shutdown(); } }); it.each(["answer_first", "failure_first"] as const)( "preserves selected answer and failure lanes after deferred admission (%s)", async (order) => { const context = await deferredChatQueueFixture(); try { await context.service.processPendingPublications(); const [queued] = await db .select() .from(chatPublications) .where( and( eq(chatPublications.endpointId, context.endpoint.id), like(chatPublications.idempotencyKey, "wake:%:queued:%"), ), ); expect(queued?.providerMessageId).toBeTruthy(); const runId = await context.promote(); const addFinal = () => addSelectedChatFinal({ agentId: context.fixture.assignedAgentId, body: "The deferred selected answer remains available", companyId: context.fixture.companyId, issueId: context.conversation.issueId, runId, }); let final = order === "answer_first" ? await addFinal() : null; if (order === "answer_first") await context.service.processPendingPublications(); await db .update(heartbeatRuns) .set({ status: "failed", errorCode: "adapter_failed", finishedAt: new Date(), updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, runId)); await enqueueChatRunMilestones(db); await context.service.processPendingPublications(); if (order === "failure_first") { final = await addFinal(); await context.service.processPendingPublications(); } const [answer] = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, final!.id)); const [failure] = await db .select() .from(chatPublications) .where( eq( chatPublications.idempotencyKey, `run:${runId}:failed:${context.endpoint.id}`, ), ); expect(answer?.state).toBe("published"); expect(failure?.state).toBe("published"); expect(answer?.providerMessageId).not.toBe(failure?.providerMessageId); expect( order === "answer_first" ? answer?.providerMessageId : failure?.providerMessageId, ).toBe(queued!.providerMessageId); const runtime = context.runtime.endpoints.get(context.endpoint.id)!; expect(runtime.posts).toHaveLength(2); expect(runtime.edits).toHaveLength(1); expect(runtime.edits[0]?.messageId).toBe(queued!.providerMessageId); const links = await db .select({ publicationId: chatMessageLinks.publicationId }) .from(chatMessageLinks) .where( and( eq(chatMessageLinks.conversationId, context.conversation.id), eq(chatMessageLinks.direction, "outbound"), ), ); expect(links).toHaveLength(2); expect(links).toEqual( expect.arrayContaining([ { publicationId: answer!.id }, { publicationId: failure!.id }, ]), ); await enqueueChatRunMilestones(db); await context.service.processPendingPublications(); expect(runtime.posts).toHaveLength(2); expect(runtime.edits).toHaveLength(1); } finally { await context.service.shutdown(); } }, ); it("reuses a pre-run FIFO notice when the successor asks before its working milestone", async () => { const context = await deferredChatQueueFixture(); try { await context.service.processPendingPublications(); const [queued] = await db .select() .from(chatPublications) .where(eq(chatPublications.endpointId, context.endpoint.id)); const runId = await context.promote(); const interactionId = randomUUID(); await db.insert(issueThreadInteractions).values({ id: interactionId, companyId: context.fixture.companyId, issueId: context.conversation.issueId, kind: "ask_user_questions", status: "pending", sourceRunId: runId, title: "Pick one", requestedByActorType: "agent", requestedByActorId: context.fixture.assignedAgentId, payload: { questions: [ { id: "color", prompt: "Pick one", options: [{ id: "amber", label: "Amber" }], }, ], }, }); await db.insert(chatPublications).values({ companyId: context.fixture.companyId, endpointId: context.endpoint.id, conversationId: context.conversation.id, issueId: context.conversation.issueId, idempotencyKey: `interaction:${interactionId}:${context.endpoint.id}`, payload: { text: "Pick one: Amber", interactionId }, }); await context.service.processPendingPublications(); const runtime = context.runtime.endpoints.get(context.endpoint.id)!; expect(runtime.posts).toHaveLength(1); expect(runtime.edits).toEqual([ { threadId: context.thread.thread.id, messageId: queued.providerMessageId, text: "Pick one: Amber", }, ]); } finally { await context.service.shutdown(); } }); it.each(["failed", "cancelled"] as const)( "retires a pre-run FIFO notice with the exact promoted %s run even before a working update", async (status) => { const context = await deferredChatQueueFixture(); try { await context.service.processPendingPublications(); const [queued] = await db .select() .from(chatPublications) .where(eq(chatPublications.endpointId, context.endpoint.id)); const runId = await context.promote(); await db.transaction(async (tx) => { await tx .update(agentWakeupRequests) .set({ status }) .where(eq(agentWakeupRequests.id, context.action.id)); await tx .update(heartbeatRuns) .set({ status, finishedAt: new Date(), updatedAt: new Date() }) .where(eq(heartbeatRuns.id, runId)); }); await db.insert(chatPublications).values({ companyId: context.fixture.companyId, endpointId: context.endpoint.id, conversationId: context.conversation.id, issueId: context.conversation.issueId, idempotencyKey: `run:${runId}:failed:${context.endpoint.id}`, payload: { text: "Maya stopped before completing this turn. Open the task in Paperclip for details.", progressState: "failed", }, }); await context.service.processPendingPublications(); await expect( context.service.processPendingPublications(), ).resolves.toBe(0); const runtime = context.runtime.endpoints.get(context.endpoint.id)!; expect(runtime.posts).toHaveLength(1); expect(runtime.edits).toEqual([ { threadId: context.thread.thread.id, messageId: queued.providerMessageId, text: "Maya stopped before completing this turn. Open the task in Paperclip for details.", }, ]); expect(JSON.stringify(runtime.edits)).not.toContain("was not started"); } finally { await context.service.shutdown(); } }, ); it.each([ "promotion", "cancelled", "reach", "source_deleted", "actor", "generation", "guest_access", ] as const)( "suppresses a pending pre-run FIFO notice after %s wins", async (change) => { const context = await deferredChatQueueFixture(); try { await expect( context.service.enqueueInboundWakeupPublications(), ).resolves.toBe(1); if (change === "promotion") await context.promote(); if (change === "cancelled") await db .update(agentWakeupRequests) .set({ status: "cancelled" }) .where(eq(agentWakeupRequests.id, context.action.id)); if (change === "reach") await db .update(chatEndpointResources) .set({ enabled: false }) .where(eq(chatEndpointResources.endpointId, context.endpoint.id)); if (change === "source_deleted") await db .update(issueComments) .set({ deletedAt: new Date() }) .where( eq(issueComments.id, String(context.action.payload.commentId)), ); if (change === "actor") await db .update(agentWakeupRequests) .set({ requestedByActorId: "forged-actor" }) .where(eq(agentWakeupRequests.id, context.action.id)); if (change === "generation") await db .update(chatConversations) .set({ sessionGeneration: context.conversation.sessionGeneration + 1, }) .where(eq(chatConversations.id, context.conversation.id)); if (change === "guest_access") await db .update(chatEndpoints) .set({ allowUnlinkedPeople: false }) .where(eq(chatEndpoints.id, context.endpoint.id)); await context.service.processPendingPublications(); expect( context.runtime.endpoints.get(context.endpoint.id)?.posts ?? [], ).toEqual([]); const publications = await db .select() .from(chatPublications) .where(eq(chatPublications.endpointId, context.endpoint.id)); expect(publications).toHaveLength(1); expect(publications[0]?.state).toBe("cancelled"); } finally { await context.service.shutdown(); } }, ); it("retires an already-visible pre-run FIFO notice once without publishing scheduler failures", async () => { const context = await deferredChatQueueFixture(); try { await context.service.processPendingPublications(); const [queued] = await db .select() .from(chatPublications) .where(eq(chatPublications.endpointId, context.endpoint.id)); await db .update(agentWakeupRequests) .set({ status: "failed", error: "PRIVATE token=secret scheduler detail", }) .where(eq(agentWakeupRequests.id, context.action.id)); await context.service.processPendingPublications(); await expect(context.service.processPendingPublications()).resolves.toBe( 0, ); const runtime = context.runtime.endpoints.get(context.endpoint.id)!; expect(runtime.posts).toHaveLength(1); expect(runtime.edits).toEqual([ { threadId: context.thread.thread.id, messageId: queued.providerMessageId, text: "This follow-up was not started. Open the task in Paperclip for details.", }, ]); expect(JSON.stringify(runtime.edits)).not.toMatch( /PRIVATE|token=|scheduler/, ); } finally { await context.service.shutdown(); } }); it("never promises a queued follow-up before durable admission succeeds", async () => { const context = await deferredChatQueueFixture(true); try { await context.service.processPendingPublications(); expect(context.action.status).toBe("issued"); expect( context.runtime.endpoints.get(context.endpoint.id)?.posts ?? [], ).toEqual([]); await expect( db .select() .from(chatPublications) .where(eq(chatPublications.endpointId, context.endpoint.id)), ).resolves.toEqual([]); await expect( db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.companyId, context.fixture.companyId)), ).resolves.toEqual([]); } finally { await context.service.shutdown(); } }); it("retires only the removed source's pre-run FIFO notice after its surviving batch promotes", async () => { const context = await deferredChatQueueFixture(); try { await context.service.processPendingPublications(); const [queued] = await db .select() .from(chatPublications) .where(eq(chatPublications.endpointId, context.endpoint.id)); await context.send("1999000.2"); const [second] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, context.endpoint.id), eq(chatActions.kind, "inbound_wakeup"), sql`${chatActions.id} <> ${context.action.id}`, ), ); const [owner] = await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, context.action.id)); const remainingContext = { ...(owner.payload!._paperclipWakeContext as Record), wakeCommentId: second.payload.commentId, commentId: second.payload.commentId, wakeCommentIds: [second.payload.commentId], }; await db.transaction(async (tx) => { await tx .update(issueComments) .set({ deletedAt: new Date() }) .where( eq(issueComments.id, String(context.action.payload.commentId)), ); await tx .update(agentWakeupRequests) .set({ payload: { ...owner.payload, wakeCommentId: second.payload.commentId, commentId: second.payload.commentId, _paperclipWakeContext: remainingContext, }, }) .where(eq(agentWakeupRequests.id, owner.id)); }); const runId = await context.promote(); await addSelectedChatFinal({ agentId: context.fixture.assignedAgentId, body: "Answer to the surviving second message", companyId: context.fixture.companyId, issueId: context.conversation.issueId, runId, }); await context.service.processPendingPublications(); await expect(context.service.processPendingPublications()).resolves.toBe( 0, ); const runtime = context.runtime.endpoints.get(context.endpoint.id)!; expect(runtime.posts.map((post) => post.text)).toEqual([ "Your follow-up is queued.", "Answer to the surviving second message", ]); expect(runtime.edits).toEqual([ { threadId: context.thread.thread.id, messageId: queued.providerMessageId, text: "This queued message was removed.", }, ]); expect(JSON.stringify(runtime.edits)).not.toContain("not started"); } finally { await context.service.shutdown(); } }); it("does not let removed-input cleanup overwrite a pre-run FIFO lane already replaced by its answer", async () => { const context = await deferredChatQueueFixture(); try { await context.service.processPendingPublications(); const runId = await context.promote(); await addSelectedChatFinal({ agentId: context.fixture.assignedAgentId, body: "Already answered", companyId: context.fixture.companyId, issueId: context.conversation.issueId, runId, }); await db .update(issueComments) .set({ deletedAt: new Date() }) .where(eq(issueComments.id, String(context.action.payload.commentId))); // Final is already ordered before the cleanup, but neither was sent // when cleanup was selected. Its transport claim must re-read the link. await expect( context.service.enqueueInboundWakeupPublications(), ).resolves.toBe(1); await context.service.processPendingPublications(); await expect(context.service.processPendingPublications()).resolves.toBe( 0, ); const runtime = context.runtime.endpoints.get(context.endpoint.id)!; expect(runtime.posts).toHaveLength(1); expect(runtime.edits).toHaveLength(1); expect(runtime.edits[0]?.text).toBe("Already answered"); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where( and( eq(chatPublications.endpointId, context.endpoint.id), like(chatPublications.idempotencyKey, "wake:%:removed:%"), ), ), ).resolves.toEqual([{ state: "cancelled" }]); } finally { await context.service.shutdown(); } }); it("retries a contended pre-run FIFO receipt before I/O and observes cancellation after the lock releases", async () => { const context = await deferredChatQueueFixture(); let release!: () => void; let locked!: () => void; const held = new Promise((resolve) => { release = resolve; }); const observed = new Promise((resolve) => { locked = resolve; }); let holder: Promise | null = null; try { await context.service.enqueueInboundWakeupPublications(); holder = db.transaction(async (tx) => { await tx .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, context.action.id)) .for("update"); locked(); await held; await tx .update(agentWakeupRequests) .set({ status: "cancelled" }) .where(eq(agentWakeupRequests.id, context.action.id)); }); await observed; await context.service.processPendingPublications(); const [queued] = await db .select() .from(chatPublications) .where(eq(chatPublications.endpointId, context.endpoint.id)); expect(queued).toMatchObject({ state: "retry", providerMessageId: null }); expect( context.runtime.endpoints.get(context.endpoint.id)?.posts ?? [], ).toEqual([]); release(); await holder; await db .update(chatPublications) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatPublications.id, queued.id)); await context.service.processPendingPublications(); await expect( db .select({ state: chatPublications.state }) .from(chatPublications) .where(eq(chatPublications.id, queued.id)), ).resolves.toEqual([{ state: "cancelled" }]); expect( context.runtime.endpoints.get(context.endpoint.id)?.posts ?? [], ).toEqual([]); } finally { release?.(); await holder; await context.service.shutdown(); } }); describe("provider timestamp provenance after closed conversations", () => { async function assertTimestampBinding(input: { context: Awaited>; fixture: Awaited>; delivery: typeof chatDeliveries.$inferSelect; expected: string | null; marker: string; }) { const { context, fixture, delivery, expected, marker } = input; const normalized = delivery.normalizedEvent.message as Record< string, unknown >; expect(normalized.providerSentAt).toBe(expected); expect(normalized.providerSentAtSource).toBe(expected ? marker : undefined); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, delivery.id)); await context.service.processPendingDeliveries(25, delivery.id); const [processed] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, delivery.id)); expect(processed).toMatchObject({ state: "processed", normalizedEvent: delivery.normalizedEvent, }); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.id, processed!.conversationId!)); const snapshot = await chatWakeContext({ endpointId: context.endpoint.id, issueId: conversation!.issueId, provider: context.endpoint.provider, providerMessageId: String(normalized.providerMessageId), }); const [action] = await db .select() .from(chatActions) .where( and( eq(chatActions.deliveryId, delivery.id), eq(chatActions.kind, "inbound_wakeup"), ), ); expect(action?.status).toBe("processed"); const runId = randomUUID(); // Real service admission; the fixture scheduler does not run a model. await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, runtimeMode: "native", nativeIssueId: conversation!.issueId, status: "running", wakeupRequestId: action!.id, startedAt: new Date(), contextSnapshot: { ...snapshot, paperclipHarnessCheckedOut: true }, }); await db .update(agentWakeupRequests) .set({ runId, status: "claimed" }) .where(eq(agentWakeupRequests.id, action!.id)); const authority = () => resolveChatRunPresentationAuthorizationReason(db, { companyId: fixture.companyId, issueId: conversation!.issueId, runId, }); // Missing timestamp provenance does not disable ordinary no-close work. await expect(authority()).resolves.toBe("allow_chat_run_presentation"); // Deliberately unproved historical-control fixture. This bare publication // is not an authorized /close and does not simulate a Teams personal // close remaining in the same generation. Valid source clocks cannot // manufacture its missing original command and authorization receipts. const [unprovedControl] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: context.endpoint.id, conversationId: conversation!.id, issueId: conversation!.issueId, idempotencyKey: `control:close:timestamp-fixture:${randomUUID()}`, payload: { kind: "text", text: "Historical conversation close" }, state: "published", providerMessageId: "historical-control", publishedAt: new Date( (expected ? Date.parse(expected) : delivery.receivedAt.getTime()) - 5_000, ), }) .returning({ id: chatPublications.id }); await expect( db .select({ id: chatActions.id }) .from(chatActions) .where( eq( chatActions.providerActionId, `task-control-authorization:${unprovedControl!.id}`, ), ), ).resolves.toEqual([]); await expect(authority()).resolves.toBe("internal_agent_write"); // A retained SDK display clock, missing marker, or foreign marker cannot // turn that unproved history into permission to publish either. const legacyMessage = { ...normalized, providerSentAt: expected ?? delivery.receivedAt.toISOString(), }; delete legacyMessage.providerSentAtSource; await db .update(chatDeliveries) .set({ normalizedEvent: { ...delivery.normalizedEvent, message: legacyMessage, }, }) .where(eq(chatDeliveries.id, delivery.id)); await expect(authority()).resolves.toBe("internal_agent_write"); await db .update(chatDeliveries) .set({ normalizedEvent: { ...delivery.normalizedEvent, message: { ...legacyMessage, providerSentAtSource: "other_provider_clock", }, }, }) .where(eq(chatDeliveries.id, delivery.id)); await expect(authority()).resolves.toBe("internal_agent_write"); await db .update(chatDeliveries) .set({ normalizedEvent: delivery.normalizedEvent }) .where(eq(chatDeliveries.id, delivery.id)); await expect(authority()).resolves.toBe("internal_agent_write"); } it.each([ "missing", "empty", "invalid", "numeric", "hour24", "calendar", "valid", "date_object", "display_override", ] as const)( "retains only the actual Teams activity clock, not SDK display fallback (%s)", async (mode) => { const fixture = await seedCompany(); const context = await configuredTeamsEndpoint(fixture, { deferWebhookProcessing: true, scheduleDeferredWork: () => undefined, }); const config = context.runtime.configurations.get( context.endpoint.id, )!.providerConfig; if (config.provider !== "microsoft-teams") throw new Error("Expected Teams"); const adapter = createTeamsAdapter({ ...config.credentials }); const timestamp = new Date(Date.now() - 1_000).toISOString(); const raw = { id: `timestamp-${randomUUID()}`, type: "message", channelId: "msteams", ...(mode === "missing" ? {} : { timestamp: mode === "empty" ? "" : mode === "invalid" ? "not-a-date" : mode === "numeric" ? Date.now() : mode === "hour24" ? "2026-09-08T24:00:00Z" : mode === "calendar" ? "2026-02-30T00:00:00Z" : mode === "date_object" ? new Date(timestamp) : timestamp, }), serviceUrl: "https://smba.trafficmanager.net/amer/", from: { id: `29:${randomUUID()}`, aadObjectId: randomUUID(), name: "Timestamp fixture", }, recipient: { id: `28:${config.credentials.appId}` }, conversation: { id: `a:${randomUUID()}`, conversationType: "personal", tenantId: config.credentials.appTenantId, }, channelData: { tenant: { id: config.credentials.appTenantId } }, text: "Keep this exact timestamp source", }; const message = adapter.parseMessage(raw); const providerRuntime = context.runtime.endpoints.get( context.endpoint.id, )!; const getThread = providerRuntime.thread.bind(providerRuntime); vi.spyOn(providerRuntime, "thread").mockImplementation((id) => ({ ...getThread(id), isDM: adapter.isDM(id), })); if (mode === "display_override") message.metadata.dateSent = new Date(); const display = message.metadata.dateSent.getTime(); try { await deliverMessage({ callbacks: context.callbacks, endpointId: context.endpoint.id, provider: "microsoft-teams", thread: makeThread({ id: message.threadId, channelId: message.threadId, isDM: true, }).thread, message, trigger: "direct_message", }); const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, context.endpoint.id)); expect(delivery?.state).toBe("received"); expect(message.metadata.dateSent.getTime()).toBe(display); await assertTimestampBinding({ context, fixture, delivery: delivery!, expected: ["valid", "date_object", "display_override"].includes( mode, ) ? timestamp : null, marker: "teams_activity_timestamp", }); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }, ); it.each(["ordinary", "task"] as const)( "retains only raw Telegram date through the pinned parser or runtime slash path (%s)", async (path) => { for (const mode of ["missing", "invalid", "valid"] as const) { const fixture = await seedCompany(); const context = await configuredTelegramEndpoint(fixture, { deferWebhookProcessing: true, scheduleDeferredWork: () => undefined, }); const configuration = context.runtime.configurations.get( context.endpoint.id, )!; if (configuration.providerConfig.provider !== "telegram") throw new Error("Expected Telegram"); const current = await context.service.get(context.endpoint.id); const providerFetch = fakeTelegramFetch( Number(current.botExternalId), ); const fetchSpy = vi .spyOn(globalThis, "fetch") .mockImplementation(async (input) => String(input).endsWith("/sendChatAction") ? Response.json({ ok: true, result: true }) : providerFetch(input), ); const pinned = createChatSdkEndpointRuntime({ ...configuration, logger: "silent", }); try { await pinned.initialize(); const providerRuntime = context.runtime.endpoints.get( context.endpoint.id, )!; Object.assign(providerRuntime, { parseTelegramCommandMessage: pinned.parseTelegramCommandMessage.bind(pinned), }); const date = Math.floor(Date.now() / 1_000) - 1; const text = path === "task" ? "/task Keep this exact timestamp source" : "Keep this exact timestamp source"; const raw = { message_id: 41, ...(mode === "missing" ? {} : { date: mode === "invalid" ? "not-a-date" : date }), chat: { id: 77117711, type: "private" }, from: { id: 77117711, is_bot: false, first_name: "Timestamp fixture", }, text, ...(path === "task" ? { entities: [{ type: "bot_command", offset: 0, length: 5 }] } : {}), }; if (path === "ordinary") { // Missing-date ordinary messages fail earlier SDK serialization; // isolate the real parser→service boundary here without claiming // that malformed ordinary input traverses the webhook dispatcher. const message = pinned.parseTelegramCommandMessage(raw)!; await deliverMessage({ callbacks: context.callbacks, endpointId: context.endpoint.id, provider: "telegram", message, thread: makeThread({ id: message.threadId, channelId: "77117711", isDM: true, }).thread, trigger: "direct_message", }); } else { const response = await pinned.handleWebhook( new Request("https://paperclip.example/timestamp-fixture", { method: "POST", headers: { "content-type": "application/json", "x-telegram-bot-api-secret-token": configuration.providerConfig.credentials.secretToken, }, body: JSON.stringify({ update_id: 900, message: raw }), }), ); expect(response.status).toBe(200); } const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, context.endpoint.id)); expect(delivery?.state).toBe("received"); await assertTimestampBinding({ context, fixture, delivery: delivery!, expected: mode === "valid" ? new Date(date * 1_000).toISOString() : null, marker: "telegram_message_date", }); } finally { await pinned.shutdown(); fetchSpy.mockRestore(); await retirePublicationFixture( context.service, context.endpoint.id, ); } } }, ); }); describe("Teams personal-recipient admission and durable restart", () => { async function recipientFixture() { const fixture = await seedCompany(); const context = await configuredTeamsEndpoint(fixture, { deferWebhookProcessing: true, scheduleDeferredWork: () => undefined, }); const configuration = context.runtime.configurations.get( context.endpoint.id, )!; if (configuration.providerConfig.provider !== "microsoft-teams") { throw new Error("Expected Teams provider configuration"); } const credentials = configuration.providerConfig.credentials; const tenantId = String(credentials.appTenantId); const botAppId = String(credentials.appId); const aadObjectId = randomUUID(); const providerUserId = `29:personal-${randomUUID()}`; const providerConversationId = `a:personal-${randomUUID()}`; const adapter = createTeamsAdapter({ appId: botAppId, appPassword: "synthetic-unused-parser-secret", appTenantId: tenantId, appType: "SingleTenant", }); const raw = { id: `recipient-${randomUUID()}`, type: "message", channelId: "msteams", timestamp: new Date().toISOString(), serviceUrl: "https://smba.trafficmanager.net/amer/", from: { id: providerUserId, aadObjectId, name: "Recipient Fixture" }, recipient: { id: `28:${botAppId}` }, conversation: { id: providerConversationId, conversationType: "personal", tenantId, }, channelData: { tenant: { id: tenantId } }, text: "Keep this exact personal request", token: "SYNTHETIC-RECIPIENT-TOKEN-NEVER-PERSIST", }; // Real installed normalization; the service runtime, provider transport // and scheduler below are fake. This is not a JWT/live Teams send proof. const message = adapter.parseMessage(raw); const sourceThread = makeThread({ id: message.threadId, channelId: message.threadId, isDM: true, name: "Personal recipient fixture", }); const restart = () => { const runtime = new FakeChatSdkRuntime(); const replace = runtime.replaceEndpoint.bind(runtime); vi.spyOn(runtime, "replaceEndpoint").mockImplementation( async (options) => { const endpointRuntime = await replace(options); const thread = endpointRuntime.thread.bind(endpointRuntime); vi.spyOn(endpointRuntime, "thread").mockImplementation((id) => ({ ...thread(id), isDM: adapter.isDM(id), })); return endpointRuntime; }, ); return createService(runtime, undefined, { scheduleDeferredWork: () => undefined, }); }; return { ...context, fixture, adapter, raw, message, sourceThread, tenantId, botAppId, aadObjectId, providerUserId, providerConversationId, restart, }; } it("preserves actual-parser personal recipient proof across database admission and reconstructed restart", async () => { const context = await recipientFixture(); let restarted: ReturnType | undefined; try { await deliverMessage({ callbacks: context.callbacks, endpointId: context.endpoint.id, provider: "microsoft-teams", thread: context.sourceThread.thread, message: context.message, trigger: "direct_message", }); const [received] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, context.endpoint.id)); expect(received).toMatchObject({ state: "received", attempts: 0, conversationId: null, }); const origin = received!.normalizedEvent.runtimeContext as { generation: number; credentialFingerprint: string; }; const admission: TeamsPersonalRecipientAdmission = { companyId: context.fixture.companyId, endpointId: context.endpoint.id, runtimeGeneration: origin.generation, credentialFingerprint: origin.credentialFingerprint, tenantId: context.tenantId, botAppId: context.botAppId, providerEventId: received!.providerEventId, threadId: context.message.threadId, isDirectMessage: true, }; const proof = received!.normalizedEvent.teamsPersonalRecipient; expect(proof).toEqual({ schema: "paperclip.teams.personal-recipient.v1", companyId: context.fixture.companyId, endpointId: context.endpoint.id, runtimeGeneration: origin.generation, credentialFingerprint: origin.credentialFingerprint, tenantId: context.tenantId, botAppId: context.botAppId, aadObjectId: context.aadObjectId, providerUserId: context.providerUserId, providerConversationId: context.providerConversationId, providerActivityId: context.raw.id, providerEventId: received!.providerEventId, }); expect(parseTeamsPersonalRecipient(proof, admission)).toEqual(proof); expect(JSON.stringify(proof)).not.toMatch( /SYNTHETIC-RECIPIENT|trafficmanager|Recipient Fixture|Keep this/, ); expect(JSON.stringify(received!.normalizedEvent)).not.toContain( "SYNTHETIC-RECIPIENT-TOKEN", ); expect(context.wakeup).not.toHaveBeenCalled(); await context.service.shutdown(); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, received!.id)); restarted = context.restart(); await restarted.service.processPendingDeliveries(1, received!.id); const [processed] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, received!.id)); expect(processed).toMatchObject({ state: "processed" }); expect(processed!.normalizedEvent).toEqual(received!.normalizedEvent); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.id, processed!.conversationId!)); const [principal] = await db .select() .from(chatExternalPrincipals) .where( and( eq(chatExternalPrincipals.companyId, context.fixture.companyId), eq(chatExternalPrincipals.externalId, context.aadObjectId), ), ); expect(principal).toBeDefined(); const scope = { admission, deliveryId: processed!.id, principalId: principal!.id, externalPrincipalId: principal!.externalId, conversationId: conversation!.id, conversationGeneration: conversation!.sessionGeneration, }; const bound = bindTeamsPersonalRecipient( processed!.normalizedEvent.teamsPersonalRecipient, scope, ); expect(bound).not.toBeNull(); expect( parseTeamsPersonalRecipientBinding( JSON.parse(JSON.stringify(bound)), scope, ), ).toEqual(bound); expect( parseTeamsPersonalRecipientBinding(bound, { ...scope, conversationGeneration: scope.conversationGeneration + 1, }), ).toBeNull(); expect(restarted.wakeup).toHaveBeenCalledTimes(1); await restarted.service.processPendingDeliveries(1, received!.id); expect(restarted.wakeup).toHaveBeenCalledTimes(1); expect( ( await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, received!.id)) )[0]!.normalizedEvent, ).toEqual(received!.normalizedEvent); } finally { try { await context.service.shutdown(); } finally { await retirePublicationFixture( restarted?.service ?? context.service, context.endpoint.id, ); } } }); it("does not invent personal recipient proof when an AAD-only legacy delivery resumes without original runtime evidence", async () => { const context = await recipientFixture(); let restarted: ReturnType | undefined; try { const providerEventId = `teams:${Buffer.from(context.providerConversationId).toString("base64url")}:${context.message.id}`; // A separate synthetic legacy receipt, not a stripped modern proof. // Its AAD principal and personal route cannot reconstruct BF from.id. const normalizedEvent = { providerEventId, kind: "direct_message", trigger: "direct_message", principal: { externalId: context.aadObjectId, displayName: "Legacy person", handle: "legacy", }, conversation: { externalConversationId: context.message.threadId, externalThreadId: context.message.threadId, label: "Legacy personal", isDirectMessage: true, providerUrl: null, }, message: { providerMessageId: context.message.id, text: "Legacy receipt", mentionedBot: false, attachments: [], }, }; const [received] = await db .insert(chatDeliveries) .values({ companyId: context.fixture.companyId, endpointId: context.endpoint.id, providerEventId, deduplicationKey: createHash("sha256") .update(providerEventId) .digest("hex"), eventKind: "direct_message", normalizedEvent, state: "received", nextAttemptAt: new Date(0), }) .returning(); await context.service.shutdown(); restarted = context.restart(); await restarted.service.processPendingDeliveries(1, received!.id); const [processed] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, received!.id)); expect(processed).toMatchObject({ state: "failed", redactedError: "This accepted chat message is no longer authorized to start work", }); expect(processed!.normalizedEvent).toEqual(normalizedEvent); expect( processed!.normalizedEvent.teamsPersonalRecipient, ).toBeUndefined(); expect(restarted.wakeup).not.toHaveBeenCalled(); expect(JSON.stringify(processed!.normalizedEvent)).not.toContain( context.providerUserId, ); } finally { try { await context.service.shutdown(); } finally { await retirePublicationFixture( restarted?.service ?? context.service, context.endpoint.id, ); } } }); it("redacts a rejected personal destination instead of persisting its derived recipient proof", async () => { const context = await recipientFixture(); try { await context.service.update( context.endpoint.id, { allowDirectMessages: false }, "owner-user", ); const callbacks = context.runtime.configurations.get( context.endpoint.id, )!.callbacks; await deliverMessage({ callbacks, endpointId: context.endpoint.id, provider: "microsoft-teams", thread: context.sourceThread.thread, message: context.message, trigger: "direct_message", }); const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, context.endpoint.id)); expect(delivery).toMatchObject({ state: "filtered", conversationId: null, normalizedEvent: { filtering: { contentRetained: false } }, }); expect( delivery!.normalizedEvent.teamsPersonalRecipient, ).toBeUndefined(); expect(JSON.stringify(delivery!.normalizedEvent)).not.toContain( context.providerUserId, ); expect(JSON.stringify(delivery!.normalizedEvent)).not.toContain( context.aadObjectId, ); expect(context.wakeup).not.toHaveBeenCalled(); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }); }); async function teamsFileAuthorityFixture( settings: { linkedRecipient?: boolean; separateSponsor?: boolean } = {}, ) { const { installTeamsFileConsentHook } = await import("../services/chat-teams-file-consent.js"); type ConsentCard = ReturnType< typeof import("../services/chat-teams-file-consent.js").buildTeamsFileConsentCard >; type FileCard = ReturnType< typeof import("../services/chat-teams-file-consent.js").buildTeamsUploadedFileCard >; const fixture = await seedCompany(); const storage = createStorageService(); const bytes = Buffer.from("Exact original Teams file bytes.\n", "utf8"); const consentCards: Array<{ threadId: string; card: ConsentCard }> = []; const fileCards: Array<{ threadId: string; card: FileCard }> = []; const controls: { beforeConsentReceipt?: () => Promise; beforeFileInfoReceipt?: () => Promise; } = {}; const handlers = new Map< string, (context: { activity: unknown }) => Promise<{ status: number }> >(); const runtime = new FakeChatSdkRuntime(); const replace = runtime.replaceEndpoint.bind(runtime); vi.spyOn(runtime, "replaceEndpoint").mockImplementation(async (options) => { const instance = await replace(options); const thread = instance.thread.bind(instance); vi.spyOn(instance, "thread").mockImplementation((id) => ({ ...thread(id), isDM: true, })); Object.assign(instance, { sendTeamsFileConsentCard: async ( threadId: string, card: ConsentCard, ) => { consentCards.push({ threadId, card }); await controls.beforeConsentReceipt?.(); return { id: `native-consent-${consentCards.length}` }; }, sendTeamsUploadedFileCard: async (threadId: string, card: FileCard) => { fileCards.push({ threadId, card }); await controls.beforeFileInfoReceipt?.(); return { id: `native-file-${fileCards.length}` }; }, }); if ( options.providerConfig.provider === "microsoft-teams" && options.callbacks.onTeamsFileConsent ) { handlers.clear(); installTeamsFileConsentHook( { on: (name, callback) => handlers.set(name, callback) }, { companyId: options.companyId, endpointId: options.endpointId, tenantId: String(options.providerConfig.credentials.appTenantId), botAppId: String(options.providerConfig.credentials.appId), onConsent: async (event) => options.callbacks.onTeamsFileConsent!({ endpointId: options.endpointId, provider: "microsoft-teams", event, }), }, ); } return instance; }); const uploadRequest = vi.fn< NonNullable >(async () => Response.json( { id: "teams-drive-item-1", name: "authority-report.txt", size: bytes.length, }, { status: 201 }, ), ); const wakeup = vi.fn( async () => ({ accepted: true }), ); const cancelRun = vi.fn(async () => ({ status: "cancelled" })); const service = chatChannelService(db, { fetch: async () => Response.json({ access_token: "synthetic-teams-access" }), runtime: runtime as unknown as ChatSdkRuntime, heartbeat: { wakeup: receiptBackedWakeup(wakeup), cancelRun }, storage: storage.storage, publicBaseUrl: "https://paperclip.example", teamsFileUploadRequest: uploadRequest, scheduleDeferredWork: () => undefined, }); const endpoint = await service.create( fixture.companyId, { provider: "microsoft-teams", assignedAgentId: fixture.assignedAgentId, name: "Teams native file authority", }, "owner-user", ); const tenantId = randomUUID(); const botAppId = randomUUID(); await service.configure( endpoint.id, { action: "configure", credentials: { clientId: botAppId, tenantId, clientSecret: "synthetic-file-secret", }, }, "owner-user", ); const aadObjectId = randomUUID(); const recipientUserId = `teams-linked-${randomUUID()}`; const sponsorUserId = settings.separateSponsor ? `teams-sponsor-${randomUUID()}` : "owner-user"; for (const userId of [ settings.linkedRecipient ? recipientUserId : null, settings.separateSponsor ? sponsorUserId : null, ]) { if (!userId) continue; await db.insert(authUsers).values({ id: userId, name: "Teams authority member", email: `${userId}@example.com`, emailVerified: true, createdAt: new Date(), updatedAt: new Date(), }); await db.insert(companyMemberships).values({ companyId: fixture.companyId, principalType: "user", principalId: userId, status: "active", membershipRole: "operator", }); } if (settings.separateSponsor) await db .update(chatEndpoints) .set({ sponsorUserId }) .where(eq(chatEndpoints.id, endpoint.id)); if (settings.linkedRecipient) { const [principal] = await db .insert(chatExternalPrincipals) .values({ companyId: fixture.companyId, provider: "microsoft-teams", providerAccountId: tenantId, externalId: aadObjectId, kind: "user", isBot: false, }) .returning(); await db.insert(chatIdentityLinks).values({ companyId: fixture.companyId, endpointId: endpoint.id, principalId: principal!.id, paperclipUserId: recipientUserId, status: "linked", confirmedAt: new Date(), }); } const providerUserId = `29:file-author-${randomUUID()}`; const providerConversationId = `a:file-authority-${randomUUID()}`; const adapter = createTeamsAdapter({ appId: botAppId, appPassword: "unused-parser-only", appTenantId: tenantId, appType: "SingleTenant", }); const raw = { id: `file-source-${randomUUID()}`, type: "message", channelId: "msteams", timestamp: new Date().toISOString(), serviceUrl: "https://smba.trafficmanager.net/amer/", from: { id: providerUserId, aadObjectId, name: "File recipient" }, recipient: { id: `28:${botAppId}` }, conversation: { id: providerConversationId, conversationType: "personal", tenantId, }, channelData: { tenant: { id: tenantId } }, text: "Use only my exact original personal conversation.", }; const message = adapter.parseMessage(raw); const sourceThread = makeThread({ id: message.threadId, channelId: message.threadId, isDM: true, name: "Native personal file", }); const deliver = async (activity = raw) => { const callbacks = runtime.configurations.get(endpoint.id)!.callbacks; await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "microsoft-teams", thread: sourceThread.thread, message: adapter.parseMessage(activity), trigger: "direct_message", }); }; await deliver(); await qualifySetupRoundTrip(service, endpoint.id, aadObjectId); await service.test(endpoint.id, "owner-user"); await service.processPendingPublications(); const [sourceDelivery] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), sql`${chatDeliveries.normalizedEvent}->'message'->>'providerMessageId' = ${raw.id}`, ), ); const [conversation] = await db .select() .from(chatConversations) .where(eq(chatConversations.id, sourceDelivery!.conversationId!)); const [issue] = await db .select() .from(issues) .where(eq(issues.id, conversation!.issueId)); const [sourceAction] = await db .select() .from(chatActions) .where( and( eq(chatActions.deliveryId, sourceDelivery!.id), eq(chatActions.kind, "inbound_wakeup"), ), ); expect(sourceDelivery).toMatchObject({ state: "processed", normalizedEvent: { teamsPersonalRecipient: { aadObjectId, providerUserId, providerConversationId, }, }, }); expect(sourceAction).toMatchObject({ status: "processed" }); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "active", }); const createFile = async () => { const stored = await storage.storage.putFile({ companyId: fixture.companyId, namespace: `issues/${issue!.id}`, originalFilename: "authority-report.txt", contentType: "text/plain", body: bytes, }); return issueService(db).createAttachment({ issueId: issue!.id, ...stored, createdByUserId: "owner-user", }); }; const dispatchConsent = async ( action: "accept" | "decline" = "accept", patch: Record = {}, retainedHandler?: (context: { activity: unknown; }) => Promise<{ status: number }>, ) => { const card = consentCards.at(-1)?.card; if (!card) throw new Error("Expected real service consent card before callback"); const activity = { id: `consent-response-${randomUUID()}`, type: "invoke", name: "fileConsent/invoke", channelId: "msteams", from: raw.from, recipient: raw.recipient, conversation: raw.conversation, channelData: raw.channelData, replyToId: `native-consent-${consentCards.length}`, value: { type: "fileUpload", action, context: action === "accept" ? card.content.acceptContext : card.content.declineContext, ...(action === "accept" ? { uploadInfo: { name: card.name, fileType: "txt", uniqueId: "teams-drive-item-1", uploadUrl: "https://fixture.sharepoint.com/upload?secret=PRIVATE-UPLOAD-CANARY", contentUrl: "https://fixture.sharepoint.com/personal/authority-report.txt", }, } : {}), }, ...patch, }; const handler = retainedHandler ?? handlers.get(`file.consent.${action}`); if (!handler) throw new Error("Expected registered service consent callback"); return handler({ activity }); }; return { fixture, storage, bytes, runtime, service, endpoint, tenantId, botAppId, aadObjectId, providerUserId, providerConversationId, adapter, raw, message, sourceThread, sourceDelivery: sourceDelivery!, sourceAction: sourceAction!, conversation: conversation!, issue: issue!, consentCards, fileCards, uploadRequest, wakeup, cancelRun, createFile, deliver, dispatchConsent, controls, handlers, recipientUserId, sponsorUserId, }; } describe("Telegram durable private draft Stop", () => { async function draftFixture( reusedBotId?: number, webhookOk: unknown = true, ) { const fixture = await seedCompany(); const providerFetch = fakeTelegramFetch(reusedBotId); const botToken = `123456:telegram-draft-${randomUUID().replaceAll("-", "")}`; let configuredWebhook: Record | null = null; let subscriptionInfo: Record | null = null; const maintenanceRequests: Array<{ method: string; body: Record; }> = []; let maintenanceHook: | (( method: string, body: Record, ) => Promise) | undefined; const administrativeFetch = (async (input, init) => { const method = new URL(String(input)).pathname.split("/").at(-1)!; const body = JSON.parse(String(init?.body ?? "{}")); if ( subscriptionInfo && new URL(String(input)).pathname.startsWith( `/bot${encodeURIComponent(botToken)}/`, ) && ["getMe", "getWebhookInfo", "setWebhook"].includes(method) ) { maintenanceRequests.push({ method, body }); const override = await maintenanceHook?.(method, body); if (override) return override; if (method === "getWebhookInfo") return Response.json({ ok: true, result: subscriptionInfo }); if (method === "setWebhook") { subscriptionInfo = { ...subscriptionInfo, url: body.url, allowed_updates: body.allowed_updates, max_connections: body.max_connections, }; return Response.json({ ok: true, result: true }); } const result = await (await providerFetch(input)).json(); return Response.json({ ...result, result: { ...result.result, is_bot: true }, }); } if (method === "setWebhook") { configuredWebhook = body; return Response.json({ ok: webhookOk, result: true }); } return providerFetch(input); }) as typeof fetch; const initial = createService( new FakeChatSdkRuntime(), administrativeFetch, ); const created = await initial.service.create( fixture.companyId, { provider: "telegram", assignedAgentId: fixture.assignedAgentId, name: "Private draft fixture", }, "owner-user", ); await initial.service.configure( created.id, { action: "configure", credentials: { botToken, }, }, "owner-user", ); let context = { ...initial, endpoint: created, callbacks: initial.runtime.configurations.get(created.id)!.callbacks, }; const { endpoint, callbacks } = context; const chatId = "77118878"; const threadId = `telegram:${chatId}:42`; const thread = makeThread({ channelId: chatId, id: threadId, isDM: true, name: "Telegram private draft", }); await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: thread.thread, message: makeMessage({ id: `${chatId}:71`, text: "Return the complete approved answer", userId: chatId, }), trigger: "direct_message", }); await qualifySetupRoundTrip(context.service, endpoint.id, chatId); await context.service.test(endpoint.id, "owner-user"); const [conversation] = await db .select() .from(chatConversations) .where( and( eq(chatConversations.endpointId, endpoint.id), eq(chatConversations.externalThreadId, threadId), ), ); if (!conversation) throw new Error("Expected exact private topic conversation"); const [currentEndpoint] = await db .select() .from(chatEndpoints) .where(eq(chatEndpoints.id, endpoint.id)); const botId = Number(currentEndpoint!.botExternalId); const requests: Array<{ method: string; body: Record }> = []; let hook: | ((method: string, body: Record) => Promise) | undefined; const originalFetch = globalThis.fetch; globalThis.fetch = vi.fn(async (input, init) => { const method = new URL(String(input)).pathname.split("/").at(-1)!; if (method === "getMe") return Response.json({ ok: true, result: { id: botId, is_bot: true, first_name: "Fixture", username: "fixture_bot", }, }); const body = JSON.parse(String(init?.body ?? "{}")); requests.push({ method, body }); await hook?.(method, body); if (method.endsWith("Draft")) return Response.json({ ok: true, result: true }); if (!["sendMessage", "sendRichMessage"].includes(method)) throw new Error("Unexpected draft fixture provider I/O"); return Response.json({ ok: true, result: { message_id: 900 + requests.length, date: 1, chat: { id: Number(chatId), type: "private" }, from: { id: botId, is_bot: true, first_name: "Fixture" }, text: "approved", }, }); }); let pinned: ReturnType; const install = async () => { const configuration = context.runtime.configurations.get(endpoint.id)!; pinned = createChatSdkEndpointRuntime({ ...configuration, logger: "silent", }); await pinned.initialize(); Object.assign(context.runtime.endpoints.get(endpoint.id)!, { thread: (threadId: string) => pinned.thread(threadId), handleWebhook: (...args: Parameters) => pinned.handleWebhook(...args), streamTelegramDraft: ( ...args: Parameters ) => pinned.streamTelegramDraft(...args), }); }; await install(); let updateId = 900; const deliver = async ( draftId: number, patch: Record = {}, validSecret = true, exactUpdateId?: number, retainedRuntime = false, ) => { const configuration = context.runtime.configurations.get(endpoint.id)!; if (configuration.providerConfig.provider !== "telegram") throw new Error("Expected Telegram"); const request = new Request("https://paperclip.example/fixture", { method: "POST", headers: { "content-type": "application/json", "x-telegram-bot-api-secret-token": validSecret ? configuration.providerConfig.credentials.secretToken : "wrong", }, body: JSON.stringify({ update_id: exactUpdateId ?? ++updateId, stopped_message_generation: { chat: { id: Number(chatId), type: "private" }, message_thread_id: 42, draft_id: draftId, ...patch, }, }), }); return retainedRuntime ? pinned.handleWebhook(request) : context.service.handleWebhook( endpoint.publicId, "telegram", request, ); }; return { fixture, endpoint, conversation, requests, maintenanceRequests, get configuredWebhook() { return configuredWebhook!; }, setSubscriptionInfo(patch: Record) { subscriptionInfo = { url: configuredWebhook!.url, has_custom_certificate: false, pending_update_count: 23, max_connections: 37, ip_address: "203.0.113.4", allowed_updates: ["message", "message_reaction", "future_update"], ...patch, }; }, setMaintenanceHook(value: typeof maintenanceHook) { maintenanceHook = value; }, async processSubscriptionAttempt() { const previousRequests = maintenanceRequests.length; const candidates = await db .select({ id: chatEndpoints.id }) .from(chatEndpoints) .where( and( eq(chatEndpoints.provider, "telegram"), eq(chatEndpoints.status, "active"), ), ); const queued = await db .select({ id: chatActions.id }) .from(chatActions) .where( and( eq(chatActions.kind, "telegram_maintenance"), inArray(chatActions.status, [ "received", "processing", "failed", ]), ), ); // The worker deliberately scans finite pages. Older fixtures may // precede this endpoint; wait for its first attempt, not a UUID's // chance placement in the first page. Stop before retrying its RPC. for ( let remaining = candidates.length + queued.length + 2; remaining > 0 && maintenanceRequests.length === previousRequests; remaining-- ) { await context.service.processPendingDeliveries(); } expect(maintenanceRequests.length).toBeGreaterThan(previousRequests); }, deliver, botId, get context() { return context; }, setHook(value: typeof hook) { hook = value; }, async send(key: string) { return context.service.publishBoardMessage( endpoint.id, conversation.id, `Approved output ${"complete safe answer. ".repeat(60)}END-${key}`, key, "owner-user", ); }, actions: () => db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "telegram_publication_draft"), ), ) .orderBy(asc(chatActions.createdAt)), async restart() { const configuration = context.runtime.configurations.get( endpoint.id, )!; if (configuration.providerConfig.provider !== "telegram") throw new Error("Expected Telegram"); await pinned.shutdown(); await context.service.shutdown(); const resumed = createService( new FakeChatSdkRuntime(), administrativeFetch, ); context = { ...context, ...resumed }; // Telegram is lazy, not a Gateway runtime. A verified inert update // initializes the new service's actual callback binding; the next // Stop below then traverses the pinned verifier/parser again. await context.service.handleWebhook( endpoint.publicId, "telegram", new Request("https://paperclip.example/fixture", { method: "POST", headers: { "content-type": "application/json", "x-telegram-bot-api-secret-token": configuration.providerConfig.credentials.secretToken, }, body: "{}", }), ); await install(); }, async close() { try { await pinned.shutdown(); } finally { try { await retirePublicationFixture(context.service, endpoint.id); } finally { globalThis.fetch = originalFetch; } } }, }; } it("durably stops during the first draft RPC without cancelling a run, and never reuses its ID after restart", async () => { const lane = await draftFixture(); try { const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: lane.fixture.companyId, agentId: lane.fixture.assignedAgentId, status: "running", contextSnapshot: {}, }); const runsBefore = await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.companyId, lane.fixture.companyId)); let stoppedId = 0; lane.setHook(async (method, body) => { if (method.endsWith("Draft") && !stoppedId) { stoppedId = Number(body.draft_id); expect((await lane.deliver(stoppedId)).status).toBe(200); } }); const publication = await lane.send("first-stopped"); expect(publication).toMatchObject({ state: "cancelled", providerMessageId: null, publishedAt: null, redactedError: "Telegram draft presentation was stopped. The saved answer and task are unchanged.", }); expect(lane.requests.map((request) => request.method)).toEqual([ "sendRichMessageDraft", ]); expect((await lane.actions())[0]).toMatchObject({ status: "cancelled", result: { phase: "stopped" }, }); const [comment] = await db .select() .from(issueComments) .where(eq(issueComments.id, publication!.commentId!)); expect(comment!.body).toContain("END-first-stopped"); expect( await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.companyId, lane.fixture.companyId)), ).toEqual(runsBefore); expect(lane.context.cancelRun).not.toHaveBeenCalled(); expect( await lane.context.service.getPublicationBatchStatus( lane.endpoint.id, lane.conversation.id, publication!.id, ), ).toMatchObject({ published: 0, total: 1, cancelled: 1, settled: 1, canDismiss: true, }); await expect( lane.context.service.replayPublication( lane.endpoint.id, publication!.id, ), ).rejects.toThrow(); expect((await lane.deliver(stoppedId)).status).toBe(200); await lane.restart(); await lane.context.service.processPendingPublications(); expect(lane.requests).toHaveLength(1); lane.setHook(async (method) => { if (method.endsWith("Draft")) expect((await lane.deliver(stoppedId)).status).toBe(200); }); const next = await lane.send("successor"); expect(next?.state).toBe("published"); const actions = await lane.actions(); expect(actions).toHaveLength(2); expect(actions[1]!.payload.draftId).not.toBe(stoppedId); expect(actions[1]).toMatchObject({ status: "processed", result: { phase: "published", providerMessageId: next!.providerMessageId, }, }); expect( lane.requests.filter((request) => !request.method.endsWith("Draft")), ).toHaveLength(1); } finally { await lane.close(); } }); it("does not reuse an old draft ID after endpoint cascade deletion and same-bot re-binding", async () => { const original = await draftFixture(); let originalClosed = false; let successor: Awaited> | undefined; try { let oldDraftId = 0; original.setHook(async (method, body) => { if (method.endsWith("Draft")) { oldDraftId = Number(body.draft_id); expect((await original.deliver(oldDraftId)).status).toBe(200); } }); expect((await original.send("before-delete"))?.state).toBe("cancelled"); await original.close(); originalClosed = true; await db .delete(chatEndpoints) .where(eq(chatEndpoints.id, original.endpoint.id)); expect(await original.actions()).toEqual([]); successor = await draftFixture(original.botId); successor.setHook(async (method) => { if (method.endsWith("Draft")) expect((await successor!.deliver(oldDraftId)).status).toBe(200); }); expect((await successor.send("after-rebind"))?.state).toBe("published"); const [newDraft] = await successor.actions(); expect(Number(newDraft!.payload.draftId)).toBeGreaterThan(oldDraftId); expect(newDraft).toMatchObject({ result: { phase: "published" } }); expect( successor.requests.filter( (request) => !request.method.endsWith("Draft"), ), ).toHaveLength(1); } finally { try { if (successor) await successor.close(); } finally { if (!originalClosed) await original.close(); } } }); it.each(["missing", "bot", "generation", "fingerprint", "url"] as const)( "sends ordinary complete output without Stop when subscription proof is %s", async (changed) => { const lane = await draftFixture(); try { const [receipt] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, lane.endpoint.id), eq(chatActions.kind, "telegram_stop_subscription"), ), ); expect(receipt?.status).toBe("processed"); if (changed === "missing") await db.delete(chatActions).where(eq(chatActions.id, receipt!.id)); else await db .update(chatActions) .set({ payload: { ...receipt!.payload, ...(changed === "bot" ? { botUserId: String(lane.botId + 1) } : changed === "generation" ? { runtimeGeneration: Number(receipt!.payload.runtimeGeneration) + 1, } : changed === "fingerprint" ? { credentialFingerprint: "stale" } : { webhookUrlSha256: "0".repeat(64) }), }, }) .where(eq(chatActions.id, receipt!.id)); const publication = await lane.send(`subscription-${changed}`); expect(publication?.state).toBe("published"); expect(lane.requests).toHaveLength(1); expect(lane.requests[0]!.method.endsWith("Draft")).toBe(false); expect(JSON.stringify(lane.requests[0]!.body)).toContain( `END-subscription-${changed}`, ); expect(lane.requests[0]!.body.can_stop).toBeUndefined(); expect(await lane.actions()).toEqual([]); expect(lane.context.cancelRun).not.toHaveBeenCalled(); } finally { await lane.close(); } }, ); it.each(["false", 1])( "does not confirm a subscription from non-boolean ok=%s", async (ok) => { const lane = await draftFixture(undefined, ok); try { expect( await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, lane.endpoint.id), eq(chatActions.kind, "telegram_stop_subscription"), ), ), ).toEqual([]); const publication = await lane.send("malformed-subscription"); expect(publication?.state).toBe("published"); expect(lane.requests).toHaveLength(1); expect(lane.requests[0]!.method.endsWith("Draft")).toBe(false); expect(JSON.stringify(lane.requests[0]!.body)).toContain( "END-malformed-subscription", ); expect(lane.requests[0]!.body.can_stop).toBeUndefined(); expect(await lane.actions()).toEqual([]); } finally { await lane.close(); } }, ); it.each(["explicit", "empty", "omitted"] as const)( "automatically repairs an existing %s subscription after restart without dropping updates", async (shape) => { const lane = await draftFixture(); const prefixIds: string[] = []; try { // Scheduler-only fixtures model a full already-confirmed first page. // They have no credentials; no provider qualification is claimed for // these rows. The target still uses actual configure/runtime/Stop. const [templateEndpoint] = await db .select() .from(chatEndpoints) .where(eq(chatEndpoints.id, lane.endpoint.id)); const [templateConnection] = await db .select() .from(toolConnections) .where(eq(toolConnections.id, lane.endpoint.connectionId)); const fingerprint = createHash("sha256").update("[]").digest("hex"); for (let index = 0; index < 25; index++) { const id = `00000000-${randomUUID().slice(9)}`; expect(id < lane.endpoint.id).toBe(true); const connectionId = randomUUID(); const publicId = randomUUID(); const botUserId = String( Number.parseInt( randomUUID().replaceAll("-", "").slice(0, 12), 16, ), ); const webhookUrlSha256 = createHash("sha256") .update( `https://paperclip.example/api/chat-webhooks/${publicId}/telegram`, ) .digest("hex"); await db.transaction(async (tx) => { await tx.insert(toolConnections).values({ ...templateConnection!, id: connectionId, uid: `prefix-${connectionId}`, credentialRefs: [], credentialSecretRefs: [], config: {}, transportConfig: {}, status: "active", enabled: true, }); await tx.insert(chatEndpoints).values({ ...templateEndpoint!, id, connectionId, publicId, botExternalId: botUserId, botUsername: `prefix_${publicId}`, status: "active", setup: { ...templateEndpoint!.setup, runtimeGeneration: 1 }, }); await tx.insert(chatActions).values({ companyId: lane.fixture.companyId, endpointId: id, kind: "telegram_stop_subscription", providerActionId: `telegram-stop-subscription:1:${fingerprint}:${webhookUrlSha256}`, payload: { version: 1, botUserId, runtimeGeneration: 1, credentialFingerprint: fingerprint, webhookUrlSha256, }, status: "processed", result: { code: "telegram_stop_subscription_confirmed" }, }); }); prefixIds.push(id); } await db .delete(chatActions) .where( and( eq(chatActions.endpointId, lane.endpoint.id), eq(chatActions.kind, "telegram_stop_subscription"), ), ); lane.setSubscriptionInfo( shape === "explicit" ? {} : { allowed_updates: shape === "empty" ? [] : undefined }, ); await lane.restart(); const prefixReceipts = await db .select() .from(chatActions) .where(inArray(chatActions.endpointId, prefixIds)) .orderBy(asc(chatActions.id)); const eligible = await db .select({ id: chatEndpoints.id }) .from(chatEndpoints) .innerJoin( toolConnections, and( eq(toolConnections.id, chatEndpoints.connectionId), eq(toolConnections.companyId, chatEndpoints.companyId), ), ) .where( and( eq(chatEndpoints.provider, "telegram"), eq(chatEndpoints.status, "active"), eq(toolConnections.status, "active"), eq(toolConnections.enabled, true), ), ) .orderBy(asc(chatEndpoints.id)); const targetIndex = eligible.findIndex( ({ id }) => id === lane.endpoint.id, ); expect(targetIndex).toBeGreaterThanOrEqual(25); const discoveryPasses = Math.floor(targetIndex / 25) + 1; const targetActions = () => db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, lane.endpoint.id), eq(chatActions.kind, "telegram_maintenance"), sql`${chatActions.payload}->>'operation' = 'stop_subscription'`, ), ); for (let page = 1; page <= discoveryPasses; page++) { await lane.context.service.processPendingDeliveries(); if (page < discoveryPasses) { expect(await targetActions()).toEqual([]); expect(lane.maintenanceRequests).toEqual([]); } } const [discovered] = await targetActions(); expect(discovered).toBeDefined(); // Discovery and action execution have independent 25-row budgets. // Include every older unsettled action, even future retries, so the // finite bound does not assume this global outbox is otherwise empty. const olderActions = await db .select({ id: chatActions.id }) .from(chatActions) .where( and( eq(chatActions.kind, "telegram_maintenance"), sql`${chatActions.createdAt} <= ${new Date(discovered!.createdAt.getTime() + 1).toISOString()}::timestamptz`, inArray(chatActions.status, [ "received", "processing", "failed", ]), ), ); for ( let remaining = olderActions.length + 1; remaining > 0 && !(await targetActions()).some( (action) => action.status === "processed", ); remaining-- ) await lane.context.service.processPendingDeliveries(); expect((await targetActions())[0]?.status).toBe("processed"); expect(lane.maintenanceRequests.map(({ method }) => method)).toEqual([ "getMe", "getWebhookInfo", "setWebhook", "getWebhookInfo", ]); expect( await db .select() .from(chatActions) .where(inArray(chatActions.endpointId, prefixIds)) .orderBy(asc(chatActions.id)), ).toEqual(prefixReceipts); const mutations = lane.maintenanceRequests.filter( ({ method }) => method === "setWebhook", ); expect(mutations).toHaveLength(1); expect(mutations[0]!.body).toEqual({ url: lane.configuredWebhook.url, secret_token: lane.configuredWebhook.secret_token, max_connections: 37, drop_pending_updates: false, allowed_updates: shape === "explicit" ? [ "message", "message_reaction", "future_update", "stopped_message_generation", ] : [], }); expect( await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, lane.endpoint.id), eq(chatActions.kind, "telegram_stop_subscription"), ), ), ).toHaveLength(1); lane.setHook(async (method, body) => { if (method.endsWith("Draft")) expect((await lane.deliver(Number(body.draft_id))).status).toBe( 200, ); }); expect((await lane.send(`upgraded-${shape}`))?.state).toBe( "cancelled", ); expect(lane.requests.map(({ method }) => method)).toEqual([ "sendRichMessageDraft", ]); await lane.context.service.processPendingDeliveries(); expect( lane.maintenanceRequests.filter( ({ method }) => method === "setWebhook", ), ).toHaveLength(1); } finally { try { if (prefixIds.length) await db .update(chatEndpoints) .set({ status: "archived", archivedAt: new Date() }) .where( and( eq(chatEndpoints.companyId, lane.fixture.companyId), inArray(chatEndpoints.id, prefixIds), ), ); } finally { await lane.close(); } } }, ); it.each([ "foreign_url", "certificate", "malformed_updates", "malformed_max", "wrong_bot", "nonboolean_ok", "missing_result", "contradictory_get", "changed_generation", "disabled_connection", ] as const)( "keeps ordinary output when automatic subscription repair is unsafe: %s", async (mode) => { const lane = await draftFixture(); try { await db .delete(chatActions) .where( and( eq(chatActions.endpointId, lane.endpoint.id), eq(chatActions.kind, "telegram_stop_subscription"), ), ); lane.setSubscriptionInfo( mode === "foreign_url" ? { url: "https://elsewhere.example/callback" } : mode === "certificate" ? { has_custom_certificate: true } : mode === "malformed_updates" ? { allowed_updates: null } : mode === "malformed_max" ? { max_connections: 101 } : {}, ); let getCount = 0; lane.setMaintenanceHook(async (method) => { if (method === "getMe" && mode === "wrong_bot") return Response.json({ ok: true, result: { id: lane.botId + 1, is_bot: true }, }); if (method === "setWebhook" && mode === "nonboolean_ok") return Response.json({ ok: "false", result: true }); if (method === "setWebhook" && mode === "missing_result") return Response.json({ ok: true }); if (method === "getWebhookInfo") { getCount++; if (mode === "contradictory_get" && getCount === 2) return Response.json({ ok: true, result: { url: lane.configuredWebhook.url, has_custom_certificate: false, max_connections: 37, allowed_updates: [ "message", "message_reaction", "future_update", ], }, }); if (mode === "changed_generation") { const [current] = await db .select() .from(chatEndpoints) .where(eq(chatEndpoints.id, lane.endpoint.id)); await db .update(chatEndpoints) .set({ setup: { ...current!.setup, runtimeGeneration: Number(current!.setup.runtimeGeneration) + 1, }, }) .where(eq(chatEndpoints.id, lane.endpoint.id)); } if (mode === "disabled_connection") await db .update(toolConnections) .set({ enabled: false }) .where(eq(toolConnections.id, lane.endpoint.connectionId)); } return undefined; }); await lane.processSubscriptionAttempt(); expect( await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, lane.endpoint.id), eq(chatActions.kind, "telegram_stop_subscription"), ), ), ).toEqual([]); expect( lane.maintenanceRequests.filter( ({ method }) => method === "setWebhook", ), ).toHaveLength( ["nonboolean_ok", "missing_result", "contradictory_get"].includes( mode, ) ? 1 : 0, ); if (mode !== "changed_generation" && mode !== "disabled_connection") { expect((await lane.send(`unconfirmed-${mode}`))?.state).toBe( "published", ); expect(lane.requests).toHaveLength(1); expect(lane.requests[0]!.method.endsWith("Draft")).toBe(false); expect(JSON.stringify(lane.requests[0]!.body)).toContain( `END-unconfirmed-${mode}`, ); } expect(await lane.actions()).toEqual([]); expect(lane.context.cancelRun).not.toHaveBeenCalled(); } finally { await lane.close(); } }, ); it("retries an unknown subscription mutation after restart using freshly observed options, not a recovered confirmation flag", async () => { const lane = await draftFixture(); try { await db .delete(chatActions) .where( and( eq(chatActions.endpointId, lane.endpoint.id), eq(chatActions.kind, "telegram_stop_subscription"), ), ); lane.setSubscriptionInfo({}); lane.setMaintenanceHook(async (method) => { if (method === "setWebhook") throw new Error("Synthetic unknown subscription response"); return undefined; }); await lane.processSubscriptionAttempt(); const [action] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, lane.endpoint.id), eq(chatActions.kind, "telegram_maintenance"), sql`${chatActions.payload}->>'operation' = 'stop_subscription'`, ), ); expect(action).toMatchObject({ status: "failed", result: { retryable: true, providerConfirmed: false }, }); expect((await lane.send("unknown-subscription"))?.state).toBe( "published", ); expect(lane.requests).toHaveLength(1); expect(lane.requests[0]!.method.endsWith("Draft")).toBe(false); await lane.context.service.processPendingDeliveries(); expect( lane.maintenanceRequests.filter( ({ method }) => method === "setWebhook", ), ).toHaveLength(1); await lane.restart(); lane.setSubscriptionInfo({ allowed_updates: ["message", "chat_member"], }); lane.setMaintenanceHook(undefined); await db .update(chatActions) .set({ result: { ...action!.result, providerConfirmed: true, retryAt: new Date(0).toISOString(), }, }) .where(eq(chatActions.id, action!.id)); await Promise.all([ lane.context.service.processPendingDeliveries(), lane.context.service.processPendingDeliveries(), ]); const mutations = lane.maintenanceRequests.filter( ({ method }) => method === "setWebhook", ); expect(mutations).toHaveLength(2); expect(mutations[1]!.body.allowed_updates).toEqual([ "message", "chat_member", "stopped_message_generation", ]); const [receipt] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, lane.endpoint.id), eq(chatActions.kind, "telegram_stop_subscription"), ), ); expect(receipt?.status).toBe("processed"); const serialized = JSON.stringify(receipt); expect(serialized).not.toContain(lane.configuredWebhook.url); expect(serialized).not.toContain(lane.configuredWebhook.secret_token); lane.requests.length = 0; lane.setHook(async (method, body) => { if (method.endsWith("Draft")) expect((await lane.deliver(Number(body.draft_id))).status).toBe( 200, ); }); expect((await lane.send("repaired-after-unknown"))?.state).toBe( "cancelled", ); } finally { await lane.close(); } }); it("returns 503 on failed Stop commit and accepts the exact verified retry before final send", async () => { const lane = await draftFixture(); const suffix = randomUUID().replaceAll("-", ""); const functionName = `stop_fail_${suffix}`; const triggerName = `stop_trigger_${suffix}`; let created = false; try { let checked = false; lane.setHook(async (method, body) => { if (!method.endsWith("Draft") || checked) return; checked = true; const [action] = await lane.actions(); await db.execute( sql.raw(`CREATE FUNCTION ${functionName}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.id = '${action!.id}'::uuid AND NEW.result->>'phase' = 'stopped' THEN RAISE EXCEPTION 'synthetic stop commit fault'; END IF; RETURN NEW; END $$`), ); await db.execute( sql.raw( `CREATE TRIGGER ${triggerName} BEFORE UPDATE ON chat_actions FOR EACH ROW EXECUTE FUNCTION ${functionName}()`, ), ); created = true; expect( (await lane.deliver(Number(body.draft_id), {}, true, 1777)).status, ).toBe(503); expect((await lane.actions())[0]).toMatchObject({ status: "processing", result: { phase: "drafting" }, }); await db.execute( sql.raw(`DROP TRIGGER ${triggerName} ON chat_actions`), ); await db.execute(sql.raw(`DROP FUNCTION ${functionName}()`)); created = false; expect( (await lane.deliver(Number(body.draft_id), {}, true, 1777)).status, ).toBe(200); }); expect((await lane.send("commit-retry"))?.state).toBe("cancelled"); expect(checked).toBe(true); expect(lane.requests.map((request) => request.method)).toEqual([ "sendRichMessageDraft", ]); expect((await lane.actions())[0]).toMatchObject({ result: { phase: "stopped", updateId: 1777 }, }); } finally { try { if (created) { await db.execute( sql.raw(`DROP TRIGGER ${triggerName} ON chat_actions`), ); await db.execute(sql.raw(`DROP FUNCTION ${functionName}()`)); } } finally { await lane.close(); } } }); it("preserves an uncertain final after restart and a late Stop without automatic replay", async () => { const lane = await draftFixture(); try { let draftId = 0; lane.setHook(async (method, body) => { if (method.endsWith("Draft")) draftId = Number(body.draft_id); else throw new TypeError("fetch failed"); }); const publication = await lane.send("unknown-final"); expect(publication).toMatchObject({ state: "delivery_unknown", providerMessageId: null, publishedAt: null, }); expect((await lane.actions())[0]).toMatchObject({ status: "processing", result: { phase: "final_sending" }, }); const before = lane.requests.length; lane.setHook(undefined); await lane.restart(); expect((await lane.deliver(draftId)).status).toBe(200); await lane.context.service.processPendingPublications(); expect(lane.requests).toHaveLength(before); const [current] = await db .select() .from(chatPublications) .where(eq(chatPublications.id, publication!.id)); expect(current!.state).toBe("delivery_unknown"); await expect( lane.context.service.replayPublication( lane.endpoint.id, publication!.id, ), ).rejects.toThrow(); expect(lane.context.cancelRun).not.toHaveBeenCalled(); } finally { await lane.close(); } }); it.each(["bot", "generation", "source", "connection"] as const)( "refuses a retained Stop and later final after current %s authority changes", async (changed) => { const lane = await draftFixture(); try { let checked = false; lane.setHook(async (method, body) => { if (!method.endsWith("Draft") || checked) return; checked = true; const [action] = await lane.actions(); const [endpoint] = await db .select() .from(chatEndpoints) .where(eq(chatEndpoints.id, lane.endpoint.id)); if (changed === "bot") await db .update(chatEndpoints) .set({ botExternalId: String(lane.botId + 1) }) .where(eq(chatEndpoints.id, lane.endpoint.id)); if (changed === "generation") await db .update(chatEndpoints) .set({ setup: { ...endpoint!.setup, runtimeGeneration: Number(endpoint!.setup.runtimeGeneration ?? 0) + 1, }, }) .where(eq(chatEndpoints.id, lane.endpoint.id)); if (changed === "connection") await db .update(toolConnections) .set({ enabled: false }) .where(eq(toolConnections.id, endpoint!.connectionId)); if (changed === "source") await db .update(chatPublications) .set({ payload: { text: "Changed current source" } }) .where( eq( chatPublications.id, String(action!.payload.publicationId), ), ); expect( ( await lane.deliver( Number(body.draft_id), {}, true, undefined, true, ) ).status, ).toBe(200); expect((await lane.actions())[0]).toMatchObject({ status: "processing", result: { phase: "drafting" }, }); }); const publication = await lane.send(`changed-${changed}`); expect(checked).toBe(true); expect(publication!.state).not.toBe("published"); expect(publication!.state).not.toBe("cancelled"); expect( lane.requests.every((request) => request.method.endsWith("Draft")), ).toBe(true); expect(lane.context.cancelRun).not.toHaveBeenCalled(); } finally { await lane.close(); } }, ); it.each([ "wrong_chat", "wrong_topic", "wrong_draft", "wrong_secret", "late_final", ] as const)( "does not withdraw an unowned or already-claimed final (%s)", async (mode) => { const lane = await draftFixture(); try { let draftId = 0; lane.setHook(async (method, body) => { if (method.endsWith("Draft")) draftId = Number(body.draft_id); if ((mode === "late_final") === !method.endsWith("Draft")) { const patch = mode === "wrong_chat" ? { chat: { id: 444, type: "private" } } : mode === "wrong_topic" ? { message_thread_id: 43 } : {}; const result = await lane.deliver( mode === "wrong_draft" ? (draftId % 2_147_483_647) + 1 : draftId, patch, mode !== "wrong_secret", ); expect(result.status).toBe(mode === "wrong_secret" ? 403 : 200); } }); const publication = await lane.send(mode); expect(publication?.state).toBe("published"); expect( lane.requests.filter( (request) => !request.method.endsWith("Draft"), ), ).toHaveLength(1); expect((await lane.actions())[0]).toMatchObject({ status: "processed", result: { phase: "published" }, }); expect(lane.context.cancelRun).not.toHaveBeenCalled(); } finally { await lane.close(); } }, ); }); describe("Teams exact self-unknown source continuation", () => { async function nativeFileFixture() { const context = await teamsFileAuthorityFixture(); const runId = randomUUID(); const contractId = randomUUID(); const sessionId = randomUUID(); const runnerId = randomUUID(); const contractSha256 = createHash("sha256") .update(contractId) .digest("hex"); await db.insert(completionContracts).values({ id: contractId, companyId: context.fixture.companyId, issueId: context.issue.id, revision: 1, schemaVersion: "paperclip.completion-contract.v1", policyVersion: "phase6-v3", risk: "low", completionAuthority: "agent_claim_policy", incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: { revision: "teams-native-v1", objective: "Return the requested file", criteria: [ { id: "response", requirement: "Return the exact selected file" }, ], }, canonicalSha256: contractSha256, createdByActorType: "system", createdByActorId: "test", }); await db.insert(heartbeatRuns).values({ id: runId, companyId: context.fixture.companyId, agentId: context.fixture.assignedAgentId, status: "running", wakeupRequestId: context.sourceAction.id, runtimeMode: "native", nativeIssueId: context.issue.id, nativeSessionId: sessionId, runnerInstanceId: runnerId, completionContractId: contractId, completionContractSha256: contractSha256, contextSnapshot: { issueId: context.issue.id, taskKey: context.issue.identifier, source: "chat:microsoft-teams", wakeCommentId: context.sourceAction.payload.commentId, wakeCommentIds: [context.sourceAction.payload.commentId], }, }); await db .update(agentWakeupRequests) .set({ status: "completed", runId }) .where(eq(agentWakeupRequests.id, context.sourceAction.id)); const stored = await context.storage.storage.putFile({ companyId: context.fixture.companyId, namespace: `issues/${context.issue.id}`, originalFilename: "authority-report.txt", contentType: "text/plain", body: context.bytes, }); const attachment = await issueService(db).createAttachment({ issueId: context.issue.id, ...stored, createdByAgentId: context.fixture.assignedAgentId, createdByRunId: runId, }); await issueService(db).addComment( context.issue.id, "Selected the exact requested file.", { agentId: context.fixture.assignedAgentId, runId }, { attachmentIds: [attachment.id], authorType: "agent", authorizationReason: "allow_self", }, ); await db .update(heartbeatRuns) .set({ status: "failed", errorCode: "adapter_failed", finishedAt: new Date(), }) .where(eq(heartbeatRuns.id, runId)); await db .update(issues) .set({ status: "in_review", executionRunId: null }) .where(eq(issues.id, context.issue.id)); const result: PrpStructuredRunResult = { schema: "paperclip.run_result.v1", reportedWorkDisposition: "yielded", summary: "Here is the exact requested file. I will wait for your next message.", completionClaim: { contractRevision: "teams-native-v1", objectiveSatisfied: true, criteria: [ { criterionId: "response", status: "satisfied", evidenceRefs: [] }, ], remainingWork: [], }, evidence: [], verification: [], attentionRequests: [], artifacts: [], continuation: { kind: "response_wake", summary: "Wait for the next authorized message.", idempotencyKey: `wait:${runId}`, }, }; const terminal: PrpTerminalState = { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "yielded", workAssessmentId: randomUUID(), statusDecisionId: randomUUID(), }; await new NativeRunCoordinatorStore(db, { companyId: context.fixture.companyId, issueId: context.issue.id, runId, agentId: context.fixture.assignedAgentId, normalizedSessionId: sessionId, runnerSourceInstanceId: runnerId, completionContractId: contractId, completionContractSha256: contractSha256, completionContractRevision: "teams-native-v1", completionContractCriterionIds: ["response"], }).completeRun({ result, terminal, turnId: `turn-${runId}` }); await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", }); await repairCommittedNativeChatResponse(db, { companyId: context.fixture.companyId, issueId: context.issue.id, runId, }); const [run] = await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, runId)); expect(run).toMatchObject({ status: "succeeded", resultJson: { nativeCommittedChatResponse: { schema: "paperclip.native_committed_chat_response.v1", }, }, }); return { ...context, runId, attachment }; } // Actual parser/admission and accepted native-result composition. Provider // sends and authenticated App routing are controlled, not live tenant proof. it("accepts the exact native file after a lost consent-card receipt without reposting", async () => { const context = await nativeFileFixture(); try { const instance = context.runtime.endpoints.get( context.endpoint.id, )! as unknown as { sendTeamsFileConsentCard: ( threadId: string, card: unknown, ) => Promise<{ id: string }>; }; const send = instance.sendTeamsFileConsentCard.bind(instance); vi.spyOn(instance, "sendTeamsFileConsentCard").mockImplementation( async (threadId, card) => { await send(threadId, card); throw new Error("controlled lost card POST receipt"); }, ); await context.service.processPendingPublications(); const [unknown] = await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.endpointId, context.endpoint.id)); expect(unknown).toMatchObject({ phase: "consent_unknown", consentMessageId: null, attemptId: null, }); expect(context.consentCards).toHaveLength(1); const wakeCount = context.wakeup.mock.calls.length; await expect(context.dispatchConsent()).resolves.toEqual({ status: 200, }); await context.service.processPendingPublications(); const [done] = await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.id, unknown!.id)); expect(done).toMatchObject({ phase: "delivered", consentMessageId: null, fileInfoMessageId: "native-file-1", }); expect(context.consentCards).toHaveLength(1); expect(context.uploadRequest).toHaveBeenCalledTimes(1); expect(context.fileCards).toHaveLength(1); expect(context.wakeup).toHaveBeenCalledTimes(wakeCount); await expect( db.transaction((tx) => context.service.prepareFailedChatRunRetry(tx, { companyId: context.fixture.companyId, issueId: context.issue.id, agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user", }), ), ).rejects.toMatchObject({ details: { code: "chat_failed_run_retry_not_authorized" }, }); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }); it.each([ "sibling_unknown", "foreign_comment_run", "unowned_attachment", "source_edited", "upload_unknown", "active_attempt", ] as const)( "does not exempt other unknown effects or changed native authority: %s", async (variant) => { const context = await nativeFileFixture(); try { context.controls.beforeConsentReceipt = async () => { throw new Error("controlled lost card POST receipt"); }; await context.service.processPendingPublications(); const [unknown] = await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.endpointId, context.endpoint.id)); expect(unknown).toMatchObject({ phase: "consent_unknown", attemptId: null, }); if (variant === "sibling_unknown") { await db.insert(chatPublications).values({ companyId: unknown!.companyId, endpointId: unknown!.endpointId, conversationId: unknown!.conversationId, issueId: unknown!.issueId, commentId: unknown!.commentId, state: "delivery_unknown", idempotencyKey: `run:${context.runId}:other-unknown`, payload: { text: "Other uncertain effect" }, }); } else if (variant === "foreign_comment_run") { await db .update(issueComments) .set({ createdByRunId: null }) .where(eq(issueComments.id, unknown!.commentId)); } else if (variant === "unowned_attachment") { await db .update(issueAttachments) .set({ originatingRunId: null }) .where(eq(issueAttachments.id, context.attachment.id)); } else if (variant === "source_edited") { await db .update(issueComments) .set({ updatedAt: new Date(Date.now() + 1) }) .where( eq( issueComments.id, String(context.sourceAction.payload.commentId), ), ); } else if (variant === "upload_unknown") { await db .update(chatTeamsFileTransfers) .set({ phase: "upload_unknown", version: unknown!.version + 1 }) .where(eq(chatTeamsFileTransfers.id, unknown!.id)); } else { await db .update(chatTeamsFileTransfers) .set({ attemptId: randomUUID(), attemptExpiresAt: new Date(Date.now() + 90_000), }) .where(eq(chatTeamsFileTransfers.id, unknown!.id)); } const [before] = await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.id, unknown!.id)); await expect(context.dispatchConsent()).resolves.toEqual({ status: 403, }); const [after] = await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.id, unknown!.id)); expect(after).toEqual(before); expect(context.consentCards).toHaveLength(1); expect(context.uploadRequest).not.toHaveBeenCalled(); expect(context.fileCards).toEqual([]); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }, ); it("retries only the confirmed native file-info stage with an exact audited phase/version, never its PUT", async () => { const context = await nativeFileFixture(); try { await context.service.processPendingPublications(); context.controls.beforeFileInfoReceipt = async () => { throw new Error("controlled lost file-info POST receipt"); }; await expect(context.dispatchConsent()).resolves.toEqual({ status: 200, }); await context.service.processPendingPublications(); const [unknown] = await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.endpointId, context.endpoint.id)); expect(unknown).toMatchObject({ phase: "file_info_unknown", attemptId: null, fileInfoMessageId: null, }); expect(context.uploadRequest).toHaveBeenCalledTimes(1); expect(context.fileCards).toHaveLength(1); for (const requested of [ undefined, { phase: "consent_unknown" as const, version: unknown!.version }, { phase: "file_info_unknown" as const, version: unknown!.version - 1, }, ]) { await expect( context.service.resolvePublication( context.endpoint.id, unknown!.publicationId, "retry_anyway", "owner-user", requested, ), ).rejects.toMatchObject({ status: 409, details: { code: "chat_file_transfer_resolution_required" }, }); } const [unchanged] = await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.id, unknown!.id)); expect(unchanged).toEqual(unknown); context.controls.beforeFileInfoReceipt = undefined; await context.service.resolvePublication( context.endpoint.id, unknown!.publicationId, "retry_anyway", "owner-user", { phase: "file_info_unknown", version: unknown!.version }, ); await context.service.processPendingPublications(); const [done] = await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.id, unknown!.id)); expect(done).toMatchObject({ phase: "delivered", fileInfoMessageId: "native-file-2", }); expect(context.consentCards).toHaveLength(1); expect(context.uploadRequest).toHaveBeenCalledTimes(1); expect(context.fileCards).toHaveLength(2); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }); }); describe("Teams current file authority and composed Board transfer", () => { // Actual installed inbound parser + real admission, intent, transfer, // projection and callback service. Runtime sends/PUT and the authenticated // App hook ingress are controlled here; this is not tenant/JWT/live proof. it("freezes the original recipient at Board Send and publishes only after consent and exact-byte PUT", async () => { const context = await teamsFileAuthorityFixture(); try { const attachment = await context.createFile(); const publication = await context.service.publishBoardMessage( context.endpoint.id, context.conversation.id, "One original file for the current task recipient", `teams-board-file-${randomUUID()}`, "owner-user", [attachment.id], ); const [intent] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, context.endpoint.id), eq(chatActions.kind, "teams_board_file_intent"), ), ); expect(intent).toMatchObject({ status: "processed", deliveryId: context.sourceDelivery.id, principalId: context.sourceAction.principalId, payload: { publicationId: publication.id, attachmentId: attachment.id, sourceActionId: context.sourceAction.id, userId: "owner-user", }, }); const [transfer] = await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.publicationId, publication.id)); expect(transfer).toMatchObject({ phase: "awaiting_consent", providerUserId: context.providerUserId, aadObjectId: context.aadObjectId, providerConversationId: context.providerConversationId, sha256: createHash("sha256").update(context.bytes).digest("hex"), byteSize: context.bytes.length, consentMessageId: "native-consent-1", fileInfoMessageId: null, }); expect(context.consentCards).toHaveLength(1); expect(context.fileCards).toEqual([]); expect(context.uploadRequest).not.toHaveBeenCalled(); const before = await context.service.getPublicationBatchStatus( context.endpoint.id, context.conversation.id, publication.id, ); expect(before).toMatchObject({ published: 1, awaitingConsent: 1, canDismiss: false, }); const wakeCount = context.wakeup.mock.calls.length; await expect(context.dispatchConsent()).resolves.toEqual({ status: 200, }); await context.service.processPendingPublications(); const [done] = await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.id, transfer!.id)); expect(done).toMatchObject({ phase: "delivered", consentMessageId: "native-consent-1", fileInfoMessageId: "native-file-1", }); expect(context.uploadRequest).toHaveBeenCalledTimes(1); expect(context.uploadRequest.mock.calls[0]![1]).toMatchObject({ method: "PUT", redirect: "manual", body: context.bytes, }); expect(context.fileCards).toHaveLength(1); const status = await context.service.getPublicationBatchStatus( context.endpoint.id, context.conversation.id, publication.id, ); expect(status).toMatchObject({ published: 2, awaitingConsent: 0, settled: 2, canDismiss: true, }); expect( JSON.stringify([ status, await context.service.listActivity(context.endpoint.id), ]), ).not.toMatch( /PRIVATE-UPLOAD-CANARY|pcfc_|ciphertext|fixture\.sharepoint/, ); await context.service.processPendingPublications(); expect(context.consentCards).toHaveLength(1); expect(context.fileCards).toHaveLength(1); expect(context.uploadRequest).toHaveBeenCalledTimes(1); expect(context.wakeup).toHaveBeenCalledTimes(wakeCount); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }); it("stages native Board file consent on the first send from a cold reconstructed service", async () => { const context = await teamsFileAuthorityFixture(); const coldRuntime = new FakeChatSdkRuntime(); const replace = coldRuntime.replaceEndpoint.bind(coldRuntime); const consentPost = vi.fn(async (_threadId: string, _card: unknown) => ({ id: "cold-consent-1", })); vi.spyOn(coldRuntime, "replaceEndpoint").mockImplementation( async (options) => { const instance = await replace(options); const thread = instance.thread.bind(instance); vi.spyOn(instance, "thread").mockImplementation((id) => ({ ...thread(id), isDM: true, })); Object.assign(instance, { sendTeamsFileConsentCard: consentPost, sendTeamsUploadedFileCard: async () => { throw new Error("No upload is authorized before consent"); }, }); return instance; }, ); const coldWakeup = vi.fn< ChatChannelServiceOptions["heartbeat"]["wakeup"] >(async () => ({ accepted: true })); let coldService: ChatChannelService | undefined; try { const attachment = await context.createFile(); await context.service.shutdown(); coldService = chatChannelService(db, { fetch: async () => Response.json({ access_token: "synthetic-cold-access" }), runtime: coldRuntime as unknown as ChatSdkRuntime, heartbeat: { wakeup: receiptBackedWakeup(coldWakeup), cancelRun: vi.fn(async () => ({ status: "cancelled" })), }, storage: context.storage.storage, publicBaseUrl: "https://paperclip.example", teamsFileUploadRequest: context.uploadRequest, scheduleDeferredWork: () => undefined, }); expect(coldRuntime.endpoints.size).toBe(0); expect(coldRuntime.replaceCount).toBe(0); await expect( coldService.get(context.endpoint.id), ).resolves.toMatchObject({ status: "active" }); const publication = await coldService.publishBoardMessage( context.endpoint.id, context.conversation.id, "First Board send after service reconstruction", `teams-cold-${randomUUID()}`, "owner-user", [attachment.id], ); const [intent] = await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, context.endpoint.id), eq(chatActions.kind, "teams_board_file_intent"), ), ); expect(intent).toMatchObject({ deliveryId: context.sourceDelivery.id, principalId: context.sourceAction.principalId, payload: { publicationId: publication.id, sourceActionId: context.sourceAction.id, attachmentId: attachment.id, }, }); expect( await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.publicationId, publication.id)), ).toEqual([ expect.objectContaining({ phase: "awaiting_consent", consentMessageId: "cold-consent-1", fileInfoMessageId: null, providerUserId: context.providerUserId, aadObjectId: context.aadObjectId, sha256: createHash("sha256").update(context.bytes).digest("hex"), }), ]); expect(consentPost).toHaveBeenCalledTimes(1); expect(coldWakeup).not.toHaveBeenCalled(); expect(context.uploadRequest).not.toHaveBeenCalled(); expect( await coldService.getPublicationBatchStatus( context.endpoint.id, context.conversation.id, publication.id, ), ).toMatchObject({ published: 1, awaitingConsent: 1, canDismiss: false, }); } finally { try { await context.service.shutdown(); } finally { await retirePublicationFixture( coldService ?? context.service, context.endpoint.id, ); } } }); async function beginTeamsBoardTransfer( context: Awaited>, ) { const attachment = await context.createFile(); const publication = await context.service.publishBoardMessage( context.endpoint.id, context.conversation.id, "The exact current original file", `teams-authority-race-${randomUUID()}`, "owner-user", [attachment.id], ); const [transfer] = await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.publicationId, publication.id)); expect(transfer).toMatchObject({ phase: "awaiting_consent", consentMessageId: "native-consent-1", }); expect(context.consentCards).toHaveLength(1); expect(context.uploadRequest).not.toHaveBeenCalled(); return { publication, transfer: transfer! }; } it.each(["linked_recipient", "sponsor", "board_author"] as const)( "rechecks revoked %s authority before acceptance and before PUT", async (role) => { for (const boundary of ["accept", "put"] as const) { const context = await teamsFileAuthorityFixture({ linkedRecipient: role === "linked_recipient", separateSponsor: true, }); try { const { transfer } = await beginTeamsBoardTransfer(context); if (boundary === "put") await expect(context.dispatchConsent()).resolves.toEqual({ status: 200, }); if (role === "linked_recipient") await db .update(chatIdentityLinks) .set({ status: "revoked", revokedAt: new Date() }) .where( and( eq(chatIdentityLinks.companyId, context.fixture.companyId), eq(chatIdentityLinks.endpointId, context.endpoint.id), eq( chatIdentityLinks.principalId, context.sourceAction.principalId!, ), ), ); else await db .update(companyMemberships) .set({ membershipRole: "viewer" }) .where( and( eq(companyMemberships.companyId, context.fixture.companyId), eq(companyMemberships.principalType, "user"), eq( companyMemberships.principalId, role === "sponsor" ? context.sponsorUserId : "owner-user", ), ), ); const [before] = await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.id, transfer.id)); if (boundary === "accept") { const outcome = await context.dispatchConsent(); // No success ACK for a revoked principal. The current callback // contract distinguishes explicit denial from closed transient failure. expect([403, 503]).toContain(outcome.status); } await context.service.processPendingPublications(); expect( await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.id, transfer.id)), ).toEqual([before]); expect(context.uploadRequest).not.toHaveBeenCalled(); expect(context.fileCards).toEqual([]); expect(context.consentCards).toHaveLength(1); } finally { await retirePublicationFixture( context.service, context.endpoint.id, ); } } }, ); it("rejects a retained old-runtime consent callback after normal pause and resume", async () => { const context = await teamsFileAuthorityFixture(); try { const { transfer } = await beginTeamsBoardTransfer(context); const oldRuntime = context.runtime.endpoints.get(context.endpoint.id); const oldHandler = context.handlers.get("file.consent.accept")!; await context.service.configure( context.endpoint.id, { action: "pause" }, "owner-user", ); await context.service.configure( context.endpoint.id, { action: "resume" }, "owner-user", ); expect(context.runtime.endpoints.get(context.endpoint.id)).not.toBe( oldRuntime, ); const [before] = await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.id, transfer.id)); await expect( context.dispatchConsent("accept", {}, oldHandler), ).resolves.toEqual({ status: 403 }); expect( await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.id, transfer.id)), ).toEqual([before]); expect(context.uploadRequest).not.toHaveBeenCalled(); expect(context.fileCards).toEqual([]); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }); it("deduplicates the same actual acceptance activity before and after file delivery", async () => { const context = await teamsFileAuthorityFixture(); try { const { transfer } = await beginTeamsBoardTransfer(context); const activity = { id: `same-accept-${randomUUID()}` }; await expect( context.dispatchConsent("accept", activity), ).resolves.toEqual({ status: 200 }); await expect( context.dispatchConsent("accept", activity), ).resolves.toEqual({ status: 200 }); await context.service.processPendingPublications(); const [before] = await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.id, transfer.id)); expect(before?.phase).toBe("delivered"); await expect( context.dispatchConsent("accept", activity), ).resolves.toEqual({ status: 200 }); await context.service.processPendingPublications(); expect( await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.id, transfer.id)), ).toEqual([before]); expect( await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, context.endpoint.id), eq(chatActions.kind, "teams_file_consent"), ), ), ).toHaveLength(1); expect(context.consentCards).toHaveLength(1); expect(context.uploadRequest).toHaveBeenCalledTimes(1); expect(context.fileCards).toHaveLength(1); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }); it("durably accepts a callback inside the consent POST before the native receipt returns", async () => { const context = await teamsFileAuthorityFixture(); try { let duringPost: typeof chatTeamsFileTransfers.$inferSelect | undefined; context.controls.beforeConsentReceipt = async () => { await expect(context.dispatchConsent()).resolves.toEqual({ status: 200, }); [duringPost] = await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.endpointId, context.endpoint.id)); expect(duringPost).toMatchObject({ phase: "consent_sending", consentMessageId: null, responseActivityId: expect.any(String), }); expect(JSON.stringify(duringPost)).not.toContain( "PRIVATE-UPLOAD-CANARY", ); expect(context.uploadRequest).not.toHaveBeenCalled(); }; const attachment = await context.createFile(); await context.service.publishBoardMessage( context.endpoint.id, context.conversation.id, "Early callback exact file", `teams-early-${randomUUID()}`, "owner-user", [attachment.id], ); expect(duringPost).toBeDefined(); const [finished] = await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.id, duringPost!.id)); expect(finished).toMatchObject({ phase: "delivered", consentMessageId: "native-consent-1", fileInfoMessageId: "native-file-1", }); expect(context.consentCards).toHaveLength(1); expect(context.uploadRequest).toHaveBeenCalledTimes(1); expect(context.fileCards).toHaveLength(1); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }); it("offers only exact conflict cancellation after valid Teams consent receipts disagree", async () => { const context = await teamsFileAuthorityFixture(); try { const { publication } = await beginTeamsBoardTransfer(context); const wakeCount = context.wakeup.mock.calls.length; await expect(context.dispatchConsent("accept")).resolves.toEqual({ status: 200, }); await expect(context.dispatchConsent("decline")).resolves.toEqual({ status: 403, }); const [conflict] = await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.publicationId, publication.id)); expect(conflict).toMatchObject({ phase: "conflict", attemptId: null, attemptExpiresAt: null, }); const app = routesApp(db, context.fixture.companyId, context.service); const before = await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, publication.commentId!)); const activity = await request(app) .get(`/api/chat-endpoints/${context.endpoint.id}/activity`) .expect(200); const item = activity.body.find( (row: { id: string }) => row.id === publication.id, ); expect(item).toMatchObject({ status: "delivery_unknown", replayable: false, resolutionActions: ["cancel"], fileTransfer: { phase: "conflict", version: conflict!.version }, }); expect(JSON.stringify(activity.body)).not.toMatch( /PRIVATE-UPLOAD-CANARY|pcfc_|ciphertext|fixture\.sharepoint/, ); expect( await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.publicationId, publication.id)), ).toEqual([conflict]); expect( await db .select() .from(chatPublications) .where(eq(chatPublications.commentId, publication.commentId!)), ).toEqual(before); const path = `/api/chat-endpoints/${context.endpoint.id}/publications/${publication.id}/resolve`; const hint = { phase: "conflict", version: conflict!.version }; for (const action of ["mark_delivered", "retry_anyway"]) await request(app) .post(path) .send({ action, fileTransfer: hint }) .expect(409); await request(app) .post(path) .send({ action: "cancel", fileTransfer: hint }) .expect(204); await request(app) .post(path) .send({ action: "cancel", fileTransfer: hint }) .expect(409); const [cancelled] = await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.publicationId, publication.id)); expect(cancelled).toMatchObject({ phase: "cancelled", reason: "operator_cancelled_conflict", version: conflict!.version + 1, fileInfoMessageId: null, }); expect( await context.service.getPublicationBatchStatus( context.endpoint.id, context.conversation.id, publication.id, ), ).toMatchObject({ published: 1, cancelled: 1, settled: 2, canDismiss: true, }); expect(context.consentCards).toHaveLength(1); expect(context.uploadRequest).not.toHaveBeenCalled(); expect(context.fileCards).toHaveLength(0); expect(context.wakeup).toHaveBeenCalledTimes(wakeCount); await expect( db .select() .from(activityLog) .where( and( eq(activityLog.companyId, context.fixture.companyId), eq(activityLog.entityId, publication.id), eq(activityLog.action, "chat.publication_cancel"), ), ), ).resolves.toEqual([ expect.objectContaining({ actorType: "user", actorId: "owner-user", details: expect.objectContaining({ previousPhase: "conflict", previousVersion: conflict!.version, nextPhase: "cancelled", duplicateRiskAcknowledged: false, }), }), ]); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }); it("retries only fileInfo after a confirmed PUT and an explicitly resolved missing fileInfo receipt", async () => { const context = await teamsFileAuthorityFixture(); try { const { transfer, publication } = await beginTeamsBoardTransfer(context); context.controls.beforeFileInfoReceipt = async () => { throw new Error( "Synthetic provider ACK timeout after attempted fileInfo POST", ); }; await expect(context.dispatchConsent()).resolves.toEqual({ status: 200, }); await context.service.processPendingPublications(); const [unknown] = await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.id, transfer.id)); expect(unknown).toMatchObject({ phase: "file_info_unknown", consentMessageId: "native-consent-1", fileInfoMessageId: null, }); expect(context.uploadRequest).toHaveBeenCalledTimes(1); expect(context.fileCards).toHaveLength(1); await context.service.processPendingPublications(); expect(context.fileCards).toHaveLength(1); context.controls.beforeFileInfoReceipt = undefined; await context.service.resolvePublication( context.endpoint.id, publication.id, "retry_anyway", "owner-user", { phase: "file_info_unknown", version: unknown!.version }, ); const [finished] = await db .select() .from(chatTeamsFileTransfers) .where(eq(chatTeamsFileTransfers.id, transfer.id)); expect(finished).toMatchObject({ phase: "delivered", fileInfoMessageId: "native-file-2", }); expect(context.consentCards).toHaveLength(1); expect(context.uploadRequest).toHaveBeenCalledTimes(1); expect(context.uploadRequest.mock.calls[0]![1]).toMatchObject({ body: context.bytes, }); expect(context.fileCards).toHaveLength(2); expect( await db .select() .from(activityLog) .where( and( eq(activityLog.companyId, context.fixture.companyId), eq(activityLog.entityId, publication.id), eq(activityLog.action, "chat.publication_retry_anyway"), ), ), ).toEqual([ expect.objectContaining({ details: expect.objectContaining({ previousPhase: "file_info_unknown", duplicateRiskAcknowledged: true, }), }), ]); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }); it.each([ "missing_original_proof", "mismatched_original_aad", "different_admitted_recipient", ] as const)( "keeps the task-link fallback with no native file intent for %s", async (variant) => { const context = await teamsFileAuthorityFixture(); try { if (variant === "different_admitted_recipient") { await context.deliver({ ...context.raw, id: `other-source-${randomUUID()}`, from: { ...context.raw.from, id: `29:other-${randomUUID()}`, aadObjectId: randomUUID(), }, }); } else { const normalized = { ...context.sourceDelivery.normalizedEvent }; if (variant === "missing_original_proof") delete normalized.teamsPersonalRecipient; else normalized.teamsPersonalRecipient = { ...(normalized.teamsPersonalRecipient as Record< string, unknown >), aadObjectId: randomUUID(), }; await db .update(chatDeliveries) .set({ normalizedEvent: normalized }) .where(eq(chatDeliveries.id, context.sourceDelivery.id)); } const attachment = await context.createFile(); const publication = await context.service.publishBoardMessage( context.endpoint.id, context.conversation.id, "Share the authorized original file", `teams-safe-fallback-${randomUUID()}`, "owner-user", [attachment.id], ); expect( await db .select() .from(chatActions) .where( and( eq(chatActions.endpointId, context.endpoint.id), eq(chatActions.kind, "teams_board_file_intent"), ), ), ).toEqual([]); expect( await db .select() .from(chatTeamsFileTransfers) .where( eq(chatTeamsFileTransfers.endpointId, context.endpoint.id), ), ).toEqual([]); expect(context.consentCards).toEqual([]); expect(context.fileCards).toEqual([]); expect(context.uploadRequest).not.toHaveBeenCalled(); expect(publication.state).toBe("published"); expect( context.runtime.endpoints .get(context.endpoint.id)! .posts.some((post) => post.text.includes(context.issue.id)), ).toBe(true); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }, ); }); describe("Lossless long Board publications", () => { async function longFixture( provider: "slack" | "github" | "microsoft-teams", ) { const fixture = await seedCompany(); const context = provider === "slack" ? await configuredSlackEndpoint(fixture) : provider === "github" ? await configuredGitHubEndpoint(fixture) : await configuredTeamsEndpoint(fixture); try { const teamsChannel = `teams:${Buffer.from("19:long-publication@thread.tacv2").toString("base64url")}:${Buffer.from("https://smba.trafficmanager.net/amer/").toString("base64url")}`; if (provider === "microsoft-teams") { await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: context.endpoint.id, type: "channel", providerResourceId: "19:long-publication@thread.tacv2", label: "Long publication channel", availability: "available", enabled: true, }); } const channel = makeThread({ channelId: provider === "slack" ? "C-LONG-BOARD" : provider === "github" ? "paperclipai/paperclip" : teamsChannel, id: provider === "slack" ? "slack:C-LONG-BOARD:9900.1" : provider === "github" ? "github:paperclipai/paperclip:issue:9900" : `teams:${Buffer.from("19:long-publication@thread.tacv2;messageid=99001").toString("base64url")}:${Buffer.from("https://smba.trafficmanager.net/amer/").toString("base64url")}`, name: "long-board", }); await deliverMessage({ callbacks: context.callbacks, endpointId: context.endpoint.id, provider, thread: channel.thread, message: makeMessage({ id: provider === "slack" ? "9900.1" : "99001", text: "@maya begin the long Board publication fixture", mentioned: true, }), trigger: "mention", }); await qualifySetupRoundTrip(context.service, context.endpoint.id); await context.service.test(context.endpoint.id, "owner-user"); const [conversation] = await context.service.listConversations( context.endpoint.id, ); if (!conversation) throw new Error("Expected long publication conversation"); const transport = context.runtime.endpoints.get(context.endpoint.id)!; transport.posts.length = 0; return { ...context, fixture, conversation, transport, channel }; } catch (error) { await retirePublicationFixture(context.service, context.endpoint.id); throw error; } } const orderedBatch = (commentId: string) => db .select() .from(chatPublications) .where(eq(chatPublications.commentId, commentId)) .orderBy( asc(chatPublications.createdAt), asc(sql`${chatPublications.payload}->'transportPart'->>'orderKey'`), ); it.each([ ["slack", "new"], ["slack", "existing"], ["github", "new"], ["github", "existing"], ["microsoft-teams", "new"], ["microsoft-teams", "existing"], ] as const)( "preserves all 100000 characters for %s %s-comment publication", async (provider, source) => { const { service, endpoint, conversation, transport } = await longFixture(provider); try { const marker = "END-OF-100000-CHARACTER-BOARD-SEND"; const body = "a".repeat(100_000 - marker.length) + marker; const comment = source === "existing" ? await issueService(db).addComment(conversation.issueId, body, { userId: "owner-user", }) : null; const publication = comment ? await service.publishComment( endpoint.id, conversation.id, comment.id, ) : await service.publishBoardMessage( endpoint.id, conversation.id, body, "lossless-long-board-send", "owner-user", ); for (let drain = 0; drain < 20; drain++) await service.processPendingPublications(); expect(publication?.commentId).toBeTruthy(); const rows = await orderedBatch(publication!.commentId!); expect(rows.every((row) => row.state === "published")).toBe(true); expect(rows.map((row) => row.payload.text).join("") === body).toBe( true, ); expect( transport.posts.map((post) => post.text).join("") === body, ).toBe(true); expect( transport.posts.every((post) => nativePublicationTextFits(provider, post.text), ), ).toBe(true); expect(transport.posts.at(-1)?.text).toContain(marker); const before = transport.posts.length; if (comment) await service.publishComment( endpoint.id, conversation.id, comment.id, ); else await service.publishBoardMessage( endpoint.id, conversation.id, body, "lossless-long-board-send", "owner-user", ); expect(transport.posts).toHaveLength(before); } finally { await retirePublicationFixture(service, endpoint.id); } }, ); it.each(["slack", "github", "microsoft-teams"] as const)( "preserves %s code, Unicode and sanitization expansion through actual durable parts", async (provider) => { const { service, endpoint, conversation, transport } = await longFixture(provider); try { const body = "```ts\n" + "const answer = '😀';\n".repeat(4_000) + "```\n\n" + "@here ".repeat(1_000) + "END-OF-RICH-BOARD"; expect(body.length).toBeLessThanOrEqual(100_000); const safe = projectSafeChatPublicationText(body); const publication = await service.publishBoardMessage( endpoint.id, conversation.id, body, "long-rich-board-send", "owner-user", ); for (let drain = 0; drain < 20; drain++) await service.processPendingPublications(); const rows = await orderedBatch(publication.commentId!); expect(rows.every((row) => row.state === "published")).toBe(true); expect(rows.map((row) => row.payload.text).join("") === safe).toBe( true, ); expect(transport.posts.map((post) => post.text)).toEqual( rows.map((row) => renderPublicationTransportText(row.payload)), ); expect( transport.posts.every((post) => nativePublicationTextFits(provider, post.text), ), ).toBe(true); expect(transport.posts.at(-1)?.text).toContain("END-OF-RICH-BOARD"); expect(JSON.stringify(transport.posts)).not.toContain("@here"); } finally { await retirePublicationFixture(service, endpoint.id); } }, ); it.each(["discord", "telegram"] as const)( "retains a complete 100000-character %s response in the existing Markdown document transport", async (provider) => { const context = await safeNativeProgressFixture( provider, provider === "discord" ? "811" : "812", ); try { await qualifySetupRoundTrip( context.service, context.endpoint.id, provider === "telegram" ? context.thread.thread.channelId : "U-SAFE-PROGRESS", ); await context.service.test(context.endpoint.id, "owner-user"); context.providerRuntime.posts.length = 0; const marker = "\n```\nEND-OF-COMPLETE-DOCUMENT"; const body = "```txt\n" + "x".repeat(100_000 - marker.length - 7) + marker; expect(body.length).toBe(100_000); const publication = await context.service.publishBoardMessage( context.endpoint.id, context.conversation.id, body, "long-document-board-send", "owner-user", ); for (let drain = 0; drain < 5; drain++) await context.service.processPendingPublications(); const rows = await orderedBatch(publication.commentId!); expect(rows).toHaveLength(1); expect(rows[0]!.state).toBe("published"); expect(rows[0]!.payload.text).toBe(body); expect(rows[0]!.payload.transportPart?.mode).toBe( `${provider}_markdown_attachment`, ); expect(context.providerRuntime.posts).toHaveLength(1); const post = context.providerRuntime.posts[0]!; const upload = ( provider === "discord" ? post.files?.[0] : post.attachments?.[0] ) as { data: Buffer }; expect(Buffer.isBuffer(upload.data)).toBe(true); expect(upload.data.equals(Buffer.from(body))).toBe(true); await context.service.processPendingPublications(); expect(context.providerRuntime.posts).toHaveLength(1); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }, ); it.each(["slack", "github"] as const)( "keeps an ambiguous %s text part blocked across restart and retries only that part and its tail", async (provider) => { const { service, endpoint, conversation, transport } = await longFixture(provider); let restarted: ChatChannelService | undefined; try { const body = "a".repeat(99_990) + "FINAL-TAIL"; let attempts = 0; transport.postHook = async () => { if (++attempts === 2) throw new Error("connection closed after request write"); }; const blocked = await service.publishBoardMessage( endpoint.id, conversation.id, body, "long-unknown-board-send", "owner-user", ); expect(blocked.state).toBe("delivery_unknown"); const initial = await orderedBatch(blocked.commentId!); expect(initial[0]!.state).toBe("published"); expect(initial[1]!.id).toBe(blocked.id); expect( initial .slice(2) .every((row) => row.state === "pending" && row.attempts === 0), ).toBe(true); expect(attempts).toBe(2); await service.shutdown(); const fresh = createService(new FakeChatSdkRuntime()); restarted = fresh.service; await restarted.processPendingPublications(); await restarted.processPendingPublications(); expect(await orderedBatch(blocked.commentId!)).toEqual(initial); expect(fresh.runtime.endpoints.get(endpoint.id)?.posts ?? []).toEqual( [], ); await expect( restarted.replayPublication(endpoint.id, blocked.id), ).rejects.toMatchObject({ status: 409 }); await restarted.resolvePublication( endpoint.id, blocked.id, "retry_anyway", "owner-user", ); for (let drain = 0; drain < 20; drain++) await restarted.processPendingPublications(); const final = await orderedBatch(blocked.commentId!); expect(final[0]).toEqual(initial[0]); expect(final.every((row) => row.state === "published")).toBe(true); expect(final[1]!.attempts).toBe(2); expect(final.slice(2).every((row) => row.attempts === 1)).toBe(true); const resumed = fresh.runtime.endpoints.get(endpoint.id)!.posts; expect( [...transport.posts, ...resumed] .map((post) => post.text) .join("") === body, ).toBe(true); expect( ( await restarted.getPublicationBatchStatus( endpoint.id, conversation.id, initial[0]!.id, ) ).published, ).toBe(final.length); await restarted.processPendingPublications(); expect(await orderedBatch(blocked.commentId!)).toEqual(final); } finally { await service.shutdown(); await retirePublicationFixture(restarted ?? service, endpoint.id); } }, ); it("preserves an accepted native Slack response whose rendered mentions exceed the Markdown cap", async () => { const summary = "@U12345678 ".repeat(1_080) + "NATIVE-END"; expect(summary.length).toBeLessThan(12_000); const context = await committedChatResponseRecoveryFixture( "slack", "coordinator", false, undefined, () => summary, ); try { expect(await context.repair()).toBe(true); const [comment] = await db .select() .from(issueComments) .where(eq(issueComments.createdByRunId, context.runId)); await context.service.processPendingPublications(100); const rows = await orderedBatch(comment!.id); expect(rows.length).toBeGreaterThan(1); expect(rows.every((row) => row.state === "published")).toBe(true); expect(rows.map((row) => row.payload.text).join("")).toBe( summary.trim(), ); expect( rows.every((row) => nativePublicationTextFits( "slack", renderPublicationTransportText(row.payload), ), ), ).toBe(true); expect(await context.repair()).toBe(false); } finally { await retirePublicationFixture(context.service, context.endpoint.id); } }); it.each([ { prefix: "https://private.example/?token=PRIVATE" }, { suffix: { token: "PRIVATE" } }, ])( "refuses malformed persisted wrappers before provider I/O %#", async (badWrapper) => { const { service, endpoint, conversation, transport, fixture } = await longFixture("slack"); try { const comment = await issueService(db).addComment( conversation.issueId, "Safe content", { userId: "owner-user" }, ); const [publication] = await db .insert(chatPublications) .values({ companyId: fixture.companyId, endpointId: endpoint.id, conversationId: conversation.id, issueId: conversation.issueId, commentId: comment.id, idempotencyKey: `explicit:${comment.id}:${endpoint.id}`, state: "pending", payload: { text: "Safe content", transportPart: { batchId: randomUUID(), count: 1, index: 0, orderKey: "fixture:0000", ...badWrapper, }, } as never, }) .returning(); await service.processPendingPublications(); expect(transport.posts).toEqual([]); const [after] = await db .select() .from(chatPublications) .where(eq(chatPublications.id, publication!.id)); expect(after!.state).toBe("failed"); expect(after!.redactedError).not.toContain("PRIVATE"); expect(after!.providerMessageId).toBeNull(); } finally { await retirePublicationFixture(service, endpoint.id); } }, ); }); describe("Telegram retained zero-message admission", () => { it.each([ { state: "received", command: false, source: "zero" }, { state: "retry", command: false, source: "zero" }, { state: "processing", command: false, source: "zero" }, { state: "received", command: true, source: "zero" }, { state: "retry", command: true, source: "zero" }, { state: "processing", command: true, source: "zero" }, { state: "issued", command: false, source: "zero" }, { state: "issued", command: false, source: "zero_sequence" }, { state: "retry", command: false, source: "zero_event" }, { state: "processing", command: false, source: "zero_microseconds" }, { state: "processing", command: false, source: "zero_locked" }, { state: "processing", command: false, source: "zero_replaced" }, { state: "received", command: false, source: "positive" }, { state: "issued", command: false, source: "positive" }, { state: "received", command: false, source: "legacy" }, ] as const)( "recovers only ordinary source identity ($state / command=$command / $source)", async ({ state, command, source }) => { const fixture = await seedCompany(); const pendingWake = state === "issued"; const replacedSource = source === "zero_replaced"; const preparingWake = source === "zero_locked" || replacedSource; const seededWake = pendingWake || preparingWake; const denied = source.startsWith("zero") && !replacedSource; const first = await configuredTelegramEndpoint(fixture, { deferWebhookProcessing: !seededWake, scheduleDeferredWork: () => undefined, ...(seededWake ? { wakeup: async () => { throw new Error("injected scheduler outage before receipt"); }, } : {}), }); let restarted: ReturnType | undefined; try { await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, first.endpoint.id)); await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: first.endpoint.id, type: "chat", providerResourceId: "-100123", label: "Retained source fixture", availability: "available", enabled: true, }); const channel = makeThread({ channelId: "-100123", id: "telegram:-100123", isDM: false, }); await deliverMessage({ callbacks: first.callbacks, endpointId: first.endpoint.id, provider: "telegram", thread: channel.thread, trigger: "mention", message: makeMessage({ id: "-100123:71", text: command ? "/task PRIVATE_RETAINED_ZERO_BODY" : "@maya PRIVATE_RETAINED_ZERO_BODY", mentioned: true, userId: "77115580", raw: { message_id: 71, chat: { id: -100123 }, from: { id: 77115580 }, date: Math.floor(Date.now() / 1000), }, }), }); const [original] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, first.endpoint.id)); expect(original).toMatchObject({ state: seededWake ? "processed" : "received", }); const actions = await db .select() .from(chatActions) .where( and( eq(chatActions.deliveryId, original.id), eq(chatActions.kind, "inbound_wakeup"), ), ); expect(actions).toHaveLength(seededWake ? 1 : 0); if (seededWake) expect(actions[0]).toMatchObject({ status: "issued" }); expect( await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.companyId, fixture.companyId)), ).toEqual([]); await first.service.shutdown(); expect(first.runtime.get(first.endpoint.id)).toBeNull(); // Simulate a receipt saved by an older process. No current runtime // callback supplies raw message_id during either recovery path. const messageId = source === "zero" || source === "zero_microseconds" || preparingWake ? "-100123:0" : source === "legacy" ? "legacy-message" : "-100123:71"; const eventId = `${channel.thread.id}:${source === "zero_event" ? "-100123:0" : messageId}`; const normalized = original.normalizedEvent as { message: Record; }; const retained = { ...original.normalizedEvent, providerEventId: eventId, message: { ...normalized.message, providerMessageId: messageId, providerMessageSequence: source === "zero_sequence" ? 0 : source === "legacy" || source === "zero" ? null : 71, }, }; await db .update(chatDeliveries) .set({ providerEventId: eventId, normalizedEvent: retained, state: pendingWake ? "processed" : (state as "received" | "retry" | "processing"), nextAttemptAt: new Date(0), updatedAt: new Date(0), }) .where(eq(chatDeliveries.id, original.id)); if (source === "zero_microseconds") { await db.execute( sql`update chat_deliveries set updated_at = '2026-09-01 12:00:00.123456+00'::timestamptz where id = ${original.id}::uuid`, ); expect( await db .select({ micros: sql`to_char(${chatDeliveries.updatedAt}, 'US')`, }) .from(chatDeliveries) .where(eq(chatDeliveries.id, original.id)), ).toEqual([{ micros: "123456" }]); } if (seededWake) { await db .update(chatActions) .set({ ...(preparingWake ? { status: "preparing" } : {}), result: { code: "inbound_wakeup_retry", retryAt: new Date(0).toISOString(), }, }) .where(eq(chatActions.id, actions[0]!.id)); await db .update(chatMessageLinks) .set({ providerMessageId: messageId }) .where(eq(chatMessageLinks.deliveryId, original.id)); } const commentsBefore = await db .select() .from(issueComments) .where(eq(issueComments.companyId, fixture.companyId)); const issuesBefore = await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)); const publicationsBefore = await db .select() .from(chatPublications) .where(eq(chatPublications.companyId, fixture.companyId)); restarted = createService( new FakeChatSdkRuntime(), fakeTelegramFetch() as typeof globalThis.fetch, { scheduleDeferredWork: () => undefined }, ); if (preparingWake) { const replacement = { ...retained, providerEventId: `${channel.thread.id}:-100123:71`, message: { ...retained.message, providerMessageId: "-100123:71", providerMessageSequence: 71, }, }; let release!: () => void; let acquired!: () => void; const held = new Promise((resolve) => { release = resolve; }); const locked = new Promise((resolve) => { acquired = resolve; }); const owner = db.transaction(async (tx) => { await tx .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, original.id)) .for("update"); await tx .update(chatDeliveries) .set({ updatedAt: new Date(), ...(replacedSource ? { providerEventId: replacement.providerEventId, normalizedEvent: replacement, } : {}), }) .where(eq(chatDeliveries.id, original.id)); if (replacedSource) await tx .update(chatMessageLinks) .set({ providerMessageId: "-100123:71" }) .where(eq(chatMessageLinks.deliveryId, original.id)); acquired(); await held; }); await locked; let drain: Promise | undefined; let deadline: ReturnType | undefined; try { drain = restarted.service.processPendingDeliveries( 25, original.id, ); await Promise.race([ drain, new Promise((_resolve, reject) => { deadline = setTimeout( () => reject( new Error( "held claim must stop the drain without waiting", ), ), 1500, ); }), ]); expect( await db .select({ state: chatDeliveries.state }) .from(chatDeliveries) .where(eq(chatDeliveries.id, original.id)), ).toEqual([{ state: "processing" }]); expect( await db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, actions[0]!.id)), ).toEqual([{ status: "preparing" }]); expect(restarted.wakeup).not.toHaveBeenCalled(); } finally { if (deadline) clearTimeout(deadline); release(); await owner; await drain; } // The newly committed live claim remains owned; filtering must // not continue from the stale pre-lock candidate snapshot. await restarted.service.processPendingDeliveries(25, original.id); if (replacedSource) expect( await db .select({ normalizedEvent: chatDeliveries.normalizedEvent }) .from(chatDeliveries) .where(eq(chatDeliveries.id, original.id)), ).toEqual([{ normalizedEvent: replacement }]); expect( await db .select({ status: chatActions.status }) .from(chatActions) .where(eq(chatActions.id, actions[0]!.id)), ).toEqual([{ status: "preparing" }]); await db.execute( sql`update chat_deliveries set updated_at = '2026-09-01 12:00:00.123456+00'::timestamptz where id = ${original.id}::uuid`, ); } await restarted.service.processPendingDeliveries(25, original.id); const [settled] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, original.id)); if (denied) { expect(restarted.wakeup).not.toHaveBeenCalled(); expect(settled).toMatchObject({ state: pendingWake ? "processed" : "filtered", }); expect(settled.normalizedEvent).toEqual(retained); expect(settled.redactedError).not.toContain( "PRIVATE_RETAINED_ZERO_BODY", ); expect( await db .select() .from(issueComments) .where(eq(issueComments.companyId, fixture.companyId)), ).toEqual(commentsBefore); expect( await db .select() .from(issues) .where(eq(issues.companyId, fixture.companyId)), ).toEqual(issuesBefore); expect( await db .select() .from(chatPublications) .where(eq(chatPublications.companyId, fixture.companyId)), ).toEqual(publicationsBefore); if (seededWake) expect( await db .select() .from(chatActions) .where(eq(chatActions.id, actions[0]!.id)), ).toEqual([ expect.objectContaining({ status: "failed", payload: actions[0]!.payload, result: expect.objectContaining({ code: preparingWake ? "inbound_wakeup_delivery_rejected" : "inbound_wakeup_authorization_changed", }), }), ]); for (const runtime of restarted.runtime.endpoints.values()) { expect(runtime.posts).toEqual([]); expect(runtime.reactions).toEqual([]); expect(runtime.rehydratedAttachmentDescriptors).toEqual([]); } } else { expect(restarted.wakeup).toHaveBeenCalledOnce(); expect(settled.state).toBe("processed"); } expect( await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.companyId, fixture.companyId)), ).toHaveLength(denied ? 0 : 1); expect( await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.companyId, fixture.companyId)), ).toEqual([]); await restarted.service.processPendingDeliveries(25, original.id); expect(restarted.wakeup).toHaveBeenCalledTimes(denied ? 0 : 1); if (denied) expect( await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.id, original.id)), ).toEqual([settled]); } finally { try { await retirePublicationFixture( restarted?.service ?? first.service, first.endpoint.id, ); } finally { await first.service.shutdown(); } } }, ); }); describe("Telegram current rich inbound compatibility", () => { it.each([ "mixed_files", "restart_topic", "revoked", "source_edit", "unknown_document", "unknown_block", "malformed_block", ] as const)( "keeps rich files and omissions scoped through lifecycle recovery (%s)", async (mode) => { const fixture = await seedCompany(); const storage = createStorageService(); const deferred = mode === "restart_topic" || mode === "revoked"; const first = await configuredTelegramEndpoint(fixture, { storage: storage.storage, deferWebhookProcessing: deferred, scheduleDeferredWork: () => undefined, }); const { service, runtime, endpoint, callbacks, wakeup } = first; const configuration = runtime.configurations.get(endpoint.id)!; const pinned = createChatSdkEndpointRuntime({ ...configuration, logger: "silent", }); const recovered = createChatSdkEndpointRuntime({ ...configuration, logger: "silent", }); let restarted: ReturnType | undefined; const topic = mode === "restart_topic"; const chatId = topic ? -10077115580 : 77115580; const threadId = `telegram:${chatId}${topic ? ":42" : ""}`; const thread = makeThread({ channelId: String(chatId), id: threadId, isDM: !topic, }).thread; const bytes = Buffer.from("Exact rich restart file\nshape=hexagon\n"); const photo = await sharp({ create: { width: 16, height: 16, channels: 3, background: "teal" }, }) .jpeg() .toBuffer(); const document = { type: "document", document: { file_id: "rich-document", file_unique_id: "rich-unique", file_name: "rich-document.txt", mime_type: "text/plain", file_size: bytes.length, }, }; const raw = { message_id: 71, date: Math.floor(Date.now() / 1_000), ...(topic ? { message_thread_id: 42 } : {}), chat: { id: chatId, type: topic ? "supergroup" : "private" }, from: { id: 77115580, is_bot: false, first_name: "Rich lifecycle fixture", }, rich_message: { blocks: mode === "unknown_block" ? [{ type: "future_block", text: "PRIVATE_UNSUPPORTED_BODY" }] : mode === "malformed_block" ? [{ type: "details", blocks: null }] : mode === "unknown_document" ? [ { type: "document", document: { file_id: "rich-document", file_unique_id: "rich-unique", file_name: "unknown.bin", file_size: bytes.length, }, }, ] : mode === "mixed_files" ? [ { type: "paragraph", text: "Before files" }, { type: "details", summary: "File section", blocks: [ document, { type: "photo", photo: [ { file_id: "rich-photo", file_unique_id: "rich-photo-unique", width: 16, height: 16, file_size: photo.length, }, ], caption: { text: "Photo caption" }, }, ], }, { type: "expandable_blockquote", text: "Quoted tail", credit: "Quote author", }, ] : [document], }, }; const requests: string[] = []; const fetchSpy = vi .spyOn(globalThis, "fetch") .mockImplementation(async (input, init) => { const url = new URL( input instanceof Request ? input.url : String(input), ); if (url.hostname !== "api.telegram.org") throw new Error("Unexpected fixture host"); if (url.pathname.endsWith("/getFile")) { const file = JSON.parse(String(init?.body)).file_id; expect(["rich-document", "rich-photo"]).toContain(file); requests.push(`getFile:${file}`); return Response.json({ ok: true, result: { file_path: `fixture/${file}` }, }); } const file = url.pathname.split("/").at(-1)!; if (!["rich-document", "rich-photo"].includes(file)) throw new Error("Unexpected fixture method"); requests.push(`download:${file}`); if (mode === "source_edit") { const updated = pinned.parseTelegramCommandMessage({ ...raw, edit_date: raw.date + 1, rich_message: { blocks: [ { ...document, document: { ...document.document, file_id: "different-file", }, }, ], }, })!; await callbacks.onMessageUpdated!({ endpointId: endpoint.id, provider: "telegram", thread, message: updated, }); } return new Response(file === "rich-photo" ? photo : bytes); }); try { Object.assign(runtime.endpoints.get(endpoint.id)!, { attachmentRecoveryDescriptor: pinned.attachmentRecoveryDescriptor.bind(pinned), rehydrateAttachment: pinned.rehydrateAttachment.bind(pinned), }); if (topic) { await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, endpoint.id)); await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "chat", providerResourceId: String(chatId), label: "Rich topic", availability: "available", enabled: true, }); } const message = pinned.parseTelegramCommandMessage(raw)!; if (topic) message.isMention = true; // Verified runtime callback addressing, not attachment authority. const ingress = deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread, message, trigger: topic ? "mention" : "direct_message", }); if (mode === "source_edit") await expect(ingress).rejects.toThrow("admitted source changed"); else await ingress; const [delivery] = await db .select() .from(chatDeliveries) .where( and( eq(chatDeliveries.endpointId, endpoint.id), eq( chatDeliveries.eventKind, topic ? "mention" : "direct_message", ), ), ); if (deferred) { expect(delivery).toMatchObject({ state: "received", attempts: 0 }); expect(requests).toEqual([]); expect(wakeup).not.toHaveBeenCalled(); const retired = vi.fn(async () => { throw new Error("Retired rich closure used"); }); message.attachments[0]!.fetchData = retired; await service.shutdown(); if (mode === "revoked") await db .update(chatEndpoints) .set({ allowDirectMessages: false }) .where(eq(chatEndpoints.id, endpoint.id)); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, delivery.id)); const nextRuntime = new FakeChatSdkRuntime(); const replace = nextRuntime.replaceEndpoint.bind(nextRuntime); vi.spyOn(nextRuntime, "replaceEndpoint").mockImplementation( async (options) => { const next = await replace(options); Object.assign(next, { attachmentRecoveryDescriptor: recovered.attachmentRecoveryDescriptor.bind(recovered), rehydrateAttachment: recovered.rehydrateAttachment.bind(recovered), }); const originalThread = next.thread.bind(next); vi.spyOn(next, "thread").mockImplementation((id) => ({ ...originalThread(id), channelId: String(chatId), isDM: !topic, })); return next; }, ); restarted = createService( nextRuntime, fakeTelegramFetch() as typeof globalThis.fetch, { storage: storage.storage, scheduleDeferredWork: () => undefined, }, ); await restarted.service.processPendingDeliveries(25, delivery.id); expect(retired).not.toHaveBeenCalled(); } const finalService = restarted?.service ?? service; const finalWake = restarted?.wakeup ?? wakeup; if (["revoked", "source_edit", "unknown_document"].includes(mode)) { expect(storage.putFile).not.toHaveBeenCalled(); expect(finalWake).not.toHaveBeenCalled(); expect(requests).toEqual( mode === "source_edit" ? ["getFile:rich-document", "download:rich-document"] : [], ); } else if (mode === "unknown_block" || mode === "malformed_block") { const comments = await db .select({ body: issueComments.body }) .from(issueComments) .where(eq(issueComments.companyId, fixture.companyId)); expect(comments[0]?.body).toContain("could not import"); expect(JSON.stringify(comments)).not.toContain( "PRIVATE_UNSUPPORTED_BODY", ); expect(requests).toEqual([]); } else { expect(storage.putFile).toHaveBeenCalledTimes( mode === "mixed_files" ? 2 : 1, ); expect(storage.putFile.mock.calls[0]![0]).toMatchObject({ body: bytes, contentType: "text/plain", }); if (mode === "mixed_files") { expect(storage.putFile.mock.calls[1]![0]).toMatchObject({ body: photo, contentType: "image/jpeg", }); const [comment] = await db .select() .from(issueComments) .where(eq(issueComments.companyId, fixture.companyId)); expect(comment.body).toBe( "Before files\n\nFile section\n\nPhoto caption\n\nQuoted tail\n\nQuote author", ); } expect(finalWake).toHaveBeenCalledOnce(); expect(await finalService.listConversations(endpoint.id)).toEqual([ expect.objectContaining({ externalThreadId: threadId }), ]); const before = [...requests]; await finalService.processPendingDeliveries(25, delivery.id); if (!deferred) await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread, message, trigger: "direct_message", }); expect(requests).toEqual(before); expect(finalWake).toHaveBeenCalledOnce(); } } finally { try { await pinned.shutdown(); } finally { try { await recovered.shutdown(); } finally { try { await retirePublicationFixture( restarted?.service ?? service, endpoint.id, ); } finally { try { await service.shutdown(); } finally { fetchSpy.mockRestore(); } } } } } }, ); it.each(["quote", "mixed", "document"] as const)( "retains current rich content through the pinned parser and service (%s)", async (mode) => { const fixture = await seedCompany(); const storage = createStorageService(); const { service, runtime, endpoint, callbacks, wakeup } = await configuredTelegramEndpoint(fixture, { storage: storage.storage, }); const pinned = createChatSdkEndpointRuntime({ ...runtime.configurations.get(endpoint.id)!, logger: "silent", }); const bytes = Buffer.from( "Exact rich document fixture\nshape=hexagon\n", ); const requests: string[] = []; const fetchSpy = vi .spyOn(globalThis, "fetch") .mockImplementation(async (input, init) => { const url = new URL( input instanceof Request ? input.url : String(input), ); if (url.hostname !== "api.telegram.org") throw new Error("Unexpected fixture host"); if (url.pathname.endsWith("/getFile")) { expect(JSON.parse(String(init?.body))).toEqual({ file_id: "rich-document", }); requests.push("getFile"); return Response.json({ ok: true, result: { file_path: "fixture/rich-document.txt" }, }); } if (!url.pathname.endsWith("/fixture/rich-document.txt")) throw new Error("Unexpected fixture method"); requests.push("download"); return new Response(bytes); }); try { Object.assign(runtime.endpoints.get(endpoint.id)!, { attachmentRecoveryDescriptor: pinned.attachmentRecoveryDescriptor.bind(pinned), rehydrateAttachment: pinned.rehydrateAttachment.bind(pinned), }); const quote = { type: "expandable_blockquote", text: ["Exact quoted ", { type: "bold", text: "tail" }], credit: "Visible credit", }; const blocks = mode === "document" ? [ { type: "document", document: { file_id: "rich-document", file_unique_id: "rich-unique", file_name: "rich-document.txt", mime_type: "text/plain", file_size: bytes.length, }, }, ] : mode === "mixed" ? [ { type: "paragraph", text: "Before" }, quote, { type: "paragraph", text: "After" }, ] : [quote]; const message = pinned.parseTelegramCommandMessage({ message_id: 70, date: Math.floor(Date.now() / 1_000), chat: { id: 77115580, type: "private" }, from: { id: 77115580, is_bot: false, first_name: "Rich fixture" }, rich_message: { blocks }, })!; await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: makeThread({ channelId: "77115580", id: "telegram:77115580", isDM: true, }).thread, message, trigger: "direct_message", }); if (mode === "document") { expect(storage.putFile).toHaveBeenCalledOnce(); expect(storage.putFile.mock.calls[0]![0]).toMatchObject({ body: bytes, contentType: "text/plain", originalFilename: "rich-document.txt", }); expect(requests).toEqual(["getFile", "download"]); } else { const [comment] = await db .select() .from(issueComments) .where(eq(issueComments.companyId, fixture.companyId)); expect(comment?.body).toBe( mode === "mixed" ? "Before\n\nExact quoted tail\n\nVisible credit\n\nAfter" : "Exact quoted tail\n\nVisible credit", ); } expect(wakeup).toHaveBeenCalledOnce(); } finally { try { await pinned.shutdown(); } finally { try { await retirePublicationFixture(service, endpoint.id); } finally { fetchSpy.mockRestore(); } } } }, ); }); describe("Telegram source-bound optional media", () => { it.each([ "topic_restart", "changed_locator", "revoked_before_fetch", "revoked_during_fetch", ] as const)( "retains exact durable media authority across restart (%s)", async (mode) => { const fixture = await seedCompany(); const storage = createStorageService(); const first = await configuredTelegramEndpoint(fixture, { storage: storage.storage, deferWebhookProcessing: true, scheduleDeferredWork: () => undefined, }); const { endpoint, runtime, service, callbacks, wakeup } = first; const configuration = runtime.configurations.get(endpoint.id)!; const pinned = createChatSdkEndpointRuntime({ ...configuration, logger: "silent", }); const recovered = createChatSdkEndpointRuntime({ ...configuration, logger: "silent", }); let restarted: ReturnType | undefined; const topic = mode === "topic_restart"; const chatId = topic ? -10077115579 : 77115579; const threadId = `telegram:${chatId}${topic ? ":42" : ""}`; const requests: string[] = []; const fetchSpy = vi .spyOn(globalThis, "fetch") .mockImplementation(async (input, init) => { const url = new URL( input instanceof Request ? input.url : String(input), ); if (url.hostname !== "api.telegram.org") throw new Error("Unexpected fixture host"); if (url.pathname.endsWith("/getFile")) { expect(JSON.parse(String(init?.body))).toEqual({ file_id: "restart-media", }); requests.push("getFile"); return Response.json({ ok: true, result: { file_path: "fixture/restart.mp4" }, }); } if (!url.pathname.endsWith("/fixture/restart.mp4")) throw new Error("Unexpected fixture method"); requests.push("download"); if (mode === "revoked_during_fetch") await db .update(chatEndpoints) .set({ allowDirectMessages: false }) .where(eq(chatEndpoints.id, endpoint.id)); return new Response(TELEGRAM_VIDEO_NOTE_MP4); }); try { Object.assign(runtime.endpoints.get(endpoint.id)!, { attachmentRecoveryDescriptor: pinned.attachmentRecoveryDescriptor.bind(pinned), rehydrateAttachment: pinned.rehydrateAttachment.bind(pinned), }); if (topic) { await db .update(chatEndpoints) .set({ status: "active" }) .where(eq(chatEndpoints.id, endpoint.id)); await db.insert(chatEndpointResources).values({ companyId: fixture.companyId, endpointId: endpoint.id, type: "chat", providerResourceId: String(chatId), label: "Fixture topic", availability: "available", enabled: true, }); } const message = pinned.parseTelegramCommandMessage({ message_id: 50, date: Math.floor(Date.now() / 1_000), ...(topic ? { message_thread_id: 42 } : {}), chat: { id: chatId, type: topic ? "supergroup" : "private" }, from: { id: 77115579, is_bot: false, first_name: "Restart fixture", }, video: { file_id: "restart-media", file_unique_id: "restart-unique", file_size: TELEGRAM_VIDEO_NOTE_MP4.length, width: 16, height: 16, duration: 1, }, })!; // The runtime's verified mention callback supplies addressing; // attachment bytes/identity still come from the actual pinned parser. if (topic) message.isMention = true; await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: makeThread({ channelId: String(chatId), id: threadId, isDM: !topic, }).thread, message, trigger: topic ? "mention" : "direct_message", }); const [received] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(received).toMatchObject({ state: "received", attempts: 0 }); expect(received.normalizedEvent).toMatchObject({ message: { attachments: [ { recovery: { locator: { kind: "telegram_media", fileId: "restart-media", fileUniqueId: "restart-unique", threadId, messageId: `${chatId}:50`, principalExternalId: "77115579", }, }, }, ], }, }); expect(requests).toEqual([]); expect(wakeup).not.toHaveBeenCalled(); const originalFetch = vi.fn(async () => { throw new Error("Retired closure used"); }); message.attachments[0]!.fetchData = originalFetch; await service.shutdown(); if (mode === "changed_locator") { const normalized = structuredClone(received.normalizedEvent) as { message: { attachments: Array<{ recovery: { locator: { fileId: string } }; }>; }; }; normalized.message.attachments[0]!.recovery.locator.fileId = "unrelated-file"; await db .update(chatDeliveries) .set({ normalizedEvent: normalized }) .where(eq(chatDeliveries.id, received.id)); } if (mode === "revoked_before_fetch") await db .update(chatEndpoints) .set({ allowDirectMessages: false }) .where(eq(chatEndpoints.id, endpoint.id)); await db .update(chatDeliveries) .set({ nextAttemptAt: new Date(0) }) .where(eq(chatDeliveries.id, received.id)); const nextRuntime = new FakeChatSdkRuntime(); const replace = nextRuntime.replaceEndpoint.bind(nextRuntime); vi.spyOn(nextRuntime, "replaceEndpoint").mockImplementation( async (options) => { const next = await replace(options); Object.assign(next, { attachmentRecoveryDescriptor: recovered.attachmentRecoveryDescriptor.bind(recovered), rehydrateAttachment: recovered.rehydrateAttachment.bind(recovered), }); const thread = next.thread.bind(next); vi.spyOn(next, "thread").mockImplementation((id) => ({ ...thread(id), channelId: String(chatId), isDM: !topic, })); return next; }, ); restarted = createService( nextRuntime, fakeTelegramFetch() as typeof globalThis.fetch, { storage: storage.storage, scheduleDeferredWork: () => undefined }, ); await restarted.service.processPendingDeliveries(25, received.id); expect(originalFetch).not.toHaveBeenCalled(); if (topic) { expect(requests).toEqual(["getFile", "download"]); expect(storage.putFile).toHaveBeenCalledOnce(); expect(storage.putFile.mock.calls[0]![0]).toMatchObject({ body: TELEGRAM_VIDEO_NOTE_MP4, contentType: "video/mp4", }); expect(restarted.wakeup).toHaveBeenCalledOnce(); expect( await restarted.service.listConversations(endpoint.id), ).toEqual([ expect.objectContaining({ externalThreadId: threadId }), ]); await restarted.service.processPendingDeliveries(25, received.id); expect(requests).toEqual(["getFile", "download"]); expect(restarted.wakeup).toHaveBeenCalledOnce(); } else { expect(storage.putFile).not.toHaveBeenCalled(); expect(restarted.wakeup).not.toHaveBeenCalled(); expect(requests).toEqual( mode === "revoked_during_fetch" ? ["getFile", "download"] : [], ); if (mode === "changed_locator") { const [comment] = await db .select() .from(issueComments) .where(eq(issueComments.companyId, fixture.companyId)); expect(comment!.body).toContain("could not safely import"); } } } finally { try { await pinned.shutdown(); } finally { try { await recovered.shutdown(); } finally { try { await retirePublicationFixture( restarted?.service ?? service, endpoint.id, ); } finally { try { await service.shutdown(); } finally { fetchSpy.mockRestore(); } } } } } }, ); it.each([ "video", "voice", "live_photo", "oversize", "unknown_document", "malformed_video", "malformed_live_photo", "malformed_bytes", "office_named_video", "office_named_invalid", ] as const)( "imports exact permitted bytes or gives accurate recovery advice (%s)", async (mode) => { const fixture = await seedCompany(); const storage = createStorageService(); const context = await configuredTelegramEndpoint(fixture, { storage: storage.storage, }); const { service, runtime, endpoint, callbacks, wakeup } = context; const pinned = createChatSdkEndpointRuntime({ ...runtime.configurations.get(endpoint.id)!, logger: "silent", }); const photo = await sharp({ create: { width: 16, height: 16, channels: 3, background: "teal" }, }) .jpeg() .toBuffer(); const bytes = mode === "voice" ? TELEGRAM_VOICE_OGG : mode === "malformed_bytes" || mode === "office_named_invalid" ? TELEGRAM_VIDEO_NOTE_MP4.subarray(0, -1) : TELEGRAM_VIDEO_NOTE_MP4; const requests: string[] = []; const fetchSpy = vi .spyOn(globalThis, "fetch") .mockImplementation(async (input, init) => { const url = new URL( input instanceof Request ? input.url : String(input), ); if (url.hostname !== "api.telegram.org") throw new Error("Unexpected fixture host"); if (url.pathname.endsWith("/getFile")) { const fileId = JSON.parse(String(init?.body)).file_id as string; expect(["optional-media", "optional-photo"]).toContain(fileId); requests.push(`getFile:${fileId}`); return Response.json({ ok: true, result: { file_path: `fixture/${fileId}` }, }); } const file = url.pathname.split("/").at(-1)!; if (!["optional-media", "optional-photo"].includes(file)) throw new Error("Unexpected fixture method"); requests.push(`download:${file}`); return new Response(file === "optional-photo" ? photo : bytes); }); try { Object.assign(runtime.endpoints.get(endpoint.id)!, { attachmentRecoveryDescriptor: pinned.attachmentRecoveryDescriptor.bind(pinned), rehydrateAttachment: pinned.rehydrateAttachment.bind(pinned), }); const media = { file_id: "optional-media", file_unique_id: "optional-media-unique", file_size: mode === "oversize" ? MAX_ATTACHMENT_BYTES + 1 : bytes.length, duration: mode === "malformed_video" || mode === "malformed_live_photo" ? -1 : 1, ...(mode.startsWith("office_named") ? { file_name: "recording.docx" } : {}), }; const raw = { message_id: 41, date: Math.floor(Date.now() / 1_000), chat: { id: 77115579, type: "private" }, from: { id: 77115579, is_bot: false, first_name: "Optional media fixture", }, ...(mode === "unknown_document" ? { document: media } : mode === "voice" ? { voice: media } : mode === "live_photo" || mode === "malformed_live_photo" ? { live_photo: { ...media, width: 16, height: 16, photo: [ { file_id: "optional-photo", file_unique_id: "optional-photo-unique", file_size: photo.length, width: 16, height: 16, }, ], }, } : { video: { ...media, width: 16, height: 16 } }), }; const message = pinned.parseTelegramCommandMessage(raw)!; await deliverMessage({ callbacks, endpointId: endpoint.id, provider: "telegram", thread: makeThread({ channelId: "77115579", id: "telegram:77115579", isDM: true, }).thread, message, trigger: "direct_message", }); if ( [ "oversize", "unknown_document", "malformed_video", "malformed_live_photo", "malformed_bytes", "office_named_invalid", ].includes(mode) ) { expect(storage.putFile).not.toHaveBeenCalled(); expect(requests).toEqual( mode === "malformed_bytes" || mode === "office_named_invalid" ? ["getFile:optional-media", "download:optional-media"] : [], ); const [comment] = await db .select() .from(issueComments) .where(eq(issueComments.companyId, fixture.companyId)); expect(comment!.body).toContain( `under ${formatAttachmentSize(MAX_ATTACHMENT_BYTES)}`, ); expect(wakeup).not.toHaveBeenCalled(); return; } expect(storage.putFile).toHaveBeenCalledTimes( mode === "live_photo" ? 2 : 1, ); expect( storage.putFile.mock.calls.some( ([input]) => input.body.equals(bytes) && input.contentType === (mode === "voice" ? "audio/ogg" : "video/mp4"), ), ).toBe(true); if (mode === "live_photo") expect( storage.putFile.mock.calls.some( ([input]) => input.body.equals(photo) && input.contentType === "image/jpeg", ), ).toBe(true); expect(wakeup).toHaveBeenCalledOnce(); const [delivery] = await db .select() .from(chatDeliveries) .where(eq(chatDeliveries.endpointId, endpoint.id)); expect(delivery).toMatchObject({ state: "processed", redactedError: null, }); // Replay this exact source, not unrelated admission retries left in // the shared fixture DB by tests exercising failed scheduler calls. await service.processPendingDeliveries(25, delivery.id); expect(requests).toHaveLength(mode === "live_photo" ? 4 : 2); expect(wakeup).toHaveBeenCalledOnce(); expect(wakeup.mock.calls[0]![0]).toBe(fixture.assignedAgentId); await expect( db .select({ id: agentWakeupRequests.id }) .from(agentWakeupRequests) .innerJoin( chatActions, eq(chatActions.id, agentWakeupRequests.id), ) .where( and( eq(chatActions.deliveryId, delivery.id), eq(agentWakeupRequests.companyId, fixture.companyId), eq(agentWakeupRequests.agentId, fixture.assignedAgentId), ), ), ).resolves.toHaveLength(1); } finally { try { await pinned.shutdown(); } finally { try { await retirePublicationFixture(service, endpoint.id); } finally { fetchSpy.mockRestore(); } } } }, ); }); });