export type ChatPublicationErrorDisposition = | { kind: "retry"; retryAfterMs: number; /** True when the provider explicitly asked Paperclip to slow down. */ providerRateLimit?: boolean; reason: string; } | { kind: "delivery_unknown"; reason: string; } | { kind: "endpoint_attention"; reason: string; } | { kind: "resource_unavailable"; reason: string; } | { kind: "failed"; reason: string; }; function finitePositive(value: unknown): number | null { return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null; } function text(error: unknown): string { return error instanceof Error ? error.message : String(error); } function responseHeader( headers: Headers | Record | undefined, name: string, ): string | null { if (!headers) return null; if (headers instanceof Headers) return headers.get(name); const target = name.toLowerCase(); const entry = Object.entries(headers).find( ([key]) => key.toLowerCase() === target, ); const value = entry?.[1]; return typeof value === "string" || typeof value === "number" ? String(value) : null; } function positiveNumber(value: string | null): number | null { if (!value) return null; const parsed = Number(value); return Number.isFinite(parsed) && parsed > 0 ? parsed : null; } // Node timers accept delays through 2^31-1 milliseconds. Keep the durable // provider hint intact up to that boundary instead of turning an hour-long // flood-control response into a rapid series of fifteen-minute retries. const MAX_PROVIDER_RETRY_AFTER_MS = 2_147_000_000; function retryAfterHeaderMilliseconds(value: string | null): number | null { if (!value) return null; const seconds = positiveNumber(value); if (seconds !== null) return seconds * 1000; const absolute = Date.parse(value); if (!Number.isFinite(absolute)) return null; return Math.max(1, absolute - Date.now()); } type StructuredProviderError = { name?: unknown; adapter?: unknown; code?: unknown; retryAfter?: unknown; retryAfterMs?: unknown; retry_after?: unknown; status?: unknown; statusCode?: unknown; subCode?: unknown; providerCodes?: unknown; data?: { error?: unknown }; response?: { status?: unknown; headers?: Headers | Record; }; cause?: unknown; original?: unknown; originalError?: unknown; details?: { code?: unknown; providerStatus?: unknown; providerSubCode?: unknown; providerCodes?: unknown; }; innerHttpError?: { statusCode?: unknown }; }; /** * Adapters sometimes wrap the provider SDK error before it reaches the * durable outbox. Follow only the documented structured wrapper properties, * with a small depth and cycle guard, so provider codes survive without ever * classifying on free-form error text. */ function structuredProviderErrors(error: unknown): StructuredProviderError[] { const records: StructuredProviderError[] = []; const pending: Array<{ value: unknown; depth: number }> = [ { value: error, depth: 0 }, ]; const seen = new Set(); while (pending.length > 0) { const current = pending.shift(); if ( !current || current.depth > 4 || !current.value || typeof current.value !== "object" || seen.has(current.value) ) { continue; } seen.add(current.value); const record = current.value as StructuredProviderError; records.push(record); for (const nested of [ record.cause, record.original, record.originalError, ]) { pending.push({ value: nested, depth: current.depth + 1 }); } } return records; } /** * Classify a provider send failure by whether an external side effect could * already have happened. Only ambiguous network failures stop automatic * retry. Explicit provider rejections are safe to retry or repair without * risking a duplicate message. */ export function classifyChatPublicationError( error: unknown, attempt: number, ): ChatPublicationErrorDisposition { const values = structuredProviderErrors(error); const firstNumber = (select: (value: StructuredProviderError) => unknown) => values .map(select) .find((candidate): candidate is number => typeof candidate === "number"); const names = values .map((value) => value.name) .filter((candidate): candidate is string => typeof candidate === "string"); const adapters = values .map((value) => value.adapter) .filter((candidate): candidate is string => typeof candidate === "string") .map((candidate) => candidate.toLowerCase()); const codes = values .map((value) => value.code) .filter((candidate): candidate is string => typeof candidate === "string"); const platformCodes = values .map((value) => value.data?.error) .filter((candidate): candidate is string => typeof candidate === "string"); const detailsCodes = values .map((value) => value.details?.code) .filter((candidate): candidate is string => typeof candidate === "string"); const discordProviderCode = values .map((value) => value.code) .find( (candidate): candidate is number => typeof candidate === "number", ) ?? null; const status = values .flatMap((value) => [ value.status, value.statusCode, value.response?.status, value.innerHttpError?.statusCode, value.details?.providerStatus, ]) .find((candidate): candidate is number => typeof candidate === "number"); const teamsProviderCodes = values .flatMap((value) => [ value.subCode, value.details?.providerSubCode, ...(Array.isArray(value.providerCodes) ? value.providerCodes : []), ...(Array.isArray(value.details?.providerCodes) ? value.details.providerCodes : []), ]) .filter((candidate): candidate is string => typeof candidate === "string") .map((candidate) => candidate.toLowerCase()); const reason = text(error); if (names.includes("PhotonError")) { if (codes.includes("delivery_unknown")) return { kind: "delivery_unknown", reason }; if (codes.includes("credentials") || codes.includes("line_unavailable") || codes.includes("history_gap")) { return { kind: "endpoint_attention", reason }; } if (codes.includes("quota") || codes.includes("network") || codes.includes("attachment_not_ready")) { return { kind: "retry", retryAfterMs: values .map((value) => finitePositive(value.retryAfterMs)) .find((value) => value !== null) ?? Math.min(60_000, 1000 * 2 ** Math.min(attempt, 6)), providerRateLimit: codes.includes("quota"), reason, }; } return { kind: "failed", reason }; } const responseHeaders = values .map((value) => value.response?.headers) .find((headers) => headers !== undefined); const retryAfterHeaderMs = retryAfterHeaderMilliseconds( responseHeader(responseHeaders, "retry-after"), ); const rateLimitRemaining = responseHeader( responseHeaders, "x-ratelimit-remaining", ); const rateLimitReset = positiveNumber( responseHeader(responseHeaders, "x-ratelimit-reset"), ); const githubRateLimit = status === 403 && (retryAfterHeaderMs !== null || rateLimitRemaining === "0" || reason.toLowerCase().includes("secondary rate limit") || reason.toLowerCase().includes("rate limit exceeded")); // Endpoint management owns the same lease as provider transport. Losing a // short contention race is a definite local pre-transport outcome, so it is // safe to retry automatically and must never be presented as an ambiguous // provider delivery. if (detailsCodes.includes("chat_endpoint_credentials_busy")) { return { kind: "retry", retryAfterMs: 1_000, reason }; } if ( names.includes("AdapterRateLimitError") || names.includes("RateLimitError") || codes.includes("RATE_LIMITED") || codes.includes("slack_webapi_rate_limited_error") || platformCodes.includes("ratelimited") || status === 429 || githubRateLimit ) { const seconds = finitePositive(firstNumber((value) => value.retryAfter)); const milliseconds = finitePositive( firstNumber((value) => value.retryAfterMs), ); const telegramSeconds = finitePositive( firstNumber((value) => value.retry_after), ); const resetMilliseconds = rateLimitReset ? Math.max(1_000, rateLimitReset * 1_000 - Date.now()) : null; const structuredSeconds = seconds ?? telegramSeconds; return { kind: "retry", retryAfterMs: Math.min( MAX_PROVIDER_RETRY_AFTER_MS, milliseconds ?? (structuredSeconds !== null ? structuredSeconds * 1000 : retryAfterHeaderMs !== null ? retryAfterHeaderMs : (resetMilliseconds ?? 2 ** Math.max(0, attempt) * 1000)), ), providerRateLimit: true, reason, }; } if ( // Discord distinguishes a destination the bot cannot access (50001) or // cannot write to (50013) from invalid credentials and app-wide failures. // The pinned adapter preserves the bounded numeric API code on its nested // DiscordApiError. Quarantine only the affected channel/conversation; // generic Discord 403s and every 401 remain endpoint-scoped below. adapters.includes("discord") && status === 403 && (discordProviderCode === 50001 || discordProviderCode === 50013) ) { return { kind: "resource_unavailable", reason }; } if ( // Telegram uses 403 for destination-local conditions such as a user // blocking the bot or removing it from a chat. Its adapter deliberately // represents those responses as PermissionError while keeping an invalid // bot token as AuthenticationError (401). Quarantine only that // conversation/resource; the same bot may still serve every other chat. adapters.includes("telegram") && names.includes("PermissionError") && codes.includes("PERMISSION_DENIED") ) { return { kind: "resource_unavailable", reason }; } if ( // A Teams app can remain healthy while Microsoft rejects writes to one // conversation after the bot is blocked, uninstalled, or loses access. // The pinned adapter preserves only bounded provider code tokens from the // 403 response body; never use the free-form message for this distinction. adapters.includes("teams") && names.includes("PermissionError") && codes.includes("PERMISSION_DENIED") && status === 403 && teamsProviderCodes.some((providerCode) => ["messagewritesblocked", "forbiddenoperationexception"].includes( providerCode, ), ) ) { return { kind: "resource_unavailable", reason }; } if ( names.includes("AuthenticationError") || names.includes("PermissionError") || codes.includes("AUTH_FAILED") || codes.includes("PERMISSION_DENIED") || status === 401 || status === 403 || [ "account_inactive", "invalid_auth", "missing_scope", "not_authed", "no_permission", "token_revoked", ].some((platformCode) => platformCodes.includes(platformCode)) ) { return { kind: "endpoint_attention", reason }; } if ( names.includes("ResourceNotFoundError") || codes.includes("NOT_FOUND") || status === 404 || status === 410 || [ "channel_not_found", "channel_is_archived", "is_archived", "message_not_found", "not_in_channel", "thread_not_found", ].some((platformCode) => platformCodes.includes(platformCode)) || (names.includes("NetworkError") && reason.toLowerCase().includes("resource not found during")) ) { return { kind: "resource_unavailable", reason }; } if ( names.includes("ValidationError") || names.includes("NotImplementedError") || codes.includes("VALIDATION_ERROR") || codes.includes("NOT_IMPLEMENTED") || // Paperclip rejected the destination locally before opening a provider // request, so delivery is definitively impossible rather than ambiguous. codes.includes("CHAT_PROVIDER_PRETRANSPORT_REJECTED") || // Adapter-contract drift is detected during runtime construction, before // any provider request can have been attempted. codes.includes("CHAT_ADAPTER_COMPATIBILITY_ERROR") ) { return { kind: "failed", reason }; } // Slack WebClient platform errors represent a completed, structured API // rejection even though the SDK does not expose an HTTP status. Unknown // platform codes are therefore definite failures, not ambiguous delivery. if (codes.includes("slack_webapi_platform_error")) { return { kind: "failed", reason }; } // Any remaining 4xx response is a definite provider rejection: the // provider returned an HTTP response and did not accept the operation. // Keep it out of delivery_unknown, which is reserved for requests whose // external side effect cannot be determined. Provider-specific repairable // cases (rate limits, auth, and missing destinations) were handled above. if (status !== undefined && status >= 400 && status < 500) { return { kind: "failed", reason }; } // Adapter NetworkError and ordinary fetch/transport errors are ambiguous: // the request may have reached the provider even when no response reached // Paperclip. An operator must inspect the native conversation before replay. return { kind: "delivery_unknown", reason }; }