import { randomUUID } from "node:crypto"; import { and, asc, eq, inArray, isNull, lte, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { companies, documents, externalObjectMentions, externalObjects, issueComments, issueDocuments, issues, plugins } from "@paperclipai/db"; import { formatExternalObjectMentionSourceLabel, type ExternalObjectCanonicalUrl, type ExternalObjectLivenessState, type ExternalObjectMentionConfidence, type ExternalObjectMentionSourceKind, type ExternalObjectStatusCategory, type ExternalObjectStatusTone, type PaperclipPluginManifestV1, } from "@paperclipai/shared"; import { extractExternalObjectCanonicalUrls } from "@paperclipai/shared/external-objects-server"; import type { PluginExternalObjectRecordSnapshot, PluginExternalObjectResolveResult } from "@paperclipai/plugin-sdk"; import { notFound } from "../errors.js"; import { logger } from "../middleware/logger.js"; import { logActivity, type LogActivityInput } from "./activity-log.js"; import { createGitHubExternalObjectProvider, type GitHubExternalObjectProviderOptions } from "./github-external-object-provider.js"; import { publishLiveEvent } from "./live-events.js"; import type { PluginWorkerManager } from "./plugin-worker-manager.js"; export interface ExternalObjectSourceContext { companyId: string; sourceIssueId: string; sourceKind: ExternalObjectMentionSourceKind; sourceRecordId: string | null; documentKey: string | null; propertyKey: string | null; } export interface ExternalObjectDetection { canonical: ExternalObjectCanonicalUrl; detectorKey: string; providerKey: string; objectType: string; externalId: string; displayKey?: string | null; iconKey?: string | null; displayTitle?: string | null; confidence?: ExternalObjectMentionConfidence; pluginId?: string | null; } export interface ExternalObjectDetector { key: string; detect(input: { companyId: string; urls: ExternalObjectCanonicalUrl[]; sourceContext: ExternalObjectSourceContext; }): Promise | ExternalObjectDetection[]; } export interface ExternalObjectResolverSnapshot { displayKey?: string | null; iconKey?: string | null; displayTitle?: string | null; statusKey?: string | null; statusLabel?: string | null; statusIconKey?: string | null; statusCategory: ExternalObjectStatusCategory; statusTone: ExternalObjectStatusTone; isTerminal?: boolean; data?: Record; remoteVersion?: string | null; etag?: string | null; ttlSeconds?: number; } export type ExternalObjectResolveResult = | { ok: true; snapshot: ExternalObjectResolverSnapshot } | { ok: false; liveness: Extract; errorCode: string; errorMessage?: string | null; retryAfterSeconds?: number; }; export interface ExternalObjectResolver { providerKey: string; objectType?: string; resolve(input: { companyId: string; object: ExternalObjectRecord; }): Promise; } type ExternalObjectRecord = typeof externalObjects.$inferSelect; type ExternalObjectMentionRecord = typeof externalObjectMentions.$inferSelect; const DEFAULT_REFRESH_TTL_SECONDS = 300; const DEFAULT_RETRY_AFTER_SECONDS = 300; const DEFAULT_REFRESH_LEASE_SECONDS = 300; const REFRESH_LEASE_RENEW_INTERVAL_MS = 60_000; function sourceWhere(input: ExternalObjectSourceContext) { const conditions = [ eq(externalObjectMentions.companyId, input.companyId), eq(externalObjectMentions.sourceIssueId, input.sourceIssueId), eq(externalObjectMentions.sourceKind, input.sourceKind), ]; if (input.sourceRecordId) { conditions.push(eq(externalObjectMentions.sourceRecordId, input.sourceRecordId)); } else { conditions.push(isNull(externalObjectMentions.sourceRecordId)); } if (input.documentKey) { conditions.push(eq(externalObjectMentions.documentKey, input.documentKey)); } else { conditions.push(isNull(externalObjectMentions.documentKey)); } if (input.propertyKey) { conditions.push(eq(externalObjectMentions.propertyKey, input.propertyKey)); } else { conditions.push(isNull(externalObjectMentions.propertyKey)); } return and(...conditions); } function addSeconds(date: Date, seconds: number) { return new Date(date.getTime() + Math.max(1, seconds) * 1000); } function visibleLiveness(object: ExternalObjectRecord, now = new Date()): ExternalObjectLivenessState { if (object.liveness === "fresh" && object.nextRefreshAt && object.nextRefreshAt <= now) { return "stale"; } return object.liveness; } function objectChanged(before: ExternalObjectRecord, after: ExternalObjectRecord) { return ( before.statusKey !== after.statusKey || before.statusLabel !== after.statusLabel || before.statusIconKey !== after.statusIconKey || before.statusCategory !== after.statusCategory || before.statusTone !== after.statusTone || before.isTerminal !== after.isTerminal ); } function sanitizeErrorMessage(message: string | null | undefined) { if (!message) return null; return message .replace(/https?:\/\/[^\s<>()]+/gi, "[redacted-url]") .replace(/\b(token|key|secret|authorization|bearer)=\S+/gi, "$1=[redacted]"); } function genericUrlDetector(): ExternalObjectDetector { return { key: "generic-url", detect({ urls }) { return urls.map((canonical) => ({ canonical, detectorKey: "generic-url", providerKey: "url", objectType: "link", externalId: canonical.canonicalIdentityHash, displayTitle: canonical.sanitizedDisplayUrl, confidence: "possible", })); }, }; } export function createExternalObjectDetectorRegistry(detectors: ExternalObjectDetector[] = []) { const entries = [...detectors, genericUrlDetector()]; async function detect(input: { companyId: string; urls: ExternalObjectCanonicalUrl[]; sourceContext: ExternalObjectSourceContext; }) { const claimed = new Set(); const detections: ExternalObjectDetection[] = []; for (const detector of entries) { const remaining = input.urls.filter((url) => !claimed.has(url.canonicalIdentityHash)); if (remaining.length === 0) break; try { const detected = await detector.detect({ ...input, urls: remaining }); for (const detection of detected) { if (claimed.has(detection.canonical.canonicalIdentityHash)) continue; claimed.add(detection.canonical.canonicalIdentityHash); detections.push({ ...detection, detectorKey: detection.detectorKey || detector.key }); } } catch (err) { logger.warn({ err, detectorKey: detector.key }, "external object detector failed"); } } return detections; } return { detect }; } export function createExternalObjectResolverRegistry(resolvers: ExternalObjectResolver[] = []) { function find(object: Pick) { return resolvers.find( (resolver) => resolver.providerKey === object.providerKey && (!resolver.objectType || resolver.objectType === object.objectType), ) ?? null; } return { find }; } function manifestProvidesObject( manifest: PaperclipPluginManifestV1, object: Pick, ) { return (manifest.objectReferences ?? []).some( (provider) => provider.providerKey === object.providerKey && provider.objectTypes.includes(object.objectType), ); } function objectSnapshot(object: ExternalObjectRecord): PluginExternalObjectRecordSnapshot { return { id: object.id, companyId: object.companyId, providerKey: object.providerKey, objectType: object.objectType, externalId: object.externalId, sanitizedCanonicalUrl: object.sanitizedCanonicalUrl, canonicalIdentityHash: object.canonicalIdentityHash, displayKey: object.displayKey, iconKey: object.iconKey, displayTitle: object.displayTitle, statusKey: object.statusKey, statusLabel: object.statusLabel, statusIconKey: object.statusIconKey, statusCategory: object.statusCategory, statusTone: object.statusTone, liveness: object.liveness, isTerminal: object.isTerminal, data: object.data as Record, remoteVersion: object.remoteVersion, etag: object.etag, }; } async function readyObjectReferencePlugins(db: Db) { return db .select({ id: plugins.id, pluginKey: plugins.pluginKey, manifestJson: plugins.manifestJson, }) .from(plugins) .where(eq(plugins.status, "ready")) .orderBy(asc(plugins.installOrder)) .then((rows) => rows.filter((row) => (row.manifestJson.objectReferences?.length ?? 0) > 0), ); } function createPluginProviderDetector( db: Db, pluginWorkerManager: PluginWorkerManager, ): ExternalObjectDetector { return { key: "plugin-object-reference-providers", async detect(input) { const providers = await readyObjectReferencePlugins(db); const detections: ExternalObjectDetection[] = []; for (const provider of providers) { const manifest = provider.manifestJson; if (!manifest.capabilities.includes("external.objects.detect")) continue; try { const result = await pluginWorkerManager.call(provider.id, "detectExternalObjects", { companyId: input.companyId, urls: input.urls.map((url) => ({ sanitizedCanonicalUrl: url.sanitizedCanonicalUrl, sanitizedDisplayUrl: url.sanitizedDisplayUrl, canonicalIdentityHash: url.canonicalIdentityHash, canonicalIdentity: url.canonicalIdentity as unknown as Record, redactedMatchedText: url.redactedMatchedText, })), sourceContext: input.sourceContext, }); const urlsByHash = new Map(input.urls.map((url) => [url.canonicalIdentityHash, url])); const declaredProviderKeys = new Set((manifest.objectReferences ?? []).map((entry) => entry.providerKey)); for (const detection of result.detections ?? []) { const canonical = urlsByHash.get(detection.urlIdentityHash); if (!canonical) continue; if (!declaredProviderKeys.has(detection.providerKey)) continue; const declaration = (manifest.objectReferences ?? []).find((entry) => entry.providerKey === detection.providerKey); if (!declaration?.objectTypes.includes(detection.objectType)) continue; detections.push({ canonical, detectorKey: `${provider.pluginKey}:${detection.providerKey}`, providerKey: detection.providerKey, objectType: detection.objectType, externalId: detection.externalId, displayKey: detection.displayKey, iconKey: detection.iconKey, displayTitle: detection.displayTitle, confidence: detection.confidence ?? "exact", pluginId: provider.id, }); } } catch (err) { logger.warn( { err, pluginId: provider.id, pluginKey: provider.pluginKey }, "plugin external object detector failed", ); } } return detections; }, }; } async function resolveViaPluginProvider( db: Db, pluginWorkerManager: PluginWorkerManager | undefined, object: ExternalObjectRecord, ): Promise { if (!pluginWorkerManager) return null; const providers = await readyObjectReferencePlugins(db); for (const provider of providers) { const manifest = provider.manifestJson; if (!manifest.capabilities.includes("external.objects.read")) continue; if (!manifestProvidesObject(manifest, object)) continue; try { return await pluginWorkerManager.call(provider.id, "resolveExternalObject", { companyId: object.companyId, providerKey: object.providerKey, objectType: object.objectType, externalId: object.externalId, object: objectSnapshot(object), }); } catch (err) { logger.warn( { err, pluginId: provider.id, pluginKey: provider.pluginKey, objectId: object.id }, "plugin external object resolver failed", ); return { ok: false, liveness: "unreachable", errorCode: "plugin_resolver_failed", errorMessage: err instanceof Error ? err.message : String(err), }; } } return null; } export function externalObjectService( db: Db, opts: { detectors?: ExternalObjectDetector[]; resolvers?: ExternalObjectResolver[]; pluginWorkerManager?: PluginWorkerManager; github?: GitHubExternalObjectProviderOptions | false; enabled?: boolean | (() => boolean | Promise); } = {}, ) { const githubProvider = opts.github === false ? null : createGitHubExternalObjectProvider(db, opts.github); const pluginProviderDetector = opts.pluginWorkerManager ? createPluginProviderDetector(db, opts.pluginWorkerManager) : null; const detectorRegistry = createExternalObjectDetectorRegistry([ ...(pluginProviderDetector ? [pluginProviderDetector] : []), ...(opts.detectors ?? []), ...(githubProvider ? [githubProvider.detector] : []), ]); const resolverRegistry = createExternalObjectResolverRegistry([ ...(opts.resolvers ?? []), ...(githubProvider?.resolvers ?? []), ]); async function isEnabled() { if (typeof opts.enabled === "function") return await opts.enabled(); return opts.enabled ?? true; } function emptySummary() { return summarizeObjectPayloads([]); } async function issueById(issueId: string, dbOrTx: any = db) { return dbOrTx .select({ id: issues.id, companyId: issues.companyId, title: issues.title, description: issues.description, }) .from(issues) .where(eq(issues.id, issueId)) .then((rows: Array<{ id: string; companyId: string; title: string; description: string | null }>) => rows[0] ?? null); } async function upsertObjectFromDetection( companyId: string, detection: ExternalObjectDetection, dbOrTx: any, ): Promise { const now = new Date(); const canonical = detection.canonical; const values = { companyId, providerKey: detection.providerKey, pluginId: detection.pluginId ?? null, objectType: detection.objectType, externalId: detection.externalId, sanitizedCanonicalUrl: canonical.sanitizedCanonicalUrl, canonicalIdentityHash: canonical.canonicalIdentityHash, displayKey: detection.displayKey ?? null, iconKey: detection.iconKey ?? null, displayTitle: detection.displayTitle ?? canonical.sanitizedDisplayUrl, nextRefreshAt: now, updatedAt: now, }; const inserted = await dbOrTx .insert(externalObjects) .values(values) .onConflictDoUpdate({ target: [ externalObjects.companyId, externalObjects.providerKey, externalObjects.objectType, externalObjects.externalId, ], set: { sanitizedCanonicalUrl: values.sanitizedCanonicalUrl, canonicalIdentityHash: values.canonicalIdentityHash, displayKey: values.displayKey, iconKey: values.iconKey, displayTitle: values.displayTitle, nextRefreshAt: sql`coalesce(${externalObjects.nextRefreshAt}, now())`, updatedAt: now, }, }) .returning(); return inserted[0]!; } async function replaceSourceMentions( input: ExternalObjectSourceContext & { text: string | null | undefined }, dbOrTx: any = db, ) { const urls = extractExternalObjectCanonicalUrls(input.text ?? ""); await dbOrTx.delete(externalObjectMentions).where(sourceWhere(input)); if (urls.length === 0) return; const detections = await detectorRegistry.detect({ companyId: input.companyId, urls, sourceContext: input, }); if (detections.length === 0) return; const seen = new Set(); const values: Array = []; for (const detection of detections) { const canonicalHash = detection.canonical.canonicalIdentityHash; const sourceKey = `${detection.providerKey}:${detection.objectType}:${canonicalHash}`; if (seen.has(sourceKey)) continue; seen.add(sourceKey); const object = await upsertObjectFromDetection(input.companyId, detection, dbOrTx); values.push({ companyId: input.companyId, sourceIssueId: input.sourceIssueId, sourceKind: input.sourceKind, sourceRecordId: input.sourceRecordId, documentKey: input.documentKey, propertyKey: input.propertyKey, matchedTextRedacted: detection.canonical.redactedMatchedText, sanitizedDisplayUrl: detection.canonical.sanitizedDisplayUrl, canonicalIdentityHash: canonicalHash, canonicalIdentity: detection.canonical.canonicalIdentity as unknown as Record, objectId: object.id, providerKey: detection.providerKey, detectorKey: detection.detectorKey, objectType: detection.objectType, confidence: detection.confidence ?? "exact", createdByPluginId: detection.pluginId ?? null, }); } if (values.length > 0) { await dbOrTx.insert(externalObjectMentions).values(values); } } async function syncIssue(issueId: string, dbOrTx: any = db) { if (!(await isEnabled())) return; const runSync = async (tx: any) => { const issue = await issueById(issueId, tx); if (!issue) throw notFound("Issue not found"); await replaceSourceMentions({ companyId: issue.companyId, sourceIssueId: issue.id, sourceKind: "title", sourceRecordId: null, documentKey: null, propertyKey: null, text: issue.title, }, tx); await replaceSourceMentions({ companyId: issue.companyId, sourceIssueId: issue.id, sourceKind: "description", sourceRecordId: null, documentKey: null, propertyKey: null, text: issue.description, }, tx); }; return dbOrTx === db ? db.transaction(runSync) : runSync(dbOrTx); } async function syncComment(commentId: string, dbOrTx: any = db) { const runSync = async (tx: any) => { if (!(await isEnabled())) return; const comment = await tx .select({ id: issueComments.id, companyId: issueComments.companyId, issueId: issueComments.issueId, body: issueComments.body, }) .from(issueComments) .where(eq(issueComments.id, commentId)) .then((rows: Array<{ id: string; companyId: string; issueId: string; body: string }>) => rows[0] ?? null); if (!comment) { await tx .delete(externalObjectMentions) .where(and(eq(externalObjectMentions.sourceKind, "comment"), eq(externalObjectMentions.sourceRecordId, commentId))); return; } await replaceSourceMentions({ companyId: comment.companyId, sourceIssueId: comment.issueId, sourceKind: "comment", sourceRecordId: comment.id, documentKey: null, propertyKey: null, text: comment.body, }, tx); }; return dbOrTx === db ? db.transaction(runSync) : runSync(dbOrTx); } async function syncDocument(documentId: string, dbOrTx: any = db) { const runSync = async (tx: any) => { if (!(await isEnabled())) return; const document = await tx .select({ documentId: documents.id, companyId: documents.companyId, issueId: issueDocuments.issueId, key: issueDocuments.key, body: documents.latestBody, }) .from(issueDocuments) .innerJoin(documents, eq(issueDocuments.documentId, documents.id)) .where(eq(documents.id, documentId)) .then((rows: Array<{ documentId: string; companyId: string; issueId: string; key: string; body: string }>) => rows[0] ?? null); if (!document) { await tx .delete(externalObjectMentions) .where(and(eq(externalObjectMentions.sourceKind, "document"), eq(externalObjectMentions.sourceRecordId, documentId))); return; } await replaceSourceMentions({ companyId: document.companyId, sourceIssueId: document.issueId, sourceKind: "document", sourceRecordId: document.documentId, documentKey: document.key, propertyKey: null, text: document.body, }, tx); }; return dbOrTx === db ? db.transaction(runSync) : runSync(dbOrTx); } async function safeSync(label: string, fn: () => Promise) { try { await fn(); } catch (err) { logger.warn({ err }, `external object ${label} sync failed`); } } async function syncIssueSafely(issueId: string) { await safeSync("issue", () => syncIssue(issueId)); } async function syncCommentSafely(commentId: string, dbOrTx: any = db) { await safeSync("comment", () => syncComment(commentId, dbOrTx)); } async function syncDocumentSafely(documentId: string) { await safeSync("document", () => syncDocument(documentId)); } function toObjectPayload(object: ExternalObjectRecord, now = new Date()) { const { refreshToken, ...payload } = object; void refreshToken; return { ...payload, liveness: visibleLiveness(object, now), }; } async function listForIssue(issueId: string) { if (!(await isEnabled())) return []; const issue = await issueById(issueId); if (!issue) throw notFound("Issue not found"); const rows = await db .select({ mention: externalObjectMentions, object: externalObjects, }) .from(externalObjectMentions) .leftJoin(externalObjects, eq(externalObjectMentions.objectId, externalObjects.id)) .where(and( eq(externalObjectMentions.companyId, issue.companyId), eq(externalObjectMentions.sourceIssueId, issue.id), )) .orderBy(asc(externalObjectMentions.sourceKind), asc(externalObjectMentions.createdAt)); const now = new Date(); const grouped = new Map | null; mentions: ExternalObjectMentionRecord[]; mentionCount: number; sourceLabels: string[]; }>(); for (const row of rows) { const key = row.object?.id ?? `mention:${row.mention.id}`; const existing = grouped.get(key) ?? { object: row.object ? toObjectPayload(row.object, now) : null, mentions: [], mentionCount: 0, sourceLabels: [], }; existing.mentions.push(row.mention); existing.mentionCount += 1; const label = formatExternalObjectMentionSourceLabel({ sourceKind: row.mention.sourceKind, documentKey: row.mention.documentKey, propertyKey: row.mention.propertyKey, }); if (!existing.sourceLabels.includes(label)) existing.sourceLabels.push(label); grouped.set(key, existing); } return [...grouped.values()]; } function summarizeObjects(objects: Array>) { const byStatusCategory: Record = {}; const byLiveness: Record = {}; let highestSeverity: ExternalObjectStatusTone = "neutral"; const severityRank: Record = { neutral: 0, muted: 0, success: 1, info: 2, warning: 3, danger: 4, }; for (const object of objects) { byStatusCategory[object.statusCategory] = (byStatusCategory[object.statusCategory] ?? 0) + 1; byLiveness[object.liveness] = (byLiveness[object.liveness] ?? 0) + 1; const livenessTone = object.liveness === "auth_required" || object.liveness === "unreachable" ? "danger" : object.liveness === "stale" ? "warning" : object.statusTone; if (severityRank[livenessTone] > severityRank[highestSeverity]) highestSeverity = livenessTone; } return { total: objects.length, byStatusCategory, byLiveness, highestSeverity, staleCount: byLiveness.stale ?? 0, authRequiredCount: byLiveness.auth_required ?? 0, unreachableCount: byLiveness.unreachable ?? 0, }; } function summarizeObjectPayloads(objects: Array>, objectLimit = objects.length) { return { ...summarizeObjects(objects), objects: objects.slice(0, objectLimit).map((object) => ({ id: object.id, providerKey: object.providerKey, objectType: object.objectType, displayKey: object.displayKey, iconKey: object.iconKey, displayTitle: object.displayTitle, statusIconKey: object.statusIconKey, statusCategory: object.statusCategory, statusTone: object.statusTone, liveness: object.liveness, isTerminal: object.isTerminal, })), }; } async function getIssueSummary(issueId: string) { if (!(await isEnabled())) return emptySummary(); const groups = await listForIssue(issueId); const objects = groups.flatMap((group) => (group.object ? [group.object] : [])); return summarizeObjectPayloads(objects); } async function getIssueSummaries(companyId: string, issueIds: string[]) { if (!(await isEnabled())) return new Map>(); const uniqueIssueIds = [...new Set(issueIds)].filter((id) => id.length > 0); const summaries = new Map>(); if (uniqueIssueIds.length === 0) return summaries; const rows = await db .select({ issueId: externalObjectMentions.sourceIssueId, object: externalObjects, }) .from(externalObjectMentions) .innerJoin(externalObjects, eq(externalObjectMentions.objectId, externalObjects.id)) .where(and( eq(externalObjectMentions.companyId, companyId), inArray(externalObjectMentions.sourceIssueId, uniqueIssueIds), )); const now = new Date(); const objectsByIssueId = new Map>>(); for (const row of rows) { const issueObjects = objectsByIssueId.get(row.issueId) ?? new Map>(); issueObjects.set(row.object.id, toObjectPayload(row.object, now)); objectsByIssueId.set(row.issueId, issueObjects); } for (const [issueId, issueObjects] of objectsByIssueId) { const objects = [...issueObjects.values()]; if (objects.length > 0) summaries.set(issueId, summarizeObjectPayloads(objects)); } return summaries; } async function getProjectSummary(projectId: string) { if (!(await isEnabled())) return emptySummary(); const projectIssues = await db .select({ id: issues.id, companyId: issues.companyId }) .from(issues) .where(and(eq(issues.projectId, projectId), inArray(issues.status, ["todo", "in_progress", "in_review", "blocked"]))); if (projectIssues.length === 0) return { ...summarizeObjects([]), objects: [] }; const companyIds = new Set(projectIssues.map((issue) => issue.companyId)); if (companyIds.size !== 1) return { ...summarizeObjects([]), objects: [] }; const issueIds = projectIssues.map((issue) => issue.id); const rows = await db .select({ object: externalObjects }) .from(externalObjectMentions) .innerJoin(externalObjects, eq(externalObjectMentions.objectId, externalObjects.id)) .where(and( eq(externalObjectMentions.companyId, projectIssues[0]!.companyId), inArray(externalObjectMentions.sourceIssueId, issueIds), )); const now = new Date(); const objectsById = new Map>(); for (const row of rows) objectsById.set(row.object.id, toObjectPayload(row.object, now)); const objects = [...objectsById.values()]; return summarizeObjectPayloads(objects, 25); } type RefreshObjectInput = { companyId: string; actor?: Pick; force?: boolean; now?: Date; }; function refreshOwnerWhere(object: ExternalObjectRecord, refreshToken: string) { return and( eq(externalObjects.id, object.id), eq(externalObjects.companyId, object.companyId), eq(externalObjects.refreshToken, refreshToken), ); } async function refreshSupersededResult(object: ExternalObjectRecord, now: Date) { const latest = await db .select() .from(externalObjects) .where(and(eq(externalObjects.id, object.id), eq(externalObjects.companyId, object.companyId))) .then((rows) => rows[0] ?? null); if (!latest) throw notFound("External object not found"); return { object: toObjectPayload(latest, now), refreshed: false, reason: "refresh_superseded" as const }; } async function resolveObjectRefresh( object: ExternalObjectRecord, input: RefreshObjectInput, now: Date, refreshToken: string, ) { const pluginResult = await resolveViaPluginProvider(db, opts.pluginWorkerManager, object); const resolver = pluginResult ? null : resolverRegistry.find(object); if (!pluginResult && !resolver) { const [updated] = await db .update(externalObjects) .set({ liveness: visibleLiveness(object, now) === "fresh" ? "stale" : object.liveness, nextRefreshAt: addSeconds(now, DEFAULT_RETRY_AFTER_SECONDS), refreshStartedAt: null, refreshToken: null, updatedAt: now, }) .where(refreshOwnerWhere(object, refreshToken)) .returning(); if (!updated) return refreshSupersededResult(object, now); return { object: toObjectPayload(updated ?? object, now), refreshed: false, reason: "no_resolver" as const }; } const result = pluginResult ?? await resolver!.resolve({ companyId: object.companyId, object }); if (!result.ok) { const [updated] = await db .update(externalObjects) .set({ liveness: result.liveness, lastErrorAt: now, lastErrorCode: result.errorCode, lastErrorMessage: sanitizeErrorMessage(result.errorMessage), nextRefreshAt: addSeconds(now, result.retryAfterSeconds ?? DEFAULT_RETRY_AFTER_SECONDS), refreshStartedAt: null, refreshToken: null, updatedAt: now, }) .where(refreshOwnerWhere(object, refreshToken)) .returning(); if (!updated) return refreshSupersededResult(object, now); publishLiveEvent({ companyId: object.companyId, type: "external_object.updated", payload: { objectId: object.id, liveness: result.liveness }, }); return { object: toObjectPayload(updated ?? object, now), refreshed: true, reason: result.liveness }; } const snapshot = result.snapshot; const patch = { displayKey: snapshot.displayKey ?? object.displayKey, iconKey: snapshot.iconKey ?? object.iconKey, displayTitle: snapshot.displayTitle ?? object.displayTitle, statusKey: snapshot.statusKey ?? object.statusKey, statusLabel: snapshot.statusLabel ?? object.statusLabel, statusIconKey: snapshot.statusIconKey ?? object.statusIconKey, statusCategory: snapshot.statusCategory, statusTone: snapshot.statusTone, isTerminal: snapshot.isTerminal ?? object.isTerminal, data: snapshot.data ?? object.data, remoteVersion: snapshot.remoteVersion ?? object.remoteVersion, etag: snapshot.etag ?? object.etag, liveness: "fresh" as ExternalObjectLivenessState, lastResolvedAt: now, lastErrorAt: null, lastErrorCode: null, lastErrorMessage: null, nextRefreshAt: addSeconds(now, snapshot.ttlSeconds ?? DEFAULT_REFRESH_TTL_SECONDS), refreshStartedAt: null, refreshToken: null, updatedAt: now, }; const [updated] = await db .update(externalObjects) .set({ ...patch, lastChangedAt: objectChanged(object, { ...object, ...patch }) ? now : object.lastChangedAt, }) .where(refreshOwnerWhere(object, refreshToken)) .returning(); if (!updated) return refreshSupersededResult(object, now); const next = updated ?? object; if (objectChanged(object, next) && input.actor) { await logActivity(db, { companyId: object.companyId, actorType: input.actor.actorType, actorId: input.actor.actorId, agentId: input.actor.agentId, runId: input.actor.runId, action: "external_object.status_changed", entityType: "external_object", entityId: object.id, details: { providerKey: object.providerKey, objectType: object.objectType, statusCategory: next.statusCategory, statusLabel: next.statusLabel, _previous: { statusCategory: object.statusCategory, statusLabel: object.statusLabel, }, }, }); } publishLiveEvent({ companyId: object.companyId, type: "external_object.updated", payload: { objectId: object.id, statusCategory: next.statusCategory, liveness: next.liveness }, }); return { object: toObjectPayload(next, now), refreshed: true, reason: "resolved" as const }; } async function claimObjectRefresh( object: ExternalObjectRecord, input: RefreshObjectInput, now: Date, ) { const staleRefreshStartedBefore = new Date(now.getTime() - DEFAULT_REFRESH_LEASE_SECONDS * 1000); const leaseAvailable = or( isNull(externalObjects.refreshStartedAt), lte(externalObjects.refreshStartedAt, staleRefreshStartedBefore), )!; const refreshToken = randomUUID(); const dueOrForced = input.force ? leaseAvailable : and( leaseAvailable, or(isNull(externalObjects.nextRefreshAt), lte(externalObjects.nextRefreshAt, now))!, )!; const [claimed] = await db .update(externalObjects) .set({ refreshStartedAt: now, refreshToken, updatedAt: now, }) .where(and( eq(externalObjects.id, object.id), eq(externalObjects.companyId, object.companyId), dueOrForced, )) .returning(); return claimed ?? null; } function startRefreshLeaseRenewal(object: ExternalObjectRecord, refreshToken: string) { const interval = setInterval(() => { const renewedAt = new Date(); void db .update(externalObjects) .set({ refreshStartedAt: renewedAt, updatedAt: renewedAt }) .where(refreshOwnerWhere(object, refreshToken)) .catch((err: unknown) => { logger.warn({ err, objectId: object.id }, "external object refresh lease renewal failed"); }); }, REFRESH_LEASE_RENEW_INTERVAL_MS); interval.unref?.(); return () => { clearInterval(interval); }; } const objectRefreshesInFlight = new Map>>>(); async function refreshObject( objectId: string, input: RefreshObjectInput, ) { const now = input.now ?? new Date(); const object = await db .select() .from(externalObjects) .where(and(eq(externalObjects.id, objectId), eq(externalObjects.companyId, input.companyId))) .then((rows) => rows[0] ?? null); if (!object) throw notFound("External object not found"); if (!input.force && object.nextRefreshAt && object.nextRefreshAt > now) { return { object: toObjectPayload(object, now), refreshed: false, reason: "backoff" as const }; } const refreshKey = `${object.companyId}:${object.id}`; const existingRefresh = objectRefreshesInFlight.get(refreshKey); if (existingRefresh) return existingRefresh; const claimed = await claimObjectRefresh(object, input, now); if (!claimed) { const latest = await db .select() .from(externalObjects) .where(and(eq(externalObjects.id, object.id), eq(externalObjects.companyId, object.companyId))) .then((rows) => rows[0] ?? null); if (!latest) throw notFound("External object not found"); return { object: toObjectPayload(latest, now), refreshed: false, reason: "refresh_in_progress" as const }; } if (!claimed.refreshToken) { throw new Error("External object refresh claim did not return a refresh token"); } const stopRenewingRefreshLease = startRefreshLeaseRenewal(object, claimed.refreshToken); const refresh = resolveObjectRefresh(object, input, now, claimed.refreshToken).finally(() => { stopRenewingRefreshLease(); objectRefreshesInFlight.delete(refreshKey); }); objectRefreshesInFlight.set(refreshKey, refresh); return refresh; } async function refreshIssueObjects(issueId: string, input: { companyId: string; objectIds?: string[]; actor?: Pick; }) { if (!(await isEnabled())) return []; const groups = await listForIssue(issueId); const objectIds = groups .flatMap((group) => (group.object ? [group.object.id] : [])) .filter((id) => !input.objectIds || input.objectIds.includes(id)); const results = []; for (const objectId of objectIds) { results.push(await refreshObject(objectId, { companyId: input.companyId, actor: input.actor })); } return results; } async function refreshDueObjectsUnchecked(companyId: string, limit = 50, now = new Date()) { const staleRefreshStartedBefore = new Date(now.getTime() - DEFAULT_REFRESH_LEASE_SECONDS * 1000); const due = await db .select({ id: externalObjects.id }) .from(externalObjects) .where( and( eq(externalObjects.companyId, companyId), eq(externalObjects.isTerminal, false), lte(externalObjects.nextRefreshAt, now), or( isNull(externalObjects.refreshStartedAt), lte(externalObjects.refreshStartedAt, staleRefreshStartedBefore), ), ), ) .limit(limit); const results = []; for (const row of due) { results.push(await refreshObject(row.id, { companyId, actor: { actorType: "system", actorId: "external-object-resolver", agentId: null, runId: null }, now, })); } return results; } async function refreshDueObjects(companyId: string, limit = 50, now = new Date()) { if (!(await isEnabled())) return []; return refreshDueObjectsUnchecked(companyId, limit, now); } async function refreshDueObjectsForActiveCompanies(limitPerCompany = 50, now = new Date()) { if (!(await isEnabled())) return { companies: 0, checked: 0, refreshed: 0 }; const activeCompanies = await db .select({ id: companies.id }) .from(companies) .where(eq(companies.status, "active")); let checked = 0; let refreshed = 0; for (const company of activeCompanies) { const results = await refreshDueObjectsUnchecked(company.id, limitPerCompany, now); checked += results.length; refreshed += results.filter((result) => result.refreshed).length; } return { companies: activeCompanies.length, checked, refreshed }; } return { syncIssue, syncComment, syncDocument, syncIssueSafely, syncCommentSafely, syncDocumentSafely, listForIssue, getIssueSummary, getIssueSummaries, getProjectSummary, refreshObject, refreshIssueObjects, refreshDueObjects, refreshDueObjectsForActiveCompanies, }; }