diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.md index b705aa9da4..fabb879125 100644 --- a/doc/SPEC-implementation.md +++ b/doc/SPEC-implementation.md @@ -924,6 +924,10 @@ The current app also exposes V1-supporting surfaces for: - issue thread interactions (`suggest_tasks`, `ask_user_questions`, `request_confirmation`) - issue approvals, issue references/search, labels, read state, inbox/archive state, and work products +- company search through `GET /companies/:companyId/search` plus agent-oriented bulk extraction through + `GET /companies/:companyId/search/extract`; extraction accepts a server-escaped literal `contains`, optional + server-owned URL expansion, issue/comment/document scopes, status/date filters, issue-level pagination, and + explicit issue/match truncation flags - execution workspaces, project workspaces, workspace runtime services, and workspace operations - task watchdog configuration and reusable watchdog issue orchestration for explicitly watched issue subtrees - routines and scheduled/API/webhook triggers diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index caefc73521..0802e3e238 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -658,6 +658,12 @@ export type { ProjectManagedByPlugin, ProjectWorkspace, CompanySearchCountType, + CompanySearchExtractIssueResult, + CompanySearchExtractKind, + CompanySearchExtractMatch, + CompanySearchExtractResponse, + CompanySearchExtractScope, + CompanySearchExtractSourceRef, CompanySearchFilterOptionCounts, CompanySearchHighlight, CompanySearchArtifactSummary, @@ -1209,7 +1215,13 @@ export type { QuotaWindow, ProviderQuotaResult, } from "./types/index.js"; -export { COMPANY_SEARCH_SCOPES, COMPANY_SEARCH_SORTS, COMPANY_SEARCH_UPDATED_WITHIN_OPTIONS } from "./types/index.js"; +export { + COMPANY_SEARCH_EXTRACT_KINDS, + COMPANY_SEARCH_EXTRACT_SCOPES, + COMPANY_SEARCH_SCOPES, + COMPANY_SEARCH_SORTS, + COMPANY_SEARCH_UPDATED_WITHIN_OPTIONS, +} from "./types/index.js"; export { ISSUE_REFERENCE_IDENTIFIER_RE, buildIssueReferenceHref, @@ -1466,12 +1478,18 @@ export { type CreateDocumentAnnotationComment, type CreateDocumentAnnotationThread, type UpdateDocumentAnnotationThread, + companySearchExtractQuerySchema, companySearchQuerySchema, + COMPANY_SEARCH_EXTRACT_DEFAULT_LIMIT, + COMPANY_SEARCH_EXTRACT_MAX_LIMIT, + COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE, + COMPANY_SEARCH_EXTRACT_MAX_OFFSET, COMPANY_SEARCH_DEFAULT_LIMIT, COMPANY_SEARCH_MAX_LIMIT, COMPANY_SEARCH_MAX_OFFSET, COMPANY_SEARCH_MAX_QUERY_LENGTH, COMPANY_SEARCH_MAX_TOKENS, + type CompanySearchExtractQuery, type CompanySearchQuery, createIssueSchema, createIssueInputSchema, diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index 8506983117..74537a5a33 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -251,6 +251,12 @@ export type { export type { Project, ProjectBudgetSummary, ProjectCodebase, ProjectCodebaseOrigin, ProjectGoalRef, ProjectManagedByPlugin, ProjectWorkspace } from "./project.js"; export type { CompanySearchCountType, + CompanySearchExtractIssueResult, + CompanySearchExtractKind, + CompanySearchExtractMatch, + CompanySearchExtractResponse, + CompanySearchExtractScope, + CompanySearchExtractSourceRef, CompanySearchFilterOptionCounts, CompanySearchHighlight, CompanySearchArtifactSummary, @@ -266,7 +272,13 @@ export type { CompanySearchZeroResults, CompanySearchZeroResultsLoosenSuggestion, } from "./search.js"; -export { COMPANY_SEARCH_SCOPES, COMPANY_SEARCH_SORTS, COMPANY_SEARCH_UPDATED_WITHIN_OPTIONS } from "./search.js"; +export { + COMPANY_SEARCH_EXTRACT_KINDS, + COMPANY_SEARCH_EXTRACT_SCOPES, + COMPANY_SEARCH_SCOPES, + COMPANY_SEARCH_SORTS, + COMPANY_SEARCH_UPDATED_WITHIN_OPTIONS, +} from "./search.js"; export type { ExecutionWorkspace, ExecutionWorkspaceSummary, diff --git a/packages/shared/src/types/search.ts b/packages/shared/src/types/search.ts index 7ed7790424..1b812bba5d 100644 --- a/packages/shared/src/types/search.ts +++ b/packages/shared/src/types/search.ts @@ -108,3 +108,45 @@ export interface CompanySearchResponse { zeroResults: CompanySearchZeroResults | null; hasMore: boolean; } + +export const COMPANY_SEARCH_EXTRACT_SCOPES = ["all", "issues", "comments", "documents"] as const; +export type CompanySearchExtractScope = (typeof COMPANY_SEARCH_EXTRACT_SCOPES)[number]; + +export const COMPANY_SEARCH_EXTRACT_KINDS = ["literal", "url"] as const; +export type CompanySearchExtractKind = (typeof COMPANY_SEARCH_EXTRACT_KINDS)[number]; + +export type CompanySearchExtractSourceRef = + | { type: "issue"; issueId: string } + | { type: "comment"; commentId: string } + | { type: "document"; documentId: string; documentKey: string }; + +export interface CompanySearchExtractMatch { + value: string; + field: "title" | "description" | "comment" | "document_title" | "document_body"; + label: string; + excerpt: string; + excerptTruncated: boolean; + source: CompanySearchExtractSourceRef; +} + +export interface CompanySearchExtractIssueResult { + issueId: string; + identifier: string | null; + title: string; + status: IssueStatus; + assigneeAgentId: string | null; + updatedAt: string; + matches: CompanySearchExtractMatch[]; + matchesTruncated: boolean; +} + +export interface CompanySearchExtractResponse { + contains: string; + kind: CompanySearchExtractKind; + scope: CompanySearchExtractScope; + limit: number; + offset: number; + results: CompanySearchExtractIssueResult[]; + hasMore: boolean; + truncated: boolean; +} diff --git a/packages/shared/src/validators/index.ts b/packages/shared/src/validators/index.ts index d3ccb2aa70..2fa890e13e 100644 --- a/packages/shared/src/validators/index.ts +++ b/packages/shared/src/validators/index.ts @@ -420,12 +420,18 @@ export { } from "./issue.js"; export { + COMPANY_SEARCH_EXTRACT_DEFAULT_LIMIT, + COMPANY_SEARCH_EXTRACT_MAX_LIMIT, + COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE, + COMPANY_SEARCH_EXTRACT_MAX_OFFSET, COMPANY_SEARCH_DEFAULT_LIMIT, COMPANY_SEARCH_MAX_LIMIT, COMPANY_SEARCH_MAX_OFFSET, COMPANY_SEARCH_MAX_QUERY_LENGTH, COMPANY_SEARCH_MAX_TOKENS, companySearchQuerySchema, + companySearchExtractQuerySchema, + type CompanySearchExtractQuery, type CompanySearchQuery, } from "./search.js"; diff --git a/packages/shared/src/validators/search.ts b/packages/shared/src/validators/search.ts index 5c995dd823..b54922c1e6 100644 --- a/packages/shared/src/validators/search.ts +++ b/packages/shared/src/validators/search.ts @@ -1,13 +1,22 @@ import { z } from "zod"; import { ISSUE_PRIORITIES, ISSUE_STATUSES } from "../constants.js"; import { isUuidLike } from "../agent-url-key.js"; -import { COMPANY_SEARCH_SCOPES, COMPANY_SEARCH_SORTS } from "../types/search.js"; +import { + COMPANY_SEARCH_EXTRACT_KINDS, + COMPANY_SEARCH_EXTRACT_SCOPES, + COMPANY_SEARCH_SCOPES, + COMPANY_SEARCH_SORTS, +} from "../types/search.js"; export const COMPANY_SEARCH_MAX_QUERY_LENGTH = 200; export const COMPANY_SEARCH_MAX_TOKENS = 8; export const COMPANY_SEARCH_DEFAULT_LIMIT = 20; export const COMPANY_SEARCH_MAX_LIMIT = 50; export const COMPANY_SEARCH_MAX_OFFSET = 200; +export const COMPANY_SEARCH_EXTRACT_DEFAULT_LIMIT = 100; +export const COMPANY_SEARCH_EXTRACT_MAX_LIMIT = 200; +export const COMPANY_SEARCH_EXTRACT_MAX_OFFSET = 5_000; +export const COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE = 20; const UPDATED_WITHIN_RE = /^[1-9]\d{0,2}(h|d|w|m)$/; @@ -180,3 +189,71 @@ export const companySearchQuerySchema = z.object({ }); export type CompanySearchQuery = z.infer; + +export const companySearchExtractQuerySchema = z.object({ + contains: z.unknown().transform((value, ctx) => { + const normalized = parseOptionalString(value, ctx, "contains"); + if (!normalized || normalized.length < 2) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "contains must be at least 2 characters" }); + return ""; + } + if (normalized.length > COMPANY_SEARCH_MAX_QUERY_LENGTH) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `contains must be at most ${COMPANY_SEARCH_MAX_QUERY_LENGTH} characters`, + }); + } + return normalized.slice(0, COMPANY_SEARCH_MAX_QUERY_LENGTH); + }), + kind: z.unknown() + .optional() + .transform((value, ctx) => { + const normalized = parseOptionalString(value, ctx, "kind") ?? "literal"; + if (!(COMPANY_SEARCH_EXTRACT_KINDS as readonly string[]).includes(normalized)) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "kind must be literal or url" }); + return "literal"; + } + return normalized as (typeof COMPANY_SEARCH_EXTRACT_KINDS)[number]; + }), + scope: z.unknown() + .optional() + .transform((value, ctx) => { + const normalized = parseOptionalString(value, ctx, "scope") ?? "all"; + if (!(COMPANY_SEARCH_EXTRACT_SCOPES as readonly string[]).includes(normalized)) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "scope must be all, issues, comments, or documents" }); + return "all"; + } + return normalized as (typeof COMPANY_SEARCH_EXTRACT_SCOPES)[number]; + }), + limit: z.unknown() + .optional() + .transform((value, ctx) => parseIntegerQuery( + value, + ctx, + "limit", + COMPANY_SEARCH_EXTRACT_DEFAULT_LIMIT, + 1, + COMPANY_SEARCH_EXTRACT_MAX_LIMIT, + )), + offset: z.unknown() + .optional() + .transform((value, ctx) => parseIntegerQuery(value, ctx, "offset", 0, 0, COMPANY_SEARCH_EXTRACT_MAX_OFFSET)), + status: z.unknown() + .optional() + .transform((value, ctx) => parseEnumList(value, ctx, "status", ISSUE_STATUSES)), + updatedWithin: z.unknown() + .optional() + .transform((value, ctx) => parseUpdatedWithin(value, ctx)), + updatedAfter: z.unknown() + .optional() + .transform((value, ctx) => parseUpdatedAfter(value, ctx)), +}).superRefine((value, ctx) => { + if (value.updatedWithin && value.updatedAfter) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "updatedWithin and updatedAfter cannot be used together", + }); + } +}); + +export type CompanySearchExtractQuery = z.infer; diff --git a/server/src/__tests__/company-search-extract-routes.test.ts b/server/src/__tests__/company-search-extract-routes.test.ts new file mode 100644 index 0000000000..67a7144c96 --- /dev/null +++ b/server/src/__tests__/company-search-extract-routes.test.ts @@ -0,0 +1,96 @@ +import express from "express"; +import request from "supertest"; +import { describe, expect, it, vi } from "vitest"; +import type { + CompanySearchExtractQuery, + CompanySearchExtractResponse, + CompanySearchQuery, + CompanySearchResponse, +} from "@paperclipai/shared"; +import { issueRoutes } from "../routes/issues.js"; +import { createCompanySearchRateLimiter } from "../services/company-search-rate-limit.js"; + +function extractResponse(query: CompanySearchExtractQuery): CompanySearchExtractResponse { + return { + contains: query.contains, + kind: query.kind, + scope: query.scope, + limit: query.limit, + offset: query.offset, + results: [], + hasMore: false, + truncated: false, + }; +} + +function unusedSearch(_companyId: string, _query: CompanySearchQuery): Promise { + throw new Error("interactive search should not be called"); +} + +function createApp(companyIds: string[], extract: (companyId: string, query: CompanySearchExtractQuery) => Promise) { + const app = express(); + app.use((req, _res, next) => { + req.actor = { + type: "board", + userId: "user-1", + companyIds, + source: "session", + isInstanceAdmin: true, + }; + next(); + }); + app.use("/api", issueRoutes({} as never, {} as never, { + searchService: { search: unusedSearch, extract }, + searchRateLimiter: createCompanySearchRateLimiter({ + maxRequests: 1, + windowMs: 60_000, + now: () => 1_000, + }), + })); + return app; +} + +describe("company extract-search route", () => { + it("parses the extraction query and invokes the service", async () => { + const extract = vi.fn(async (_companyId: string, query: CompanySearchExtractQuery) => extractResponse(query)); + const app = createApp(["company-1"], extract); + + const response = await request(app) + .get("/api/companies/company-1/search/extract") + .query({ contains: "github.com/example/repo/pull", kind: "url", scope: "comments", limit: "200" }) + .expect(200); + + expect(extract).toHaveBeenCalledWith("company-1", expect.objectContaining({ + contains: "github.com/example/repo/pull", + kind: "url", + scope: "comments", + limit: 200, + })); + expect(response.body).toMatchObject({ kind: "url", scope: "comments", truncated: false }); + }); + + it("denies cross-company access before invoking the service", async () => { + const extract = vi.fn(async (_companyId: string, query: CompanySearchExtractQuery) => extractResponse(query)); + const app = createApp(["company-1"], extract); + + await request(app) + .get("/api/companies/company-2/search/extract?contains=needle") + .expect(403); + + expect(extract).not.toHaveBeenCalled(); + }); + + it("shares the company-search rate limiter", async () => { + const extract = vi.fn(async (_companyId: string, query: CompanySearchExtractQuery) => extractResponse(query)); + const app = createApp(["company-1"], extract); + + await request(app).get("/api/companies/company-1/search/extract?contains=needle").expect(200); + const limited = await request(app) + .get("/api/companies/company-1/search/extract?contains=needle") + .expect(429); + + expect(extract).toHaveBeenCalledTimes(1); + expect(limited.body).toEqual({ error: "Search rate limit exceeded", retryAfterSeconds: 60 }); + expect(limited.headers["retry-after"]).toBe("60"); + }); +}); diff --git a/server/src/__tests__/company-search-extract-service.test.ts b/server/src/__tests__/company-search-extract-service.test.ts new file mode 100644 index 0000000000..e03b7c6de6 --- /dev/null +++ b/server/src/__tests__/company-search-extract-service.test.ts @@ -0,0 +1,244 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { + companies, + createDb, + documents, + issueComments, + issueDocuments, + issues, +} from "@paperclipai/db"; +import { + COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE, + companySearchExtractQuerySchema, +} from "@paperclipai/shared"; +import { + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, +} from "./helpers/embedded-postgres.js"; +import { companySearchExtractService } from "../services/company-search-extract.js"; + +const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; + +if (!embeddedPostgresSupport.supported) { + console.warn( + `Skipping embedded Postgres extract-search tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, + ); +} + +describe("extract-search query validation", () => { + it("accepts supported extraction filters and rejects unsafe or ambiguous input", () => { + const parsed = companySearchExtractQuerySchema.parse({ + contains: "github.com/paperclipai/paperclip/pull", + kind: "url", + scope: "comments", + status: "in_progress,in_review", + limit: "200", + offset: "5000", + updatedWithin: "30d", + }); + + expect(parsed.kind).toBe("url"); + expect(parsed.scope).toBe("comments"); + expect(parsed.status).toEqual(["in_progress", "in_review"]); + expect(() => companySearchExtractQuerySchema.parse({ contains: ".*", kind: "regex" })).toThrow(); + expect(() => companySearchExtractQuerySchema.parse({ contains: "x" })).toThrow(); + expect(() => companySearchExtractQuerySchema.parse({ contains: "needle", limit: "201" })).toThrow(); + expect(() => companySearchExtractQuerySchema.parse({ + contains: "needle", + updatedWithin: "30d", + updatedAfter: "2026-01-01T00:00:00.000Z", + })).toThrow(); + }); +}); + +describeEmbeddedPostgres("companySearchExtractService", () => { + let db!: ReturnType; + let svc!: ReturnType; + let tempDb: Awaited> | null = null; + + beforeAll(async () => { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-company-search-extract-"); + db = createDb(tempDb.connectionString); + svc = companySearchExtractService(db); + }, 20_000); + + afterEach(async () => { + await db.delete(issueDocuments); + await db.delete(documents); + await db.delete(issueComments); + await db.delete(issues); + await db.delete(companies); + }); + + afterAll(async () => { + await tempDb?.cleanup(); + }); + + async function createCompany(name = "Paperclip") { + const companyId = randomUUID(); + await db.insert(companies).values({ + id: companyId, + name, + issuePrefix: `E${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, + requireBoardApprovalForNewAgents: false, + }); + return companyId; + } + + async function createIssue(companyId: string, values: Partial = {}) { + const id = values.id ?? randomUUID(); + await db.insert(issues).values({ + id, + companyId, + identifier: values.identifier ?? "EXT-1", + title: values.title ?? "Extract target", + description: values.description ?? null, + status: values.status ?? "in_progress", + priority: values.priority ?? "medium", + ...values, + }); + return id; + } + + it("expands and deduplicates URLs across issue, comment, and document sources", async () => { + const companyId = await createCompany(); + const firstUrl = "https://github.com/paperclipai/paperclip/pull/123"; + const secondUrl = "https://github.com/paperclipai/paperclip/pull/456"; + const thirdUrl = "https://github.com/paperclipai/paperclip/pull/789"; + const issueId = await createIssue(companyId, { + description: `Primary ${firstUrl} and duplicate ${firstUrl}.`, + }); + await db.insert(issueComments).values({ + companyId, + issueId, + body: `Review ${secondUrl} and repeat ${firstUrl}`, + }); + const documentId = randomUUID(); + await db.insert(documents).values({ + id: documentId, + companyId, + title: `PR notes ${thirdUrl}`, + latestBody: `Also see [the second PR](${secondUrl}).`, + }); + await db.insert(issueDocuments).values({ + companyId, + issueId, + documentId, + key: "plan", + }); + + const result = await svc.extract(companyId, companySearchExtractQuerySchema.parse({ + contains: "github.com/paperclipai/paperclip/pull", + kind: "url", + })); + + expect(result.results).toHaveLength(1); + expect(result.results[0]?.matches.map((match) => match.value)).toEqual([firstUrl, secondUrl, thirdUrl]); + expect(result.results[0]?.matches.map((match) => match.field)).toEqual([ + "description", + "comment", + "document_title", + ]); + expect(result.results[0]?.matchesTruncated).toBe(false); + expect(result.truncated).toBe(false); + }); + + it("keeps URL sources selected by scheme-less queries", async () => { + const companyId = await createCompany(); + const titleUrl = "https://github.com/paperclipai/paperclip/pull/101"; + const descriptionUrl = "https://github.com/paperclipai/paperclip/pull/102"; + const documentTitleUrl = "https://github.com/paperclipai/paperclip/pull/103"; + const documentBodyUrl = "https://github.com/paperclipai/paperclip/pull/104"; + const issueId = await createIssue(companyId, { + title: `Review ${titleUrl}`, + description: `Then merge ${descriptionUrl}`, + }); + const documentId = randomUUID(); + await db.insert(documents).values({ + id: documentId, + companyId, + title: `Tracking ${documentTitleUrl}`, + latestBody: `Final follow-up ${documentBodyUrl}`, + }); + await db.insert(issueDocuments).values({ + companyId, + issueId, + documentId, + key: "plan", + }); + + const result = await svc.extract(companyId, companySearchExtractQuerySchema.parse({ + contains: "github.com/paperclipai/paperclip/pull", + kind: "url", + scope: "all", + })); + + expect(result.results[0]?.matches.map((match) => [match.field, match.value])).toEqual([ + ["title", titleUrl], + ["description", descriptionUrl], + ["document_title", documentTitleUrl], + ["document_body", documentBodyUrl], + ]); + }); + + it("filters by issue update window and status", async () => { + const companyId = await createCompany(); + const now = Date.now(); + const recentId = await createIssue(companyId, { + identifier: "EXT-RECENT", + description: "needle", + status: "in_review", + updatedAt: new Date(now - 24 * 60 * 60 * 1000), + }); + await createIssue(companyId, { + identifier: "EXT-OLD", + description: "needle", + status: "in_review", + updatedAt: new Date(now - 60 * 24 * 60 * 60 * 1000), + }); + await createIssue(companyId, { + identifier: "EXT-DONE", + description: "needle", + status: "done", + updatedAt: new Date(now - 24 * 60 * 60 * 1000), + }); + + const result = await svc.extract(companyId, companySearchExtractQuerySchema.parse({ + contains: "needle", + updatedWithin: "30d", + status: "in_review", + })); + + expect(result.results.map((row) => row.issueId)).toEqual([recentId]); + }); + + it("caps distinct matches per issue and marks truncation explicitly", async () => { + const companyId = await createCompany(); + const urls = Array.from( + { length: COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE + 1 }, + (_, index) => `https://github.com/paperclipai/paperclip/pull/${index + 1}`, + ); + await createIssue(companyId, { description: urls.join(" ") }); + + const result = await svc.extract(companyId, companySearchExtractQuerySchema.parse({ + contains: "github.com/paperclipai/paperclip/pull", + kind: "url", + })); + + expect(result.results[0]?.matches).toHaveLength(COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE); + expect(result.results[0]?.matchesTruncated).toBe(true); + expect(result.truncated).toBe(true); + }); + + it("does not return matching issues from another company", async () => { + const companyId = await createCompany(); + const otherCompanyId = await createCompany("Other"); + await createIssue(otherCompanyId, { description: "needle" }); + + const result = await svc.extract(companyId, companySearchExtractQuerySchema.parse({ contains: "needle" })); + + expect(result.results).toEqual([]); + }); +}); diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index 80a99b51c0..a7bf135606 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -27,6 +27,7 @@ import { acceptIssueThreadInteractionSchema, attachmentArtifactWorkProductMetadataSchema, cancelIssueThreadInteractionSchema, + companySearchExtractQuerySchema, companySearchQuerySchema, createIssueAttachmentMetadataSchema, createIssueThreadInteractionSchema, @@ -63,6 +64,8 @@ import { isUuidLike, normalizeIssueIdentifier as normalizeIssueReferenceIdentifier, type CompactIssue, + type CompanySearchExtractQuery, + type CompanySearchExtractResponse, type CompanySearchQuery, type CompanySearchResponse, type ExecutionWorkspace, @@ -251,6 +254,7 @@ type RecoveryRevalidationTrigger = | "work_product" | "read_projection"; type CompanySearchService = { + extract(companyId: string, query: CompanySearchExtractQuery): Promise; search(companyId: string, query: CompanySearchQuery): Promise; }; type ActivityIssueRelationSummary = { @@ -4531,6 +4535,40 @@ export function issueRoutes( }); }); + router.get("/companies/:companyId/search/extract", async (req, res) => { + const companyId = req.params.companyId as string; + assertCompanyAccess(req, companyId); + const companyScopeDecision = await access.decide({ + actor: req.actor, + action: "company_scope:read", + resource: { type: "company", companyId }, + }); + if (!companyScopeDecision.allowed) { + res.status(403).json({ error: "Company search is outside this actor's authorization boundary" }); + return; + } + const parsedQuery = companySearchExtractQuerySchema.safeParse(req.query); + if (!parsedQuery.success) { + res.status(400).json({ + error: parsedQuery.error.issues[0]?.message ?? "Invalid extract search query", + }); + return; + } + const rateLimit = searchRateLimiter.consume(companySearchRateLimitActor(req, companyId)); + res.setHeader("X-RateLimit-Limit", String(rateLimit.limit)); + res.setHeader("X-RateLimit-Remaining", String(rateLimit.remaining)); + if (!rateLimit.allowed) { + res.setHeader("Retry-After", String(rateLimit.retryAfterSeconds)); + res.status(429).json({ + error: "Search rate limit exceeded", + retryAfterSeconds: rateLimit.retryAfterSeconds, + }); + return; + } + const result = await getSearchService().extract(companyId, parsedQuery.data); + res.json(result); + }); + router.get("/companies/:companyId/search", async (req, res) => { const companyId = req.params.companyId as string; assertCompanyAccess(req, companyId); diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index 774d53fa89..bdb8bec2b5 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -5142,6 +5142,7 @@ registerCurrentRoute({ for (const route of [ ["get", "/api/companies/import/jobs/{jobId}", "Get company import job status"], ["get", "/api/companies/{companyId}/search", "Search company data"], + ["get", "/api/companies/{companyId}/search/extract", "Extract company search matches"], ["get", "/api/companies/{companyId}/issues/count", "Count issues in a company"], ] as const) { registerCurrentRoute({ diff --git a/server/src/services/company-search-extract.ts b/server/src/services/company-search-extract.ts new file mode 100644 index 0000000000..6462b33445 --- /dev/null +++ b/server/src/services/company-search-extract.ts @@ -0,0 +1,349 @@ +import { and, asc, desc, eq, gte, inArray, isNull, or, sql } from "drizzle-orm"; +import type { SQL, SQLWrapper } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { documents, issueComments, issueDocuments, issues } from "@paperclipai/db"; +import { + COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE, + type CompanySearchExtractIssueResult, + type CompanySearchExtractMatch, + type CompanySearchExtractQuery, + type CompanySearchExtractResponse, + type CompanySearchExtractSourceRef, +} from "@paperclipai/shared"; +import { visibleIssueCondition } from "./issue-visibility.js"; + +const EXCERPT_MAX_CHARS = 180; +const URL_PATTERN = /(?:https?:\/\/|www\.)[^\s<>"'`]+|(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,}\/[^\s<>"'`]+/giu; + +type ExtractSource = { + issueId: string; + field: CompanySearchExtractMatch["field"]; + label: string; + text: string; + source: CompanySearchExtractSourceRef; +}; + +function escapeLikePattern(value: string): string { + return value.replace(/[\\%_]/g, "\\$&"); +} + +function escapeRegexPattern(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function urlContainsPattern(contains: string): string { + const literal = escapeRegexPattern(contains); + return `(?=[^[:space:]<>"']*${literal})(?:(?:https?://|www\\.)[^[:space:]<>"']+|(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z]{2,}/[^[:space:]<>"']+)`; +} + +function contentMatch( + column: SQLWrapper, + query: CompanySearchExtractQuery, + containsPattern: string, + urlPattern: string, +): SQL { + return query.kind === "url" + ? sql`${column} ~* ${urlPattern}` + : sql`${column} ILIKE ${containsPattern}`; +} + +function updatedWithinStart(value: string | undefined, now = new Date()): Date | null { + if (!value) return null; + const match = /^(\d+)(h|d|w|m)$/.exec(value); + if (!match) return null; + const amount = Number.parseInt(match[1]!, 10); + const unit = match[2]; + const hours = unit === "h" ? amount : unit === "d" ? amount * 24 : unit === "w" ? amount * 24 * 7 : amount * 24 * 30; + return new Date(now.getTime() - hours * 60 * 60 * 1000); +} + +function scopeIncludes(scope: CompanySearchExtractQuery["scope"], source: Exclude) { + return scope === "all" || scope === source; +} + +function trimUrlToken(value: string): string { + let result = value.replace(/[.,;:!?]+$/g, ""); + const pairs: Array<[string, string]> = [["(", ")"], ["[", "]"], ["{", "}"]]; + for (const [open, close] of pairs) { + while (result.endsWith(close) && result.split(close).length > result.split(open).length) { + result = result.slice(0, -1); + } + } + return result; +} + +function literalOccurrences(text: string, contains: string) { + const lowerText = text.toLowerCase(); + const lowerContains = contains.toLowerCase(); + const matches: Array<{ value: string; start: number }> = []; + let start = 0; + while (start <= text.length - contains.length) { + const index = lowerText.indexOf(lowerContains, start); + if (index < 0) break; + matches.push({ value: text.slice(index, index + contains.length), start: index }); + start = index + Math.max(contains.length, 1); + } + return matches; +} + +function urlOccurrences(text: string, contains: string) { + const lowerContains = contains.toLowerCase(); + const matches: Array<{ value: string; start: number }> = []; + for (const match of text.matchAll(URL_PATTERN)) { + const raw = match[0]; + const value = trimUrlToken(raw); + if (!value.toLowerCase().includes(lowerContains)) continue; + matches.push({ value, start: match.index ?? 0 }); + } + return matches; +} + +function sourceOccurrences(text: string, query: CompanySearchExtractQuery) { + return query.kind === "url" + ? urlOccurrences(text, query.contains) + : literalOccurrences(text, query.contains); +} + +function excerpt(text: string, start: number, length: number) { + if (text.length <= EXCERPT_MAX_CHARS) { + return { value: text, truncated: false }; + } + const context = Math.max(0, Math.floor((EXCERPT_MAX_CHARS - length) / 2)); + let excerptStart = Math.max(0, start - context); + let excerptEnd = Math.min(text.length, excerptStart + EXCERPT_MAX_CHARS); + excerptStart = Math.max(0, excerptEnd - EXCERPT_MAX_CHARS); + const prefix = excerptStart > 0 ? "…" : ""; + const suffix = excerptEnd < text.length ? "…" : ""; + return { + value: `${prefix}${text.slice(excerptStart, excerptEnd).replace(/\s+/g, " ").trim()}${suffix}`, + truncated: true, + }; +} + +function extractMatches(sources: ExtractSource[], query: CompanySearchExtractQuery) { + const matches: CompanySearchExtractMatch[] = []; + const seen = new Set(); + let matchesTruncated = false; + + for (const source of sources) { + const occurrences = sourceOccurrences(source.text, query); + for (const occurrence of occurrences) { + const dedupeKey = occurrence.value.toLowerCase(); + if (seen.has(dedupeKey)) continue; + seen.add(dedupeKey); + if (matches.length >= COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE) { + matchesTruncated = true; + continue; + } + const matchExcerpt = excerpt(source.text, occurrence.start, occurrence.value.length); + matches.push({ + value: occurrence.value, + field: source.field, + label: source.label, + excerpt: matchExcerpt.value, + excerptTruncated: matchExcerpt.truncated, + source: source.source, + }); + } + } + + return { matches, matchesTruncated }; +} + +export function companySearchExtractService(db: Db) { + return { + extract: async (companyId: string, query: CompanySearchExtractQuery): Promise => { + const containsPattern = `%${escapeLikePattern(query.contains)}%`; + const urlPattern = urlContainsPattern(query.contains); + const scopeConditions: SQL[] = []; + if (scopeIncludes(query.scope, "issues")) { + scopeConditions.push(or( + contentMatch(issues.title, query, containsPattern, urlPattern), + contentMatch(issues.description, query, containsPattern, urlPattern), + )!); + } + if (scopeIncludes(query.scope, "comments")) { + scopeConditions.push(sql`EXISTS ( + SELECT 1 + FROM issue_comments extract_comments + WHERE extract_comments.company_id = ${companyId} + AND extract_comments.issue_id = ${issues.id} + AND extract_comments.deleted_at IS NULL + AND ${query.kind === "url" + ? sql`extract_comments.body ~* ${urlPattern}` + : sql`extract_comments.body ILIKE ${containsPattern}`} + )`); + } + if (scopeIncludes(query.scope, "documents")) { + scopeConditions.push(sql`EXISTS ( + SELECT 1 + FROM issue_documents extract_issue_documents + INNER JOIN documents extract_documents + ON extract_documents.id = extract_issue_documents.document_id + AND extract_documents.company_id = extract_issue_documents.company_id + WHERE extract_issue_documents.company_id = ${companyId} + AND extract_issue_documents.issue_id = ${issues.id} + AND ( + ${query.kind === "url" + ? sql`extract_documents.title ~* ${urlPattern}` + : sql`extract_documents.title ILIKE ${containsPattern}`} + OR ${query.kind === "url" + ? sql`extract_documents.latest_body ~* ${urlPattern}` + : sql`extract_documents.latest_body ILIKE ${containsPattern}`} + ) + )`); + } + + const conditions: SQL[] = [ + eq(issues.companyId, companyId), + visibleIssueCondition(), + or(...scopeConditions)!, + ]; + if (query.status.length > 0) conditions.push(inArray(issues.status, query.status)); + const updatedWithin = updatedWithinStart(query.updatedWithin); + if (updatedWithin) conditions.push(gte(issues.updatedAt, updatedWithin)); + if (query.updatedAfter) conditions.push(gte(issues.updatedAt, new Date(query.updatedAfter))); + + const candidateRows = await db + .select({ + id: issues.id, + identifier: issues.identifier, + title: issues.title, + description: issues.description, + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + updatedAt: issues.updatedAt, + }) + .from(issues) + .where(and(...conditions)) + .orderBy(desc(issues.updatedAt), desc(issues.id)) + .limit(query.limit + 1) + .offset(query.offset); + + const hasMore = candidateRows.length > query.limit; + const pageRows = candidateRows.slice(0, query.limit); + const issueIds = pageRows.map((row) => row.id); + const sourcesByIssue = new Map(); + const addSource = (source: ExtractSource) => { + const sources = sourcesByIssue.get(source.issueId) ?? []; + sources.push(source); + sourcesByIssue.set(source.issueId, sources); + }; + + if (scopeIncludes(query.scope, "issues")) { + for (const row of pageRows) { + if (sourceOccurrences(row.title, query).length > 0) { + addSource({ + issueId: row.id, + field: "title", + label: "Issue title", + text: row.title, + source: { type: "issue", issueId: row.id }, + }); + } + if (row.description && sourceOccurrences(row.description, query).length > 0) { + addSource({ + issueId: row.id, + field: "description", + label: "Issue description", + text: row.description, + source: { type: "issue", issueId: row.id }, + }); + } + } + } + + if (issueIds.length > 0 && scopeIncludes(query.scope, "comments")) { + const commentRows = await db + .select({ id: issueComments.id, issueId: issueComments.issueId, body: issueComments.body }) + .from(issueComments) + .where(and( + eq(issueComments.companyId, companyId), + inArray(issueComments.issueId, issueIds), + isNull(issueComments.deletedAt), + contentMatch(issueComments.body, query, containsPattern, urlPattern), + )) + .orderBy(asc(issueComments.createdAt), asc(issueComments.id)); + for (const row of commentRows) { + addSource({ + issueId: row.issueId, + field: "comment", + label: "Comment", + text: row.body, + source: { type: "comment", commentId: row.id }, + }); + } + } + + if (issueIds.length > 0 && scopeIncludes(query.scope, "documents")) { + const documentRows = await db + .select({ + id: documents.id, + issueId: issueDocuments.issueId, + key: issueDocuments.key, + title: documents.title, + body: documents.latestBody, + }) + .from(issueDocuments) + .innerJoin(documents, and( + eq(documents.id, issueDocuments.documentId), + eq(documents.companyId, issueDocuments.companyId), + )) + .where(and( + eq(issueDocuments.companyId, companyId), + inArray(issueDocuments.issueId, issueIds), + or( + contentMatch(documents.title, query, containsPattern, urlPattern), + contentMatch(documents.latestBody, query, containsPattern, urlPattern), + ), + )) + .orderBy(asc(issueDocuments.key), asc(documents.id)); + for (const row of documentRows) { + const source = { type: "document" as const, documentId: row.id, documentKey: row.key }; + if (row.title && sourceOccurrences(row.title, query).length > 0) { + addSource({ + issueId: row.issueId, + field: "document_title", + label: `Document title (${row.key})`, + text: row.title, + source, + }); + } + if (sourceOccurrences(row.body, query).length > 0) { + addSource({ + issueId: row.issueId, + field: "document_body", + label: `Document (${row.key})`, + text: row.body, + source, + }); + } + } + } + + const results: CompanySearchExtractIssueResult[] = pageRows.map((row) => { + const extracted = extractMatches(sourcesByIssue.get(row.id) ?? [], query); + return { + issueId: row.id, + identifier: row.identifier, + title: row.title, + status: row.status as CompanySearchExtractIssueResult["status"], + assigneeAgentId: row.assigneeAgentId, + updatedAt: row.updatedAt.toISOString(), + ...extracted, + }; + }); + + return { + contains: query.contains, + kind: query.kind, + scope: query.scope, + limit: query.limit, + offset: query.offset, + results, + hasMore, + truncated: hasMore || results.some((result) => result.matchesTruncated), + }; + }, + }; +} diff --git a/server/src/services/company-search.ts b/server/src/services/company-search.ts index 8b90e7b1a8..4de7430cc9 100644 --- a/server/src/services/company-search.ts +++ b/server/src/services/company-search.ts @@ -37,6 +37,7 @@ import { type CompanySearchUpdatedWithinOption, } from "@paperclipai/shared"; import { companyArtifactsService } from "./company-artifacts.js"; +import { companySearchExtractService } from "./company-search-extract.js"; import { visibleIssueCondition } from "./issue-visibility.js"; const MIN_TOKEN_LENGTH = 2; @@ -554,7 +555,9 @@ export function companySearchBranchFetchLimit(limit: number, offset = 0) { } export function companySearchService(db: Db) { + const extractService = companySearchExtractService(db); return { + extract: extractService.extract, search: async (companyId: string, query: CompanySearchQuery): Promise => { const normalizedQuery = normalizeQuery(query.q); const hasSearchText = normalizedQuery.length > 0; diff --git a/server/src/services/index.ts b/server/src/services/index.ts index bce7184929..b7745b470f 100644 --- a/server/src/services/index.ts +++ b/server/src/services/index.ts @@ -1,6 +1,7 @@ export { companyService } from "./companies.js"; export { companyArtifactsService } from "./company-artifacts.js"; export { companySearchService } from "./company-search.js"; +export { companySearchExtractService } from "./company-search-extract.js"; export { feedbackService } from "./feedback.js"; export { companySkillService } from "./company-skills.js"; export { companySkillPolicyService, normalizeSkillPolicySourceType } from "./company-skill-policy.js";