152 lines
5.2 KiB
TypeScript
152 lines
5.2 KiB
TypeScript
import type { OllamaChatMessage } from './ollama.js'
|
|
|
|
export type EmbedJobWithProgress = {
|
|
jobId: string
|
|
fileName: string
|
|
filePath: string
|
|
progress: number
|
|
status: string
|
|
error?: string
|
|
/** ms epoch of last completed batch; multi-batch ZIMs update this each batch. */
|
|
lastBatchAt?: number
|
|
/** ms epoch of first batch start; used as a fallback when lastBatchAt unset. */
|
|
startedAt?: number
|
|
/** Total chunks embedded across this job's batches so far. */
|
|
chunks?: number
|
|
}
|
|
|
|
export type ProcessAndEmbedFileResponse = {
|
|
success: boolean
|
|
message: string
|
|
chunks?: number
|
|
hasMoreBatches?: boolean
|
|
articlesProcessed?: number
|
|
totalArticles?: number
|
|
}
|
|
export type ProcessZIMFileResponse = ProcessAndEmbedFileResponse
|
|
|
|
export type RAGResult = {
|
|
text: string
|
|
score: number
|
|
keywords: string
|
|
chunk_index: number
|
|
created_at: number
|
|
article_title?: string
|
|
section_title?: string
|
|
full_title?: string
|
|
hierarchy?: string
|
|
document_id?: string
|
|
content_type?: string
|
|
source?: string
|
|
}
|
|
|
|
export type RerankedRAGResult = Omit<RAGResult, 'keywords'> & {
|
|
finalScore: number
|
|
}
|
|
|
|
/** One entry in a recorded retrieval stage: just enough to score a ranking. */
|
|
export type StageEntry = { source?: string; score: number }
|
|
|
|
/**
|
|
* The three ranked lists retrieval produces internally, captured so the eval
|
|
* harness can score each stage separately and show whether the heuristic
|
|
* reranker and the source-diversity penalty are earning their complexity.
|
|
*
|
|
* `dense` is the raw cosine ordering from Qdrant, `reranked` adds the
|
|
* keyword/heading boosts, `diversified` adds the same-document penalty.
|
|
*/
|
|
export type RetrievalStages = {
|
|
dense?: StageEntry[]
|
|
reranked?: StageEntry[]
|
|
diversified?: StageEntry[]
|
|
}
|
|
|
|
/**
|
|
* A chunk as returned by `RagService.searchSimilarDocuments` — the shape the
|
|
* chat pipeline consumes and the eval harness scores.
|
|
*/
|
|
export type RetrievedChunk = {
|
|
text: string
|
|
score: number
|
|
metadata?: Record<string, any>
|
|
}
|
|
|
|
/**
|
|
* Knobs on a single pipeline run. Everything is optional: the defaults
|
|
* reproduce production chat exactly. The non-default paths exist so the eval
|
|
* harness can ablate one stage at a time without a parallel implementation.
|
|
*/
|
|
export type PipelineOptions = {
|
|
topK?: number
|
|
scoreThreshold?: number
|
|
collection?: string
|
|
/** Skip the history-aware rewrite (which is an LLM call, and therefore
|
|
* non-deterministic). Retrieval then runs on the raw last user message. */
|
|
skipQueryRewrite?: boolean
|
|
/** Bypass retrieval entirely and inject these chunks as the context. Used by
|
|
* the `oracle` eval mode to isolate generation quality from retrieval. */
|
|
oracleContext?: RetrievedChunk[]
|
|
/** Ignore the user's NOMAD.md. Off in production; on in evals, where a
|
|
* developer's personal instructions would silently skew every result. */
|
|
skipNomadMd?: boolean
|
|
}
|
|
|
|
/**
|
|
* Everything the pipeline decided on the way to a prompt. The controller uses
|
|
* only `messages` and `numCtx`; the eval harness scores the rest. Returning it
|
|
* unconditionally keeps one code path for both.
|
|
*/
|
|
export type PipelineTrace = {
|
|
/** null when retrieval was skipped entirely (empty KB, or no user message). */
|
|
rewrittenQuery: string | null
|
|
/** True when the rewrite LLM call actually ran (it is skipped on turn 1). */
|
|
didRewrite: boolean
|
|
/** Everything retrieval returned, pre-trim. */
|
|
retrieved: RetrievedChunk[]
|
|
/** What actually made it into the prompt, post model-size trim. */
|
|
injected: RetrievedChunk[]
|
|
/** The exact payload handed to Ollama. */
|
|
messages: OllamaChatMessage[]
|
|
numCtx: number | undefined
|
|
contextLimits: { maxResults: number; maxTokens: number }
|
|
timings: { rewriteMs: number; retrievalMs: number }
|
|
}
|
|
|
|
export type FileWarning =
|
|
| { kind: 'zero_chunks'; fileSizeBytes: number }
|
|
| { kind: 'partial_stall'; chunksEmbedded: number; chunksExpected: number }
|
|
|
|
/**
|
|
* Row returned by `GET /api/rag/files`. `state` is null for sources that exist
|
|
* in Qdrant but have no `kb_ingest_state` row (pre-RFC-883 installs where the
|
|
* scanner hasn't yet backfilled). `chunksEmbedded` mirrors the state-machine
|
|
* field; 0 for state-row-less or zero-chunk files.
|
|
*/
|
|
export type StoredFileInfo = {
|
|
source: string
|
|
state: import('./kb_ingest_state.js').KbIngestStateValue | null
|
|
chunksEmbedded: number
|
|
/** Filename portion of `source` (last path segment). */
|
|
fileName: string
|
|
/** File size in bytes from disk; null if the file is missing or unreadable. */
|
|
size: number | null
|
|
/** Last-modified timestamp from disk (ISO 8601); null if unavailable. */
|
|
uploadedAt: string | null
|
|
/** True when `source` lives under the user-uploads directory. Drives which
|
|
* rows offer view/download in the UI. */
|
|
isUserUpload: boolean
|
|
/** Subject/category tag, or null if uncategorized. */
|
|
collection: string | null
|
|
}
|
|
|
|
/**
|
|
* Result of computing per-file warnings. `ok: false` means the computation
|
|
* itself failed (Qdrant unreachable, DB outage, FS read error) — distinct from
|
|
* `ok: true` with an empty map, which means every scanned file is healthy.
|
|
* The frontend should surface a neutral "warnings unavailable" indicator on
|
|
* `!ok` rather than implying everything is fine.
|
|
*/
|
|
export type FileWarningsResult = {
|
|
ok: boolean
|
|
warnings: Record<string, FileWarning[]>
|
|
} |