Merge branch 'dev' into feat/add-details-on-marker

This commit is contained in:
Kenneth Brewer 2026-04-28 10:14:21 -04:00
commit 845ebe09de
14 changed files with 281 additions and 14 deletions

View File

@ -30,7 +30,14 @@ We are committed to providing a welcoming environment for everyone. Disrespectfu
## Before You Start
**Open an issue first.** Before writing any code, please [open an issue](../../issues/new) to discuss your proposed change. This helps avoid duplicate work and ensures your contribution aligns with the project's direction.
**Open an issue first.** Before writing any code for a non-trivial change, you must [open an issue](../../issues/new) to discuss your proposed change. This helps avoid duplicate work and ensures your contribution aligns with the project's direction. **Pull requests submitted without a corresponding issue may be closed at the maintainers' discretion.**
**Trivial fixes are exempt** and may be submitted directly as a PR. Examples:
- Typo and grammar corrections
- Documentation clarifications
- Small one-line bug fixes with an obvious cause
If you're not sure whether your change qualifies as trivial, open an issue first.
When opening an issue:
- Use a clear, descriptive title
@ -149,7 +156,7 @@ This project uses [Semantic Versioning](https://semver.org/). Versions are manag
2. Open a pull request against the `dev` branch of this repository
3. In the PR description:
- Summarize what your changes do and why
- Reference the related issue (e.g., `Closes #123`)
- Reference the related issue (e.g., `Closes #123`) — required for non-trivial changes
- Note any relevant testing steps or environment details
4. Be responsive to feedback — maintainers may request changes. Pull requests with no activity for an extended period may be closed.

View File

@ -55,6 +55,7 @@ export default defineConfig({
() => import('@adonisjs/transmit/transmit_provider'),
() => import('#providers/map_static_provider'),
() => import('#providers/kiwix_migration_provider'),
() => import('#providers/qdrant_restart_policy_provider'),
],
/*

View File

@ -97,4 +97,9 @@ export default class RagController {
return response.status(500).json({ error: 'Error scanning and syncing storage' })
}
}
public async health({ response }: HttpContext) {
const result = await this.ragService.checkQdrantHealth()
return response.status(200).json(result)
}
}

View File

@ -3,7 +3,21 @@ import type { HttpContext } from '@adonisjs/core/http'
import type { NextFn } from '@adonisjs/core/types/http'
import compression from 'compression'
const compress = env.get('DISABLE_COMPRESSION') ? null : compression()
// Skip compression for Server-Sent Events. The compression library buffers
// response writes to determine encoding, which collapses per-token streaming
// into a single block delivered after generation completes (regression in
// v1.31.0-rc.2, reported in #781 by @toasterking).
const compress = env.get('DISABLE_COMPRESSION')
? null
: compression({
filter: (req: any, res: any) => {
const contentType = res.getHeader('Content-Type')
if (typeof contentType === 'string' && contentType.includes('text/event-stream')) {
return false
}
return compression.filter(req, res)
},
})
export default class CompressionMiddleware {
async handle({ request, response }: HttpContext, next: NextFn) {

View File

@ -480,10 +480,21 @@ export class OllamaService {
}
try {
// Prefer Ollama native endpoint (supports batch input natively)
// Prefer Ollama native endpoint (supports batch input natively).
// Pass num_ctx explicitly so we don't depend on the embedding model's
// modelfile defaults. Some installs ship nomic-embed-text:v1.5 with
// num_ctx=2048, which our chunker (sized for ~1500 tokens) can exceed
// on dense content, causing "input length exceeds context length" errors.
// truncate:true is a runtime safety net for any chunk that still overshoots.
// 8192 matches nomic-embed-text:v1.5's RoPE-extrapolated max.
const response = await axios.post(
`${this.baseUrl}/api/embed`,
{ model, input },
{
model,
input,
truncate: true,
options: { num_ctx: 8192 },
},
{ timeout: 60000 }
)
// Some backends (e.g. LM Studio) return HTTP 200 for unknown endpoints with an incompatible

View File

@ -52,14 +52,33 @@ export class RagService {
this.qdrantInitPromise = (async () => {
const qdrantUrl = await this.dockerService.getServiceURL(SERVICE_NAMES.QDRANT)
if (!qdrantUrl) {
throw new Error('Qdrant service is not installed or running.')
throw new Error('Qdrant vector database is offline. Restart the AI Assistant service in Settings to restore the Knowledge Base.')
}
this.qdrant = new QdrantClient({ url: qdrantUrl })
})()
})().catch((err) => {
this.qdrantInitPromise = null
this.qdrant = null
throw err
})
}
return this.qdrantInitPromise
}
public async checkQdrantHealth(): Promise<{ online: boolean; message?: string }> {
try {
await this._ensureDependencies()
await this.qdrant!.getCollections()
return { online: true }
} catch {
this.qdrant = null
this.qdrantInitPromise = null
return {
online: false,
message: 'Qdrant vector database is offline. Restart the AI Assistant service in Settings to restore the Knowledge Base.',
}
}
}
private async _ensureDependencies() {
if (!this.qdrant) {
await this._initializeQdrantClient()

View File

@ -40,7 +40,21 @@ export class ZimService {
await ensureDirectoryExists(dirPath)
const all = await listDirectoryContents(dirPath)
const files = all.filter((item) => item.name.endsWith('.zim'))
const zimEntries = all.filter((item) => item.name.endsWith('.zim'))
const files = await Promise.all(
zimEntries.map(async (entry) => {
const filePath = entry.type === 'file' ? entry.key : join(dirPath, entry.name)
const stats = await getFileStatsIfExists(filePath)
return {
...entry,
title: null,
summary: null,
author: null,
size_bytes: stats ? Number(stats.size) : null,
}
})
)
return {
files,

View File

@ -148,6 +148,15 @@ ZIM files provide offline Wikipedia, books, and other content via Kiwix.
| POST | `/api/maps/download-collection` | Download an entire collection by slug (async) |
| DELETE | `/api/maps/:filename` | Delete a local map file |
### Map Markers
| Method | Path | Description |
|--------|------|-------------|
| GET | `/api/maps/markers` | List map markers |
| POST | `/api/maps/markers` | Add map marker (body: {"name": "Test Marker", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) |
| PATCH | `/api/maps/markers/{id}` | Update a map marker (body: {"name": "Test Marker", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) fields that don't change can be omitted|
| DELETE | `/api/maps/markers/{id}` | Delete a map marker |
---
## Downloads

View File

@ -1,5 +1,5 @@
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
import { useRef, useState } from 'react'
import { useEffect, useRef, useState } from 'react'
import FileUploader from '~/components/file-uploader'
import StyledButton from '~/components/StyledButton'
import StyledSectionHeader from '~/components/StyledSectionHeader'
@ -10,6 +10,7 @@ import { IconX } from '@tabler/icons-react'
import { useModals } from '~/context/ModalContext'
import StyledModal from '../StyledModal'
import ActiveEmbedJobs from '~/components/ActiveEmbedJobs'
import { SERVICE_NAMES } from '../../../constants/service_names'
interface KnowledgeBaseModalProps {
aiAssistantName?: string
@ -30,6 +31,19 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
const { openModal, closeModal } = useModals()
const queryClient = useQueryClient()
const [isStartingQdrant, setIsStartingQdrant] = useState(false)
const { data: healthStatus } = useQuery({
queryKey: ['qdrantHealth'],
queryFn: () => api.checkRAGHealth(),
refetchInterval: isStartingQdrant ? 3_000 : 30_000,
})
const qdrantOffline = healthStatus?.online === false
useEffect(() => {
if (!qdrantOffline) setIsStartingQdrant(false)
}, [qdrantOffline])
const { data: storedFiles = [], isLoading: isLoadingFiles } = useQuery({
queryKey: ['storedFiles'],
queryFn: () => api.getStoredRAGFiles(),
@ -64,6 +78,17 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
},
})
const startQdrantMutation = useMutation({
mutationFn: () => api.affectService(SERVICE_NAMES.QDRANT, 'start'),
onSuccess: () => {
setIsStartingQdrant(true)
queryClient.invalidateQueries({ queryKey: ['qdrantHealth'] })
},
onError: (error: any) => {
addNotification({ type: 'error', message: error?.message || 'Failed to start Qdrant.' })
},
})
const syncMutation = useMutation({
mutationFn: () => api.syncRAGStorage(),
onSuccess: (data) => {
@ -149,6 +174,22 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
</button>
</div>
<div className="overflow-y-auto flex-1 p-6">
{qdrantOffline && (
<div className="mb-4 p-4 bg-red-50 border border-red-200 rounded-lg text-red-700 text-sm dark:bg-red-950 dark:border-red-800 dark:text-red-300 flex items-center justify-between gap-4">
<span>
<strong>Knowledge Base unavailable:</strong> The Qdrant vector database is offline.
</span>
<StyledButton
variant="danger"
size="sm"
onClick={() => startQdrantMutation.mutate()}
loading={startQdrantMutation.isPending || isStartingQdrant}
disabled={startQdrantMutation.isPending || isStartingQdrant}
>
{isStartingQdrant ? 'Starting…' : 'Start Qdrant'}
</StyledButton>
</div>
)}
<div className="bg-surface-primary rounded-lg border shadow-md overflow-hidden">
<div className="p-6">
<FileUploader
@ -165,7 +206,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
size="lg"
icon="IconUpload"
onClick={handleUpload}
disabled={files.length === 0 || isUploading}
disabled={files.length === 0 || isUploading || qdrantOffline}
loading={isUploading}
>
Upload
@ -236,7 +277,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
icon="IconTrash"
onClick={() => cleanupFailedMutation.mutate()}
loading={cleanupFailedMutation.isPending}
disabled={cleanupFailedMutation.isPending}
disabled={cleanupFailedMutation.isPending || qdrantOffline}
>
Clean Up Failed
</StyledButton>
@ -252,7 +293,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
size="md"
icon='IconRefresh'
onClick={handleConfirmSync}
disabled={syncMutation.isPending || isUploading}
disabled={syncMutation.isPending || isUploading || qdrantOffline}
loading={syncMutation.isPending || isUploading}
>
Sync Storage

View File

@ -451,6 +451,13 @@ class API {
})()
}
async checkRAGHealth() {
return catchInternal(async () => {
const response = await this.client.get<{ online: boolean; message?: string }>('/rag/health')
return response.data
})()
}
async getStoredRAGFiles() {
return catchInternal(async () => {
const response = await this.client.get<{ files: string[] }>('/rag/files')

View File

@ -1,5 +1,6 @@
import { Head } from '@inertiajs/react'
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
import { useMemo, useState } from 'react'
import StyledTable from '~/components/StyledTable'
import SettingsLayout from '~/layouts/SettingsLayout'
import api from '~/lib/api'
@ -10,11 +11,18 @@ import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus'
import Alert from '~/components/Alert'
import { ZimFileWithMetadata } from '../../../../types/zim'
import { SERVICE_NAMES } from '../../../../constants/service_names'
import { formatBytes } from '~/lib/util'
import { IconArrowDown, IconArrowUp, IconArrowsSort } from '@tabler/icons-react'
type SortKey = 'name' | 'size'
type SortDirection = 'asc' | 'desc'
export default function ZimPage() {
const queryClient = useQueryClient()
const { openModal, closeAllModals } = useModals()
const { isInstalled } = useServiceInstalledStatus(SERVICE_NAMES.KIWIX)
const [sortKey, setSortKey] = useState<SortKey>('size')
const [sortDirection, setSortDirection] = useState<SortDirection>('desc')
const { data, isLoading } = useQuery<ZimFileWithMetadata[]>({
queryKey: ['zim-files'],
queryFn: getFiles,
@ -25,6 +33,49 @@ export default function ZimPage() {
return res.data.files
}
const sortedData = useMemo(() => {
if (!data) return []
const copy = [...data]
copy.sort((a, b) => {
let cmp = 0
if (sortKey === 'size') {
const aSize = a.size_bytes ?? 0
const bSize = b.size_bytes ?? 0
cmp = aSize - bSize
} else {
const aName = (a.title || a.name).toLowerCase()
const bName = (b.title || b.name).toLowerCase()
cmp = aName.localeCompare(bName)
}
return sortDirection === 'asc' ? cmp : -cmp
})
return copy
}, [data, sortKey, sortDirection])
function toggleSort(key: SortKey) {
if (sortKey === key) {
setSortDirection((d) => (d === 'asc' ? 'desc' : 'asc'))
} else {
setSortKey(key)
setSortDirection(key === 'size' ? 'desc' : 'asc')
}
}
function renderSortHeader(label: string, key: SortKey) {
const active = sortKey === key
const Icon = !active ? IconArrowsSort : sortDirection === 'asc' ? IconArrowUp : IconArrowDown
return (
<button
type="button"
onClick={() => toggleSort(key)}
className="flex items-center gap-1 font-semibold text-text-primary hover:text-desert-orange"
>
{label}
<Icon className="size-4" />
</button>
)
}
async function confirmDeleteFile(file: ZimFileWithMetadata) {
openModal(
<StyledModal
@ -83,7 +134,7 @@ export default function ZimPage() {
columns={[
{
accessor: 'title',
title: 'Title',
title: renderSortHeader('Title', 'name'),
render: (record) => (
<span className="font-medium">
{record.title || record.name}
@ -99,6 +150,15 @@ export default function ZimPage() {
</span>
),
},
{
accessor: 'size_bytes',
title: renderSortHeader('Size', 'size'),
render: (record) => (
<span className="text-text-secondary tabular-nums">
{record.size_bytes ? formatBytes(record.size_bytes, 1) : '—'}
</span>
),
},
{
accessor: 'actions',
title: 'Actions',
@ -117,7 +177,7 @@ export default function ZimPage() {
),
},
]}
data={data || []}
data={sortedData}
/>
</main>
</div>

View File

@ -0,0 +1,62 @@
import logger from '@adonisjs/core/services/logger'
import type { ApplicationService } from '@adonisjs/core/types'
/**
* Ensures the nomad_qdrant container has the `unless-stopped` restart policy.
*
* Existing installations may have been created before this policy was enforced
* in the service seeder. Docker allows updating a container's restart policy
* without recreating it via the container.update() API.
*
* This provider runs once on every admin startup. If the policy is already
* correct, the check is a no-op.
*/
export default class QdrantRestartPolicyProvider {
constructor(protected app: ApplicationService) {}
async boot() {
if (this.app.getEnvironment() !== 'web') return
setImmediate(async () => {
try {
const Service = (await import('#models/service')).default
const { SERVICE_NAMES } = await import('../constants/service_names.js')
const Docker = (await import('dockerode')).default
const qdrantService = await Service.query()
.where('service_name', SERVICE_NAMES.QDRANT)
.first()
if (!qdrantService?.installed) {
logger.info('[QdrantRestartPolicyProvider] Qdrant not installed — skipping restart policy check.')
return
}
const docker = new Docker({ socketPath: '/var/run/docker.sock' })
const containers = await docker.listContainers({ all: true })
const containerInfo = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.QDRANT}`))
if (!containerInfo) {
logger.warn('[QdrantRestartPolicyProvider] Qdrant container not found — skipping restart policy check.')
return
}
const container = docker.getContainer(containerInfo.Id)
const inspected = await container.inspect()
const currentPolicy = inspected.HostConfig?.RestartPolicy?.Name
if (currentPolicy === 'unless-stopped') {
logger.info('[QdrantRestartPolicyProvider] Qdrant already has unless-stopped restart policy — no update needed.')
return
}
logger.info(`[QdrantRestartPolicyProvider] Qdrant restart policy is "${currentPolicy ?? 'none'}" — updating to unless-stopped.`)
await container.update({ RestartPolicy: { Name: 'unless-stopped', MaximumRetryCount: 0 } })
logger.info('[QdrantRestartPolicyProvider] Qdrant restart policy updated successfully.')
} catch (err: any) {
logger.error(`[QdrantRestartPolicyProvider] Failed to update Qdrant restart policy: ${err.message}`)
// Non-fatal: the container will still run, just without auto-restart on crash.
}
})
}
}

View File

@ -143,6 +143,7 @@ router
router.delete('/failed-jobs', [RagController, 'cleanupFailedJobs'])
router.get('/job-status', [RagController, 'getJobStatus'])
router.post('/sync', [RagController, 'scanAndSync'])
router.get('/health', [RagController, 'health'])
})
.prefix('/api/rag')

View File

@ -86,6 +86,21 @@ check_is_debian_based() {
echo -e "${GREEN}#${RESET} This script is running on a Debian-based system.\\n"
}
check_is_x86_64() {
local arch
arch="$(uname -m)"
if [[ "${arch}" != "x86_64" && "${arch}" != "amd64" ]]; then
echo -e "${YELLOW}#${RESET} WARNING: Detected architecture '${arch}'. NOMAD officially supports x86_64 only.\\n"
echo -e "${YELLOW}#${RESET} ARM64/aarch64 support is tracked in PR #419 and is not yet ready.\\n"
echo -e "${YELLOW}#${RESET} Continuing on an unsupported architecture will likely fail and may leave\\n"
echo -e "${YELLOW}#${RESET} partial Docker images and files behind that you'll need to clean up manually.\\n"
echo -e "${YELLOW}#${RESET} Continuing in 10 seconds... press Ctrl+C now to abort.\\n"
sleep 10
return
fi
echo -e "${GREEN}#${RESET} Architecture check passed (${arch}).\\n"
}
ensure_dependencies_installed() {
local missing_deps=()
@ -539,6 +554,7 @@ success_message() {
# Pre-flight checks
check_is_debian_based
check_is_x86_64
check_is_bash
check_has_sudo
ensure_dependencies_installed