diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1fda7fb..46e0558 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,7 +30,14 @@ We are committed to providing a welcoming environment for everyone. Disrespectfu ## Before You Start -**Open an issue first.** Before writing any code, please [open an issue](../../issues/new) to discuss your proposed change. This helps avoid duplicate work and ensures your contribution aligns with the project's direction. +**Open an issue first.** Before writing any code for a non-trivial change, you must [open an issue](../../issues/new) to discuss your proposed change. This helps avoid duplicate work and ensures your contribution aligns with the project's direction. **Pull requests submitted without a corresponding issue may be closed at the maintainers' discretion.** + +**Trivial fixes are exempt** and may be submitted directly as a PR. Examples: +- Typo and grammar corrections +- Documentation clarifications +- Small one-line bug fixes with an obvious cause + +If you're not sure whether your change qualifies as trivial, open an issue first. When opening an issue: - Use a clear, descriptive title @@ -149,7 +156,7 @@ This project uses [Semantic Versioning](https://semver.org/). Versions are manag 2. Open a pull request against the `dev` branch of this repository 3. In the PR description: - Summarize what your changes do and why - - Reference the related issue (e.g., `Closes #123`) + - Reference the related issue (e.g., `Closes #123`) — required for non-trivial changes - Note any relevant testing steps or environment details 4. Be responsive to feedback — maintainers may request changes. Pull requests with no activity for an extended period may be closed. diff --git a/admin/adonisrc.ts b/admin/adonisrc.ts index 37046d2..a091ce2 100644 --- a/admin/adonisrc.ts +++ b/admin/adonisrc.ts @@ -55,6 +55,7 @@ export default defineConfig({ () => import('@adonisjs/transmit/transmit_provider'), () => import('#providers/map_static_provider'), () => import('#providers/kiwix_migration_provider'), + () => import('#providers/qdrant_restart_policy_provider'), ], /* diff --git a/admin/app/controllers/rag_controller.ts b/admin/app/controllers/rag_controller.ts index 149ba7e..c836393 100644 --- a/admin/app/controllers/rag_controller.ts +++ b/admin/app/controllers/rag_controller.ts @@ -97,4 +97,9 @@ export default class RagController { return response.status(500).json({ error: 'Error scanning and syncing storage' }) } } + + public async health({ response }: HttpContext) { + const result = await this.ragService.checkQdrantHealth() + return response.status(200).json(result) + } } diff --git a/admin/app/middleware/compression_middleware.ts b/admin/app/middleware/compression_middleware.ts index 0661ac7..9c11411 100644 --- a/admin/app/middleware/compression_middleware.ts +++ b/admin/app/middleware/compression_middleware.ts @@ -3,7 +3,21 @@ import type { HttpContext } from '@adonisjs/core/http' import type { NextFn } from '@adonisjs/core/types/http' import compression from 'compression' -const compress = env.get('DISABLE_COMPRESSION') ? null : compression() +// Skip compression for Server-Sent Events. The compression library buffers +// response writes to determine encoding, which collapses per-token streaming +// into a single block delivered after generation completes (regression in +// v1.31.0-rc.2, reported in #781 by @toasterking). +const compress = env.get('DISABLE_COMPRESSION') + ? null + : compression({ + filter: (req: any, res: any) => { + const contentType = res.getHeader('Content-Type') + if (typeof contentType === 'string' && contentType.includes('text/event-stream')) { + return false + } + return compression.filter(req, res) + }, + }) export default class CompressionMiddleware { async handle({ request, response }: HttpContext, next: NextFn) { diff --git a/admin/app/services/ollama_service.ts b/admin/app/services/ollama_service.ts index 27f5cac..fe0cb1c 100644 --- a/admin/app/services/ollama_service.ts +++ b/admin/app/services/ollama_service.ts @@ -480,10 +480,21 @@ export class OllamaService { } try { - // Prefer Ollama native endpoint (supports batch input natively) + // Prefer Ollama native endpoint (supports batch input natively). + // Pass num_ctx explicitly so we don't depend on the embedding model's + // modelfile defaults. Some installs ship nomic-embed-text:v1.5 with + // num_ctx=2048, which our chunker (sized for ~1500 tokens) can exceed + // on dense content, causing "input length exceeds context length" errors. + // truncate:true is a runtime safety net for any chunk that still overshoots. + // 8192 matches nomic-embed-text:v1.5's RoPE-extrapolated max. const response = await axios.post( `${this.baseUrl}/api/embed`, - { model, input }, + { + model, + input, + truncate: true, + options: { num_ctx: 8192 }, + }, { timeout: 60000 } ) // Some backends (e.g. LM Studio) return HTTP 200 for unknown endpoints with an incompatible diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 81145f8..bd5371d 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -52,14 +52,33 @@ export class RagService { this.qdrantInitPromise = (async () => { const qdrantUrl = await this.dockerService.getServiceURL(SERVICE_NAMES.QDRANT) if (!qdrantUrl) { - throw new Error('Qdrant service is not installed or running.') + throw new Error('Qdrant vector database is offline. Restart the AI Assistant service in Settings to restore the Knowledge Base.') } this.qdrant = new QdrantClient({ url: qdrantUrl }) - })() + })().catch((err) => { + this.qdrantInitPromise = null + this.qdrant = null + throw err + }) } return this.qdrantInitPromise } + public async checkQdrantHealth(): Promise<{ online: boolean; message?: string }> { + try { + await this._ensureDependencies() + await this.qdrant!.getCollections() + return { online: true } + } catch { + this.qdrant = null + this.qdrantInitPromise = null + return { + online: false, + message: 'Qdrant vector database is offline. Restart the AI Assistant service in Settings to restore the Knowledge Base.', + } + } + } + private async _ensureDependencies() { if (!this.qdrant) { await this._initializeQdrantClient() diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index c6f427c..db6b5b7 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -40,7 +40,21 @@ export class ZimService { await ensureDirectoryExists(dirPath) const all = await listDirectoryContents(dirPath) - const files = all.filter((item) => item.name.endsWith('.zim')) + const zimEntries = all.filter((item) => item.name.endsWith('.zim')) + + const files = await Promise.all( + zimEntries.map(async (entry) => { + const filePath = entry.type === 'file' ? entry.key : join(dirPath, entry.name) + const stats = await getFileStatsIfExists(filePath) + return { + ...entry, + title: null, + summary: null, + author: null, + size_bytes: stats ? Number(stats.size) : null, + } + }) + ) return { files, diff --git a/admin/docs/api-reference.md b/admin/docs/api-reference.md index 5cf126f..928b1e3 100644 --- a/admin/docs/api-reference.md +++ b/admin/docs/api-reference.md @@ -148,6 +148,15 @@ ZIM files provide offline Wikipedia, books, and other content via Kiwix. | POST | `/api/maps/download-collection` | Download an entire collection by slug (async) | | DELETE | `/api/maps/:filename` | Delete a local map file | +### Map Markers + +| Method | Path | Description | +|--------|------|-------------| +| GET | `/api/maps/markers` | List map markers | +| POST | `/api/maps/markers` | Add map marker (body: {"name": "Test Marker", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) | +| PATCH | `/api/maps/markers/{id}` | Update a map marker (body: {"name": "Test Marker", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) fields that don't change can be omitted| +| DELETE | `/api/maps/markers/{id}` | Delete a map marker | + --- ## Downloads diff --git a/admin/inertia/components/chat/KnowledgeBaseModal.tsx b/admin/inertia/components/chat/KnowledgeBaseModal.tsx index e77a0c9..6230398 100644 --- a/admin/inertia/components/chat/KnowledgeBaseModal.tsx +++ b/admin/inertia/components/chat/KnowledgeBaseModal.tsx @@ -1,5 +1,5 @@ import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' -import { useRef, useState } from 'react' +import { useEffect, useRef, useState } from 'react' import FileUploader from '~/components/file-uploader' import StyledButton from '~/components/StyledButton' import StyledSectionHeader from '~/components/StyledSectionHeader' @@ -10,6 +10,7 @@ import { IconX } from '@tabler/icons-react' import { useModals } from '~/context/ModalContext' import StyledModal from '../StyledModal' import ActiveEmbedJobs from '~/components/ActiveEmbedJobs' +import { SERVICE_NAMES } from '../../../constants/service_names' interface KnowledgeBaseModalProps { aiAssistantName?: string @@ -30,6 +31,19 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o const { openModal, closeModal } = useModals() const queryClient = useQueryClient() + const [isStartingQdrant, setIsStartingQdrant] = useState(false) + + const { data: healthStatus } = useQuery({ + queryKey: ['qdrantHealth'], + queryFn: () => api.checkRAGHealth(), + refetchInterval: isStartingQdrant ? 3_000 : 30_000, + }) + const qdrantOffline = healthStatus?.online === false + + useEffect(() => { + if (!qdrantOffline) setIsStartingQdrant(false) + }, [qdrantOffline]) + const { data: storedFiles = [], isLoading: isLoadingFiles } = useQuery({ queryKey: ['storedFiles'], queryFn: () => api.getStoredRAGFiles(), @@ -64,6 +78,17 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o }, }) + const startQdrantMutation = useMutation({ + mutationFn: () => api.affectService(SERVICE_NAMES.QDRANT, 'start'), + onSuccess: () => { + setIsStartingQdrant(true) + queryClient.invalidateQueries({ queryKey: ['qdrantHealth'] }) + }, + onError: (error: any) => { + addNotification({ type: 'error', message: error?.message || 'Failed to start Qdrant.' }) + }, + }) + const syncMutation = useMutation({ mutationFn: () => api.syncRAGStorage(), onSuccess: (data) => { @@ -149,6 +174,22 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
+ {qdrantOffline && ( +
+ + Knowledge Base unavailable: The Qdrant vector database is offline. + + startQdrantMutation.mutate()} + loading={startQdrantMutation.isPending || isStartingQdrant} + disabled={startQdrantMutation.isPending || isStartingQdrant} + > + {isStartingQdrant ? 'Starting…' : 'Start Qdrant'} + +
+ )}
Upload @@ -236,7 +277,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o icon="IconTrash" onClick={() => cleanupFailedMutation.mutate()} loading={cleanupFailedMutation.isPending} - disabled={cleanupFailedMutation.isPending} + disabled={cleanupFailedMutation.isPending || qdrantOffline} > Clean Up Failed @@ -252,7 +293,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o size="md" icon='IconRefresh' onClick={handleConfirmSync} - disabled={syncMutation.isPending || isUploading} + disabled={syncMutation.isPending || isUploading || qdrantOffline} loading={syncMutation.isPending || isUploading} > Sync Storage diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index dc1c7ed..0df95ae 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -451,6 +451,13 @@ class API { })() } + async checkRAGHealth() { + return catchInternal(async () => { + const response = await this.client.get<{ online: boolean; message?: string }>('/rag/health') + return response.data + })() + } + async getStoredRAGFiles() { return catchInternal(async () => { const response = await this.client.get<{ files: string[] }>('/rag/files') diff --git a/admin/inertia/pages/settings/zim/index.tsx b/admin/inertia/pages/settings/zim/index.tsx index 9d22937..68ae2b1 100644 --- a/admin/inertia/pages/settings/zim/index.tsx +++ b/admin/inertia/pages/settings/zim/index.tsx @@ -1,5 +1,6 @@ import { Head } from '@inertiajs/react' import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { useMemo, useState } from 'react' import StyledTable from '~/components/StyledTable' import SettingsLayout from '~/layouts/SettingsLayout' import api from '~/lib/api' @@ -10,11 +11,18 @@ import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus' import Alert from '~/components/Alert' import { ZimFileWithMetadata } from '../../../../types/zim' import { SERVICE_NAMES } from '../../../../constants/service_names' +import { formatBytes } from '~/lib/util' +import { IconArrowDown, IconArrowUp, IconArrowsSort } from '@tabler/icons-react' + +type SortKey = 'name' | 'size' +type SortDirection = 'asc' | 'desc' export default function ZimPage() { const queryClient = useQueryClient() const { openModal, closeAllModals } = useModals() const { isInstalled } = useServiceInstalledStatus(SERVICE_NAMES.KIWIX) + const [sortKey, setSortKey] = useState('size') + const [sortDirection, setSortDirection] = useState('desc') const { data, isLoading } = useQuery({ queryKey: ['zim-files'], queryFn: getFiles, @@ -25,6 +33,49 @@ export default function ZimPage() { return res.data.files } + const sortedData = useMemo(() => { + if (!data) return [] + const copy = [...data] + copy.sort((a, b) => { + let cmp = 0 + if (sortKey === 'size') { + const aSize = a.size_bytes ?? 0 + const bSize = b.size_bytes ?? 0 + cmp = aSize - bSize + } else { + const aName = (a.title || a.name).toLowerCase() + const bName = (b.title || b.name).toLowerCase() + cmp = aName.localeCompare(bName) + } + return sortDirection === 'asc' ? cmp : -cmp + }) + return copy + }, [data, sortKey, sortDirection]) + + function toggleSort(key: SortKey) { + if (sortKey === key) { + setSortDirection((d) => (d === 'asc' ? 'desc' : 'asc')) + } else { + setSortKey(key) + setSortDirection(key === 'size' ? 'desc' : 'asc') + } + } + + function renderSortHeader(label: string, key: SortKey) { + const active = sortKey === key + const Icon = !active ? IconArrowsSort : sortDirection === 'asc' ? IconArrowUp : IconArrowDown + return ( + + ) + } + async function confirmDeleteFile(file: ZimFileWithMetadata) { openModal( ( {record.title || record.name} @@ -99,6 +150,15 @@ export default function ZimPage() { ), }, + { + accessor: 'size_bytes', + title: renderSortHeader('Size', 'size'), + render: (record) => ( + + {record.size_bytes ? formatBytes(record.size_bytes, 1) : '—'} + + ), + }, { accessor: 'actions', title: 'Actions', @@ -117,7 +177,7 @@ export default function ZimPage() { ), }, ]} - data={data || []} + data={sortedData} />
diff --git a/admin/providers/qdrant_restart_policy_provider.ts b/admin/providers/qdrant_restart_policy_provider.ts new file mode 100644 index 0000000..b909242 --- /dev/null +++ b/admin/providers/qdrant_restart_policy_provider.ts @@ -0,0 +1,62 @@ +import logger from '@adonisjs/core/services/logger' +import type { ApplicationService } from '@adonisjs/core/types' + +/** + * Ensures the nomad_qdrant container has the `unless-stopped` restart policy. + * + * Existing installations may have been created before this policy was enforced + * in the service seeder. Docker allows updating a container's restart policy + * without recreating it via the container.update() API. + * + * This provider runs once on every admin startup. If the policy is already + * correct, the check is a no-op. + */ +export default class QdrantRestartPolicyProvider { + constructor(protected app: ApplicationService) {} + + async boot() { + if (this.app.getEnvironment() !== 'web') return + + setImmediate(async () => { + try { + const Service = (await import('#models/service')).default + const { SERVICE_NAMES } = await import('../constants/service_names.js') + const Docker = (await import('dockerode')).default + + const qdrantService = await Service.query() + .where('service_name', SERVICE_NAMES.QDRANT) + .first() + + if (!qdrantService?.installed) { + logger.info('[QdrantRestartPolicyProvider] Qdrant not installed — skipping restart policy check.') + return + } + + const docker = new Docker({ socketPath: '/var/run/docker.sock' }) + const containers = await docker.listContainers({ all: true }) + const containerInfo = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.QDRANT}`)) + + if (!containerInfo) { + logger.warn('[QdrantRestartPolicyProvider] Qdrant container not found — skipping restart policy check.') + return + } + + const container = docker.getContainer(containerInfo.Id) + const inspected = await container.inspect() + const currentPolicy = inspected.HostConfig?.RestartPolicy?.Name + + if (currentPolicy === 'unless-stopped') { + logger.info('[QdrantRestartPolicyProvider] Qdrant already has unless-stopped restart policy — no update needed.') + return + } + + logger.info(`[QdrantRestartPolicyProvider] Qdrant restart policy is "${currentPolicy ?? 'none'}" — updating to unless-stopped.`) + await container.update({ RestartPolicy: { Name: 'unless-stopped', MaximumRetryCount: 0 } }) + logger.info('[QdrantRestartPolicyProvider] Qdrant restart policy updated successfully.') + } catch (err: any) { + logger.error(`[QdrantRestartPolicyProvider] Failed to update Qdrant restart policy: ${err.message}`) + // Non-fatal: the container will still run, just without auto-restart on crash. + } + }) + } +} diff --git a/admin/start/routes.ts b/admin/start/routes.ts index d201174..07ee6b9 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -143,6 +143,7 @@ router router.delete('/failed-jobs', [RagController, 'cleanupFailedJobs']) router.get('/job-status', [RagController, 'getJobStatus']) router.post('/sync', [RagController, 'scanAndSync']) + router.get('/health', [RagController, 'health']) }) .prefix('/api/rag') diff --git a/install/install_nomad.sh b/install/install_nomad.sh index 76ca807..ced178f 100644 --- a/install/install_nomad.sh +++ b/install/install_nomad.sh @@ -86,6 +86,21 @@ check_is_debian_based() { echo -e "${GREEN}#${RESET} This script is running on a Debian-based system.\\n" } +check_is_x86_64() { + local arch + arch="$(uname -m)" + if [[ "${arch}" != "x86_64" && "${arch}" != "amd64" ]]; then + echo -e "${YELLOW}#${RESET} WARNING: Detected architecture '${arch}'. NOMAD officially supports x86_64 only.\\n" + echo -e "${YELLOW}#${RESET} ARM64/aarch64 support is tracked in PR #419 and is not yet ready.\\n" + echo -e "${YELLOW}#${RESET} Continuing on an unsupported architecture will likely fail and may leave\\n" + echo -e "${YELLOW}#${RESET} partial Docker images and files behind that you'll need to clean up manually.\\n" + echo -e "${YELLOW}#${RESET} Continuing in 10 seconds... press Ctrl+C now to abort.\\n" + sleep 10 + return + fi + echo -e "${GREEN}#${RESET} Architecture check passed (${arch}).\\n" +} + ensure_dependencies_installed() { local missing_deps=() @@ -539,6 +554,7 @@ success_message() { # Pre-flight checks check_is_debian_based +check_is_x86_64 check_is_bash check_has_sudo ensure_dependencies_installed