Merge branch 'dev' into feat/add-details-on-marker
This commit is contained in:
commit
845ebe09de
|
|
@ -30,7 +30,14 @@ We are committed to providing a welcoming environment for everyone. Disrespectfu
|
|||
|
||||
## Before You Start
|
||||
|
||||
**Open an issue first.** Before writing any code, please [open an issue](../../issues/new) to discuss your proposed change. This helps avoid duplicate work and ensures your contribution aligns with the project's direction.
|
||||
**Open an issue first.** Before writing any code for a non-trivial change, you must [open an issue](../../issues/new) to discuss your proposed change. This helps avoid duplicate work and ensures your contribution aligns with the project's direction. **Pull requests submitted without a corresponding issue may be closed at the maintainers' discretion.**
|
||||
|
||||
**Trivial fixes are exempt** and may be submitted directly as a PR. Examples:
|
||||
- Typo and grammar corrections
|
||||
- Documentation clarifications
|
||||
- Small one-line bug fixes with an obvious cause
|
||||
|
||||
If you're not sure whether your change qualifies as trivial, open an issue first.
|
||||
|
||||
When opening an issue:
|
||||
- Use a clear, descriptive title
|
||||
|
|
@ -149,7 +156,7 @@ This project uses [Semantic Versioning](https://semver.org/). Versions are manag
|
|||
2. Open a pull request against the `dev` branch of this repository
|
||||
3. In the PR description:
|
||||
- Summarize what your changes do and why
|
||||
- Reference the related issue (e.g., `Closes #123`)
|
||||
- Reference the related issue (e.g., `Closes #123`) — required for non-trivial changes
|
||||
- Note any relevant testing steps or environment details
|
||||
4. Be responsive to feedback — maintainers may request changes. Pull requests with no activity for an extended period may be closed.
|
||||
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ export default defineConfig({
|
|||
() => import('@adonisjs/transmit/transmit_provider'),
|
||||
() => import('#providers/map_static_provider'),
|
||||
() => import('#providers/kiwix_migration_provider'),
|
||||
() => import('#providers/qdrant_restart_policy_provider'),
|
||||
],
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -97,4 +97,9 @@ export default class RagController {
|
|||
return response.status(500).json({ error: 'Error scanning and syncing storage' })
|
||||
}
|
||||
}
|
||||
|
||||
public async health({ response }: HttpContext) {
|
||||
const result = await this.ragService.checkQdrantHealth()
|
||||
return response.status(200).json(result)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,21 @@ import type { HttpContext } from '@adonisjs/core/http'
|
|||
import type { NextFn } from '@adonisjs/core/types/http'
|
||||
import compression from 'compression'
|
||||
|
||||
const compress = env.get('DISABLE_COMPRESSION') ? null : compression()
|
||||
// Skip compression for Server-Sent Events. The compression library buffers
|
||||
// response writes to determine encoding, which collapses per-token streaming
|
||||
// into a single block delivered after generation completes (regression in
|
||||
// v1.31.0-rc.2, reported in #781 by @toasterking).
|
||||
const compress = env.get('DISABLE_COMPRESSION')
|
||||
? null
|
||||
: compression({
|
||||
filter: (req: any, res: any) => {
|
||||
const contentType = res.getHeader('Content-Type')
|
||||
if (typeof contentType === 'string' && contentType.includes('text/event-stream')) {
|
||||
return false
|
||||
}
|
||||
return compression.filter(req, res)
|
||||
},
|
||||
})
|
||||
|
||||
export default class CompressionMiddleware {
|
||||
async handle({ request, response }: HttpContext, next: NextFn) {
|
||||
|
|
|
|||
|
|
@ -480,10 +480,21 @@ export class OllamaService {
|
|||
}
|
||||
|
||||
try {
|
||||
// Prefer Ollama native endpoint (supports batch input natively)
|
||||
// Prefer Ollama native endpoint (supports batch input natively).
|
||||
// Pass num_ctx explicitly so we don't depend on the embedding model's
|
||||
// modelfile defaults. Some installs ship nomic-embed-text:v1.5 with
|
||||
// num_ctx=2048, which our chunker (sized for ~1500 tokens) can exceed
|
||||
// on dense content, causing "input length exceeds context length" errors.
|
||||
// truncate:true is a runtime safety net for any chunk that still overshoots.
|
||||
// 8192 matches nomic-embed-text:v1.5's RoPE-extrapolated max.
|
||||
const response = await axios.post(
|
||||
`${this.baseUrl}/api/embed`,
|
||||
{ model, input },
|
||||
{
|
||||
model,
|
||||
input,
|
||||
truncate: true,
|
||||
options: { num_ctx: 8192 },
|
||||
},
|
||||
{ timeout: 60000 }
|
||||
)
|
||||
// Some backends (e.g. LM Studio) return HTTP 200 for unknown endpoints with an incompatible
|
||||
|
|
|
|||
|
|
@ -52,14 +52,33 @@ export class RagService {
|
|||
this.qdrantInitPromise = (async () => {
|
||||
const qdrantUrl = await this.dockerService.getServiceURL(SERVICE_NAMES.QDRANT)
|
||||
if (!qdrantUrl) {
|
||||
throw new Error('Qdrant service is not installed or running.')
|
||||
throw new Error('Qdrant vector database is offline. Restart the AI Assistant service in Settings to restore the Knowledge Base.')
|
||||
}
|
||||
this.qdrant = new QdrantClient({ url: qdrantUrl })
|
||||
})()
|
||||
})().catch((err) => {
|
||||
this.qdrantInitPromise = null
|
||||
this.qdrant = null
|
||||
throw err
|
||||
})
|
||||
}
|
||||
return this.qdrantInitPromise
|
||||
}
|
||||
|
||||
public async checkQdrantHealth(): Promise<{ online: boolean; message?: string }> {
|
||||
try {
|
||||
await this._ensureDependencies()
|
||||
await this.qdrant!.getCollections()
|
||||
return { online: true }
|
||||
} catch {
|
||||
this.qdrant = null
|
||||
this.qdrantInitPromise = null
|
||||
return {
|
||||
online: false,
|
||||
message: 'Qdrant vector database is offline. Restart the AI Assistant service in Settings to restore the Knowledge Base.',
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async _ensureDependencies() {
|
||||
if (!this.qdrant) {
|
||||
await this._initializeQdrantClient()
|
||||
|
|
|
|||
|
|
@ -40,7 +40,21 @@ export class ZimService {
|
|||
await ensureDirectoryExists(dirPath)
|
||||
|
||||
const all = await listDirectoryContents(dirPath)
|
||||
const files = all.filter((item) => item.name.endsWith('.zim'))
|
||||
const zimEntries = all.filter((item) => item.name.endsWith('.zim'))
|
||||
|
||||
const files = await Promise.all(
|
||||
zimEntries.map(async (entry) => {
|
||||
const filePath = entry.type === 'file' ? entry.key : join(dirPath, entry.name)
|
||||
const stats = await getFileStatsIfExists(filePath)
|
||||
return {
|
||||
...entry,
|
||||
title: null,
|
||||
summary: null,
|
||||
author: null,
|
||||
size_bytes: stats ? Number(stats.size) : null,
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
return {
|
||||
files,
|
||||
|
|
|
|||
|
|
@ -148,6 +148,15 @@ ZIM files provide offline Wikipedia, books, and other content via Kiwix.
|
|||
| POST | `/api/maps/download-collection` | Download an entire collection by slug (async) |
|
||||
| DELETE | `/api/maps/:filename` | Delete a local map file |
|
||||
|
||||
### Map Markers
|
||||
|
||||
| Method | Path | Description |
|
||||
|--------|------|-------------|
|
||||
| GET | `/api/maps/markers` | List map markers |
|
||||
| POST | `/api/maps/markers` | Add map marker (body: {"name": "Test Marker", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) |
|
||||
| PATCH | `/api/maps/markers/{id}` | Update a map marker (body: {"name": "Test Marker", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) fields that don't change can be omitted|
|
||||
| DELETE | `/api/maps/markers/{id}` | Delete a map marker |
|
||||
|
||||
---
|
||||
|
||||
## Downloads
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
|
||||
import { useRef, useState } from 'react'
|
||||
import { useEffect, useRef, useState } from 'react'
|
||||
import FileUploader from '~/components/file-uploader'
|
||||
import StyledButton from '~/components/StyledButton'
|
||||
import StyledSectionHeader from '~/components/StyledSectionHeader'
|
||||
|
|
@ -10,6 +10,7 @@ import { IconX } from '@tabler/icons-react'
|
|||
import { useModals } from '~/context/ModalContext'
|
||||
import StyledModal from '../StyledModal'
|
||||
import ActiveEmbedJobs from '~/components/ActiveEmbedJobs'
|
||||
import { SERVICE_NAMES } from '../../../constants/service_names'
|
||||
|
||||
interface KnowledgeBaseModalProps {
|
||||
aiAssistantName?: string
|
||||
|
|
@ -30,6 +31,19 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
|
|||
const { openModal, closeModal } = useModals()
|
||||
const queryClient = useQueryClient()
|
||||
|
||||
const [isStartingQdrant, setIsStartingQdrant] = useState(false)
|
||||
|
||||
const { data: healthStatus } = useQuery({
|
||||
queryKey: ['qdrantHealth'],
|
||||
queryFn: () => api.checkRAGHealth(),
|
||||
refetchInterval: isStartingQdrant ? 3_000 : 30_000,
|
||||
})
|
||||
const qdrantOffline = healthStatus?.online === false
|
||||
|
||||
useEffect(() => {
|
||||
if (!qdrantOffline) setIsStartingQdrant(false)
|
||||
}, [qdrantOffline])
|
||||
|
||||
const { data: storedFiles = [], isLoading: isLoadingFiles } = useQuery({
|
||||
queryKey: ['storedFiles'],
|
||||
queryFn: () => api.getStoredRAGFiles(),
|
||||
|
|
@ -64,6 +78,17 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
|
|||
},
|
||||
})
|
||||
|
||||
const startQdrantMutation = useMutation({
|
||||
mutationFn: () => api.affectService(SERVICE_NAMES.QDRANT, 'start'),
|
||||
onSuccess: () => {
|
||||
setIsStartingQdrant(true)
|
||||
queryClient.invalidateQueries({ queryKey: ['qdrantHealth'] })
|
||||
},
|
||||
onError: (error: any) => {
|
||||
addNotification({ type: 'error', message: error?.message || 'Failed to start Qdrant.' })
|
||||
},
|
||||
})
|
||||
|
||||
const syncMutation = useMutation({
|
||||
mutationFn: () => api.syncRAGStorage(),
|
||||
onSuccess: (data) => {
|
||||
|
|
@ -149,6 +174,22 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
|
|||
</button>
|
||||
</div>
|
||||
<div className="overflow-y-auto flex-1 p-6">
|
||||
{qdrantOffline && (
|
||||
<div className="mb-4 p-4 bg-red-50 border border-red-200 rounded-lg text-red-700 text-sm dark:bg-red-950 dark:border-red-800 dark:text-red-300 flex items-center justify-between gap-4">
|
||||
<span>
|
||||
<strong>Knowledge Base unavailable:</strong> The Qdrant vector database is offline.
|
||||
</span>
|
||||
<StyledButton
|
||||
variant="danger"
|
||||
size="sm"
|
||||
onClick={() => startQdrantMutation.mutate()}
|
||||
loading={startQdrantMutation.isPending || isStartingQdrant}
|
||||
disabled={startQdrantMutation.isPending || isStartingQdrant}
|
||||
>
|
||||
{isStartingQdrant ? 'Starting…' : 'Start Qdrant'}
|
||||
</StyledButton>
|
||||
</div>
|
||||
)}
|
||||
<div className="bg-surface-primary rounded-lg border shadow-md overflow-hidden">
|
||||
<div className="p-6">
|
||||
<FileUploader
|
||||
|
|
@ -165,7 +206,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
|
|||
size="lg"
|
||||
icon="IconUpload"
|
||||
onClick={handleUpload}
|
||||
disabled={files.length === 0 || isUploading}
|
||||
disabled={files.length === 0 || isUploading || qdrantOffline}
|
||||
loading={isUploading}
|
||||
>
|
||||
Upload
|
||||
|
|
@ -236,7 +277,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
|
|||
icon="IconTrash"
|
||||
onClick={() => cleanupFailedMutation.mutate()}
|
||||
loading={cleanupFailedMutation.isPending}
|
||||
disabled={cleanupFailedMutation.isPending}
|
||||
disabled={cleanupFailedMutation.isPending || qdrantOffline}
|
||||
>
|
||||
Clean Up Failed
|
||||
</StyledButton>
|
||||
|
|
@ -252,7 +293,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
|
|||
size="md"
|
||||
icon='IconRefresh'
|
||||
onClick={handleConfirmSync}
|
||||
disabled={syncMutation.isPending || isUploading}
|
||||
disabled={syncMutation.isPending || isUploading || qdrantOffline}
|
||||
loading={syncMutation.isPending || isUploading}
|
||||
>
|
||||
Sync Storage
|
||||
|
|
|
|||
|
|
@ -451,6 +451,13 @@ class API {
|
|||
})()
|
||||
}
|
||||
|
||||
async checkRAGHealth() {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.get<{ online: boolean; message?: string }>('/rag/health')
|
||||
return response.data
|
||||
})()
|
||||
}
|
||||
|
||||
async getStoredRAGFiles() {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.get<{ files: string[] }>('/rag/files')
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { Head } from '@inertiajs/react'
|
||||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
|
||||
import { useMemo, useState } from 'react'
|
||||
import StyledTable from '~/components/StyledTable'
|
||||
import SettingsLayout from '~/layouts/SettingsLayout'
|
||||
import api from '~/lib/api'
|
||||
|
|
@ -10,11 +11,18 @@ import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus'
|
|||
import Alert from '~/components/Alert'
|
||||
import { ZimFileWithMetadata } from '../../../../types/zim'
|
||||
import { SERVICE_NAMES } from '../../../../constants/service_names'
|
||||
import { formatBytes } from '~/lib/util'
|
||||
import { IconArrowDown, IconArrowUp, IconArrowsSort } from '@tabler/icons-react'
|
||||
|
||||
type SortKey = 'name' | 'size'
|
||||
type SortDirection = 'asc' | 'desc'
|
||||
|
||||
export default function ZimPage() {
|
||||
const queryClient = useQueryClient()
|
||||
const { openModal, closeAllModals } = useModals()
|
||||
const { isInstalled } = useServiceInstalledStatus(SERVICE_NAMES.KIWIX)
|
||||
const [sortKey, setSortKey] = useState<SortKey>('size')
|
||||
const [sortDirection, setSortDirection] = useState<SortDirection>('desc')
|
||||
const { data, isLoading } = useQuery<ZimFileWithMetadata[]>({
|
||||
queryKey: ['zim-files'],
|
||||
queryFn: getFiles,
|
||||
|
|
@ -25,6 +33,49 @@ export default function ZimPage() {
|
|||
return res.data.files
|
||||
}
|
||||
|
||||
const sortedData = useMemo(() => {
|
||||
if (!data) return []
|
||||
const copy = [...data]
|
||||
copy.sort((a, b) => {
|
||||
let cmp = 0
|
||||
if (sortKey === 'size') {
|
||||
const aSize = a.size_bytes ?? 0
|
||||
const bSize = b.size_bytes ?? 0
|
||||
cmp = aSize - bSize
|
||||
} else {
|
||||
const aName = (a.title || a.name).toLowerCase()
|
||||
const bName = (b.title || b.name).toLowerCase()
|
||||
cmp = aName.localeCompare(bName)
|
||||
}
|
||||
return sortDirection === 'asc' ? cmp : -cmp
|
||||
})
|
||||
return copy
|
||||
}, [data, sortKey, sortDirection])
|
||||
|
||||
function toggleSort(key: SortKey) {
|
||||
if (sortKey === key) {
|
||||
setSortDirection((d) => (d === 'asc' ? 'desc' : 'asc'))
|
||||
} else {
|
||||
setSortKey(key)
|
||||
setSortDirection(key === 'size' ? 'desc' : 'asc')
|
||||
}
|
||||
}
|
||||
|
||||
function renderSortHeader(label: string, key: SortKey) {
|
||||
const active = sortKey === key
|
||||
const Icon = !active ? IconArrowsSort : sortDirection === 'asc' ? IconArrowUp : IconArrowDown
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => toggleSort(key)}
|
||||
className="flex items-center gap-1 font-semibold text-text-primary hover:text-desert-orange"
|
||||
>
|
||||
{label}
|
||||
<Icon className="size-4" />
|
||||
</button>
|
||||
)
|
||||
}
|
||||
|
||||
async function confirmDeleteFile(file: ZimFileWithMetadata) {
|
||||
openModal(
|
||||
<StyledModal
|
||||
|
|
@ -83,7 +134,7 @@ export default function ZimPage() {
|
|||
columns={[
|
||||
{
|
||||
accessor: 'title',
|
||||
title: 'Title',
|
||||
title: renderSortHeader('Title', 'name'),
|
||||
render: (record) => (
|
||||
<span className="font-medium">
|
||||
{record.title || record.name}
|
||||
|
|
@ -99,6 +150,15 @@ export default function ZimPage() {
|
|||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
accessor: 'size_bytes',
|
||||
title: renderSortHeader('Size', 'size'),
|
||||
render: (record) => (
|
||||
<span className="text-text-secondary tabular-nums">
|
||||
{record.size_bytes ? formatBytes(record.size_bytes, 1) : '—'}
|
||||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
accessor: 'actions',
|
||||
title: 'Actions',
|
||||
|
|
@ -117,7 +177,7 @@ export default function ZimPage() {
|
|||
),
|
||||
},
|
||||
]}
|
||||
data={data || []}
|
||||
data={sortedData}
|
||||
/>
|
||||
</main>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,62 @@
|
|||
import logger from '@adonisjs/core/services/logger'
|
||||
import type { ApplicationService } from '@adonisjs/core/types'
|
||||
|
||||
/**
|
||||
* Ensures the nomad_qdrant container has the `unless-stopped` restart policy.
|
||||
*
|
||||
* Existing installations may have been created before this policy was enforced
|
||||
* in the service seeder. Docker allows updating a container's restart policy
|
||||
* without recreating it via the container.update() API.
|
||||
*
|
||||
* This provider runs once on every admin startup. If the policy is already
|
||||
* correct, the check is a no-op.
|
||||
*/
|
||||
export default class QdrantRestartPolicyProvider {
|
||||
constructor(protected app: ApplicationService) {}
|
||||
|
||||
async boot() {
|
||||
if (this.app.getEnvironment() !== 'web') return
|
||||
|
||||
setImmediate(async () => {
|
||||
try {
|
||||
const Service = (await import('#models/service')).default
|
||||
const { SERVICE_NAMES } = await import('../constants/service_names.js')
|
||||
const Docker = (await import('dockerode')).default
|
||||
|
||||
const qdrantService = await Service.query()
|
||||
.where('service_name', SERVICE_NAMES.QDRANT)
|
||||
.first()
|
||||
|
||||
if (!qdrantService?.installed) {
|
||||
logger.info('[QdrantRestartPolicyProvider] Qdrant not installed — skipping restart policy check.')
|
||||
return
|
||||
}
|
||||
|
||||
const docker = new Docker({ socketPath: '/var/run/docker.sock' })
|
||||
const containers = await docker.listContainers({ all: true })
|
||||
const containerInfo = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.QDRANT}`))
|
||||
|
||||
if (!containerInfo) {
|
||||
logger.warn('[QdrantRestartPolicyProvider] Qdrant container not found — skipping restart policy check.')
|
||||
return
|
||||
}
|
||||
|
||||
const container = docker.getContainer(containerInfo.Id)
|
||||
const inspected = await container.inspect()
|
||||
const currentPolicy = inspected.HostConfig?.RestartPolicy?.Name
|
||||
|
||||
if (currentPolicy === 'unless-stopped') {
|
||||
logger.info('[QdrantRestartPolicyProvider] Qdrant already has unless-stopped restart policy — no update needed.')
|
||||
return
|
||||
}
|
||||
|
||||
logger.info(`[QdrantRestartPolicyProvider] Qdrant restart policy is "${currentPolicy ?? 'none'}" — updating to unless-stopped.`)
|
||||
await container.update({ RestartPolicy: { Name: 'unless-stopped', MaximumRetryCount: 0 } })
|
||||
logger.info('[QdrantRestartPolicyProvider] Qdrant restart policy updated successfully.')
|
||||
} catch (err: any) {
|
||||
logger.error(`[QdrantRestartPolicyProvider] Failed to update Qdrant restart policy: ${err.message}`)
|
||||
// Non-fatal: the container will still run, just without auto-restart on crash.
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -143,6 +143,7 @@ router
|
|||
router.delete('/failed-jobs', [RagController, 'cleanupFailedJobs'])
|
||||
router.get('/job-status', [RagController, 'getJobStatus'])
|
||||
router.post('/sync', [RagController, 'scanAndSync'])
|
||||
router.get('/health', [RagController, 'health'])
|
||||
})
|
||||
.prefix('/api/rag')
|
||||
|
||||
|
|
|
|||
|
|
@ -86,6 +86,21 @@ check_is_debian_based() {
|
|||
echo -e "${GREEN}#${RESET} This script is running on a Debian-based system.\\n"
|
||||
}
|
||||
|
||||
check_is_x86_64() {
|
||||
local arch
|
||||
arch="$(uname -m)"
|
||||
if [[ "${arch}" != "x86_64" && "${arch}" != "amd64" ]]; then
|
||||
echo -e "${YELLOW}#${RESET} WARNING: Detected architecture '${arch}'. NOMAD officially supports x86_64 only.\\n"
|
||||
echo -e "${YELLOW}#${RESET} ARM64/aarch64 support is tracked in PR #419 and is not yet ready.\\n"
|
||||
echo -e "${YELLOW}#${RESET} Continuing on an unsupported architecture will likely fail and may leave\\n"
|
||||
echo -e "${YELLOW}#${RESET} partial Docker images and files behind that you'll need to clean up manually.\\n"
|
||||
echo -e "${YELLOW}#${RESET} Continuing in 10 seconds... press Ctrl+C now to abort.\\n"
|
||||
sleep 10
|
||||
return
|
||||
fi
|
||||
echo -e "${GREEN}#${RESET} Architecture check passed (${arch}).\\n"
|
||||
}
|
||||
|
||||
ensure_dependencies_installed() {
|
||||
local missing_deps=()
|
||||
|
||||
|
|
@ -539,6 +554,7 @@ success_message() {
|
|||
|
||||
# Pre-flight checks
|
||||
check_is_debian_based
|
||||
check_is_x86_64
|
||||
check_is_bash
|
||||
check_has_sudo
|
||||
ensure_dependencies_installed
|
||||
|
|
|
|||
Loading…
Reference in New Issue