RomanNum3ral
/
project-nomad
mirror of https://github.com/Crosstalk-Solutions/project-nomad.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(zim): add zim uploader in content manager 

adds a collapsible file uploader to accept zim file uploads into kiwix.
This commit is contained in:
Henry Estela 2026-04-19 22:51:50 -07:00 committed by jakeaturner
parent fe6735fdcb
commit a7d05859d4
No known key found for this signature in database
GPG Key ID: B1072EBDEECE328D
9 changed files with 397 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
admin/app/controllers/zim_controller.ts
View File

@ -8,7 +8,12 @@ import {
} from '#validators/common'
import { addCustomLibraryValidator, browseLibraryValidator, idParamValidator, listRemoteZimValidator } from '#validators/zim'
import { inject } from '@adonisjs/core'
import logger from '@adonisjs/core/services/logger'
import type { HttpContext } from '@adonisjs/core/http'
import { createWriteStream } from 'fs'
import { rename } from 'fs/promises'
import { join, resolve, sep } from 'path'
import { ZIM_STORAGE_PATH, ensureDirectoryExists, sanitizeFilename } from '../utils/fs.js'
@inject()
export default class ZimController {
@ -83,6 +88,87 @@ export default class ZimController {
}
}
async upload({ request, response }: HttpContext) {
let filename: string | null = null
let tmpPath: string | null = null
let uploadError: string | null = null
try {
const basePath = resolve(join(process.cwd(), ZIM_STORAGE_PATH))
await ensureDirectoryExists(basePath)
request.multipart.onFile('*', {}, async (part) => {
const clientName = part.filename || ''
if (!clientName.toLowerCase().endsWith('.zim')) {
part.resume()
uploadError = 'INVALID_TYPE'
return
}
const sanitized = sanitizeFilename(clientName)
const finalPath = resolve(join(basePath, sanitized))
if (!finalPath.startsWith(basePath + sep)) {
part.resume()
uploadError = 'INVALID_FILENAME'
return
}
const { access } = await import('fs/promises')
const exists = await access(finalPath).then(() => true).catch(() => false)
if (exists) {
part.resume()
uploadError = 'DUPLICATE_FILENAME'
return
}
filename = sanitized
tmpPath = finalPath + '.tmp'
const ws = createWriteStream(tmpPath)
await new Promise<void>((res, rej) => {
ws.on('error', rej)
ws.on('finish', res)
part.on('error', rej)
part.pipe(ws)
})
await rename(tmpPath, finalPath)
tmpPath = null
})
await request.multipart.process()
if (uploadError === 'INVALID_TYPE') {
return response.status(422).send({ message: 'Only .zim files are accepted' })
}
if (uploadError === 'INVALID_FILENAME') {
return response.status(422).send({ message: 'Invalid filename' })
}
if (uploadError === 'DUPLICATE_FILENAME') {
return response.status(409).send({ message: 'A ZIM file with that name already exists' })
}
if (!filename) {
return response.status(400).send({ message: 'No file received' })
}
const { added } = await this.zimService.registerLocalUpload(filename)
return response.status(201).send({
message: 'ZIM file uploaded and registered successfully',
filename,
added,
})
} catch (error) {
logger.error('[ZimController] Upload failed:', error)
if (tmpPath) {
const { unlink } = await import('fs/promises')
await unlink(tmpPath).catch(() => {})
}
return response.status(500).send({ message: 'Upload failed' })
}
}
// Wikipedia selector endpoints
async getWikipediaState({}: HttpContext) {

107
admin/app/services/zim_service.ts
View File

@ -25,6 +25,7 @@ import vine from '@vinejs/vine'
import { wikipediaOptionsFileSchema } from '#validators/curated_collections'
import WikipediaSelection from '#models/wikipedia_selection'
import InstalledResource from '#models/installed_resource'
import CollectionManifest from '#models/collection_manifest'
import { RunDownloadJob } from '#jobs/run_download_job'
import { SERVICE_NAMES } from '../../constants/service_names.js'
import { CollectionManifestService } from './collection_manifest_service.js'
@ -469,6 +470,97 @@ export class ZimService {
return { before, after, added: Math.max(0, after - before) }
}
async registerLocalUpload(filename: string): Promise<{ added: number }> {
let added = 0
try {
const result = await this.rescanLibrary()
added = result.added
} catch (err) {
logger.error('[ZimService] Failed to rebuild kiwix library after local upload:', err)
}
const parsed = CollectionManifestService.parseZimFilename(filename)
if (parsed) {
const filepath = join(process.cwd(), ZIM_STORAGE_PATH, filename)
const stats = await getFileStatsIfExists(filepath)
try {
const { DateTime } = await import('luxon')
await InstalledResource.updateOrCreate(
{ resource_id: parsed.resource_id, resource_type: 'zim' },
{
version: parsed.version,
url: `local-upload://${filename}`,
file_path: filepath,
file_size_bytes: stats ? Number(stats.size) : null,
installed_at: DateTime.now(),
}
)
} catch (error) {
logger.error(`[ZimService] Failed to create InstalledResource for ${filename}:`, error)
}
}
// If the uploaded file matches a known Wikipedia option, mark it as installed
try {
const manifest = await CollectionManifest.find('wikipedia')
if (manifest) {
const spec = manifest.spec_data as { options: Array<{ id: string; url: string | null }> }
const matchedOption = spec.options.find(
(opt) => opt.url && opt.url.split('/').pop() === filename
)
if (matchedOption && matchedOption.url) {
const existing = await WikipediaSelection.query().first()
if (existing) {
existing.option_id = matchedOption.id
existing.url = matchedOption.url
existing.filename = filename
existing.status = 'installed'
await existing.save()
} else {
await WikipediaSelection.create({
option_id: matchedOption.id,
url: matchedOption.url,
filename,
status: 'installed',
})
}
logger.info(`[ZimService] Marked Wikipedia option '${matchedOption.id}' as installed from local upload`)
// Remove any other wikipedia_en_*.zim files, same as the download flow
const allFiles = await this.list()
const staleWikipediaFiles = allFiles.files.filter(
(f) => f.name.startsWith('wikipedia_en_') && f.name !== filename
)
for (const stale of staleWikipediaFiles) {
try {
await this.delete(stale.name)
logger.info(`[ZimService] Deleted stale Wikipedia file after upload: ${stale.name}`)
} catch (err) {
logger.warn(`[ZimService] Could not delete stale Wikipedia file: ${stale.name}`, err)
}
}
}
}
} catch (error) {
logger.error(`[ZimService] Failed to update WikipediaSelection for ${filename}:`, error)
}
const ollamaUrl = await this.dockerService.getServiceURL('nomad_ollama')
if (ollamaUrl) {
try {
const { EmbedFileJob } = await import('#jobs/embed_file_job')
await EmbedFileJob.dispatch({
fileName: filename,
filePath: join(process.cwd(), ZIM_STORAGE_PATH, filename),
})
} catch (error) {
logger.error(`[ZimService] EmbedFileJob dispatch failed after local upload:`, error)
}
}
return { added }
}
async delete(file: string): Promise<void> {
let fileName = file
if (!fileName.endsWith('.zim')) {
@ -505,6 +597,21 @@ export class ZimService {
.delete()
logger.info(`[ZimService] Deleted InstalledResource entry for: ${parsed.resource_id}`)
}
// If this file was the active Wikipedia selection, clear the selection
try {
const selection = await WikipediaSelection.query().first()
if (selection && selection.filename === fileName) {
selection.option_id = 'none'
selection.status = 'none'
selection.filename = null
selection.url = null
await selection.save()
logger.info(`[ZimService] Cleared WikipediaSelection after deleting ${fileName}`)
}
} catch (error) {
logger.error(`[ZimService] Failed to clear WikipediaSelection after deleting ${fileName}:`, error)
}
}
// Wikipedia selector methods

2
admin/config/bodyparser.ts
View File

@ -41,7 +41,7 @@ const bodyParserConfig = defineConfig({
*/
autoProcess: true,
convertEmptyStringsToNull: true,
processManually: [],
processManually: ['/api/zim/upload'],
/**
* Maximum limit of data to parse including all files

77
admin/inertia/components/ZimUploader/index.tsx Normal file
View File

@ -0,0 +1,77 @@
import Uppy from '@uppy/core'
import Dashboard from '@uppy/react/dashboard'
import XHRUpload from '@uppy/xhr-upload'
import '@uppy/core/css/style.min.css'
import '@uppy/dashboard/css/style.min.css'
import { useEffect, useRef, useState } from 'react'
interface ZimUploaderProps {
onUploadComplete: (added: number) => void
existingFilenames: string[]
}
export default function ZimUploader({ onUploadComplete, existingFilenames }: ZimUploaderProps) {
const existingFilenamesRef = useRef(existingFilenames)
useEffect(() => {
existingFilenamesRef.current = existingFilenames
}, [existingFilenames])
const [uppy] = useState(() =>
new Uppy({
restrictions: {
maxNumberOfFiles: 5,
allowedFileTypes: ['.zim'],
},
autoProceed: false,
}).use(XHRUpload, {
endpoint: '/api/zim/upload',
fieldName: 'file',
getResponseError: (responseText) => {
try {
const body = JSON.parse(responseText)
if (body?.message) return new Error(body.message)
} catch {}
return new Error('Upload failed')
},
})
)
useEffect(() => {
const handleFileAdded = (file: { id: string; name: string }) => {
if (existingFilenamesRef.current.includes(file.name)) {
uppy.removeFile(file.id)
uppy.info('A ZIM file with that name already exists', 'error', 6000)
return
}
const isWikipedia = (name: string) => name.startsWith('wikipedia_en_')
if (isWikipedia(file.name)) {
const alreadyQueued = uppy.getFiles().some((f) => f.id !== file.id && isWikipedia(f.name))
if (alreadyQueued) {
uppy.removeFile(file.id)
uppy.info('Only one Wikipedia file can be uploaded at a time', 'error', 6000)
}
}
}
const handleComplete = (result: { successful: Array<{ response?: { body?: { added?: number } } }> }) => {
const added = result.successful.reduce((sum, f) => sum + (f.response?.body?.added ?? 0), 0)
onUploadComplete(added)
}
uppy.on('file-added', handleFileAdded)
uppy.on('complete', handleComplete)
return () => {
uppy.off('file-added', handleFileAdded)
uppy.off('complete', handleComplete)
uppy.destroy()
}
}, [uppy, onUploadComplete])
return (
<Dashboard
uppy={uppy}
width="100%"
height={300}
note="ZIM files only. Large files (up to 20 GB) are supported. For best results, upload from the same machine or over a stable LAN connection. Larger files should be copied directly to the storage volume"
/>
)
}

48
admin/inertia/pages/settings/zim/index.tsx
View File

@ -10,6 +10,7 @@ import StyledModal from '~/components/StyledModal'
import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus'
import Alert from '~/components/Alert'
import { useNotifications } from '~/context/NotificationContext'
import ZimUploader from '~/components/ZimUploader'
import { ZimFileWithMetadata } from '../../../../types/zim'
import { SERVICE_NAMES } from '../../../../constants/service_names'
import { formatBytes } from '~/lib/util'
@ -25,6 +26,7 @@ export default function ZimPage() {
const { isInstalled } = useServiceInstalledStatus(SERVICE_NAMES.KIWIX)
const [sortKey, setSortKey] = useState<SortKey>('size')
const [sortDirection, setSortDirection] = useState<SortDirection>('desc')
const [showUploader, setShowUploader] = useState(false)
const { data, isLoading } = useQuery<ZimFileWithMetadata[]>({
queryKey: ['zim-files'],
queryFn: getFiles,
@ -135,18 +137,50 @@ export default function ZimPage() {
Manage your stored content files.
</p>
</div>
{isInstalled && (
<div className="flex items-center gap-2">
<StyledButton
variant="secondary"
icon={'IconRefresh'}
loading={rescanMutation.isPending}
title="Rebuild the Kiwix library index from the files on disk. Use this after manually adding ZIM files outside of NOMAD."
onClick={() => rescanMutation.mutate()}
icon={showUploader ? 'IconX' : 'IconUpload'}
onClick={() => setShowUploader((v) => !v)}
>
Rescan Library
{showUploader ? 'Hide Uploader' : 'Upload ZIM File'}
</StyledButton>
)}
{isInstalled && (
<StyledButton
variant="secondary"
icon={'IconRefresh'}
loading={rescanMutation.isPending}
title="Rebuild the Kiwix library index from the files on disk. Use this after manually adding ZIM files outside of NOMAD."
onClick={() => rescanMutation.mutate()}
>
Rescan Library
</StyledButton>
)}
</div>
</div>
{showUploader && (
<div className="mt-6">
<p className="text-text-muted text-sm mb-3">
Upload a ZIM file from your browser. Files up to 20 GB are supported. For best results upload from the same machine or over a stable LAN connection. Larger files should be copied directly to the storage volume.
</p>
<ZimUploader
existingFilenames={data?.map((f) => f.name) ?? []}
onUploadComplete={(added) => {
queryClient.invalidateQueries({ queryKey: ['zim-files'] })
queryClient.invalidateQueries({ queryKey: ['wikipedia-state'] })
queryClient.invalidateQueries({ queryKey: ['curated-categories'] })
setShowUploader(false)
addNotification({
type: 'success',
message:
added > 0
? `Upload complete. ${added} new ${added === 1 ? 'book' : 'books'} added to the library.`
: 'Upload complete. Library is up to date.',
})
}}
/>
</div>
)}
{!isInstalled && (
<Alert
title="The Kiwix application is not installed. Please install it to view downloaded ZIM files"

18
admin/inertia/pages/settings/zim/remote-explorer.tsx
View File

@ -40,10 +40,12 @@ import type { CategoryWithStatus, SpecTier } from '../../../../types/collections
import useDownloads from '~/hooks/useDownloads'
import ActiveDownloads from '~/components/ActiveDownloads'
import { SERVICE_NAMES } from '../../../../constants/service_names'
import { ZimFileWithMetadata } from '../../../../types/zim'
const CURATED_CATEGORIES_KEY = 'curated-categories'
const WIKIPEDIA_STATE_KEY = 'wikipedia-state'
const CUSTOM_LIBRARIES_KEY = 'custom-libraries'
const ZIM_FILES_KEY = 'zim-files'
type CustomLibrary = { id: number; name: string; base_url: string; is_default: boolean }
type BrowseResult = {
@ -104,6 +106,15 @@ export default function ZimRemoteExplorer() {
refetchOnWindowFocus: false,
})
const { data: localFiles } = useQuery<ZimFileWithMetadata[]>({
queryKey: [ZIM_FILES_KEY],
queryFn: async () => {
const res = await api.listZimFiles()
return res.data.files
},
refetchOnWindowFocus: false,
})
const { data: downloads, invalidate: invalidateDownloads } = useDownloads({
filetype: 'zim',
enabled: true,
@ -152,15 +163,16 @@ export default function ZimRemoteExplorer() {
const flatData = useMemo(() => {
const mapped = data?.pages.flatMap((page) => page.items) || []
// remove items that are currently downloading
const localNames = new Set(localFiles?.map((f) => f.name) ?? [])
return mapped.filter((item) => {
const isDownloading = downloads?.some((download) => {
const filename = item.download_url.split('/').pop()
return filename && download.filepath.endsWith(filename)
})
return !isDownloading
const isPresent = localNames.has(item.file_name)
return !isDownloading && !isPresent
})
}, [data, downloads])
}, [data, downloads, localFiles])
const hasMore = useMemo(() => data?.pages[data.pages.length - 1]?.has_more || false, [data])
const fetchOnBottomReached = useCallback(

69
admin/package-lock.json generated
View File

@ -4589,6 +4589,7 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4605,6 +4606,7 @@
"cpu": [
"x64"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4621,6 +4623,7 @@
"cpu": [
"arm"
],
"dev": true,
"license": "Apache-2.0",
"optional": true,
"os": [
@ -4713,6 +4716,7 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4729,6 +4733,7 @@
"cpu": [
"ia32"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4745,6 +4750,7 @@
"cpu": [
"x64"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -5684,6 +5690,12 @@
"@types/react": "^19.2.0"
}
},
"node_modules/@types/retry": {
"version": "0.12.2",
"resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.2.tgz",
"integrity": "sha512-XISRgDJ2Tc5q4TRqvgJtzsRkFYNJzZrhTdtMoGVBttwzzQJkPnS3WWTFc7kuDRoPtPakl+T+OfdEUjYJj7Jbow==",
"license": "MIT"
},
"node_modules/@types/send": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz",
@ -6030,6 +6042,20 @@
"integrity": "sha512-mUFwbeTqrVgDQxFveS+df2yfap6iuP20NAKAsBt5jDEoOTDew+zwLAOilHCeQJOVSvmgCX4ogqIrA0mnyr08yQ==",
"license": "ISC"
},
"node_modules/@uppy/companion-client": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/@uppy/companion-client/-/companion-client-5.1.1.tgz",
"integrity": "sha512-DzrOWTbIZHvtgAFXBMYHk2wD27NjpBSVhY2tEiEIUhPd2CxbFRZjHM/N3HOt3VwZEAP471QWFLlJRWPcIY3A2Q==",
"license": "MIT",
"dependencies": {
"@uppy/utils": "^7.1.1",
"namespace-emitter": "^2.0.1",
"p-retry": "^6.1.0"
},
"peerDependencies": {
"@uppy/core": "^5.1.1"
}
},
"node_modules/@uppy/components": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@uppy/components/-/components-1.2.0.tgz",
@ -6175,6 +6201,19 @@
"preact": "^10.26.10"
}
},
"node_modules/@uppy/xhr-upload": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/@uppy/xhr-upload/-/xhr-upload-5.2.0.tgz",
"integrity": "sha512-3LV/X5Of6BINnKplP+CwUJ0a4/7cRFfzxwGyXnW+uCrNQHoo09dttcz3begWHejGvzenQHuUnMO3Fxyc71Pryg==",
"license": "MIT",
"dependencies": {
"@uppy/companion-client": "^5.1.1",
"@uppy/utils": "^7.2.0"
},
"peerDependencies": {
"@uppy/core": "^5.2.0"
}
},
"node_modules/@vavite/multibuild": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/@vavite/multibuild/-/multibuild-5.1.0.tgz",
@ -10514,6 +10553,18 @@
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/is-network-error": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.3.1.tgz",
"integrity": "sha512-6QCxa49rQbmUWLfk0nuGqzql9U8uaV2H6279bRErPBHe/109hCzsLUBUHfbEtvLIHBd6hyXbgedBSHevm43Edw==",
"license": "MIT",
"engines": {
"node": ">=16"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/is-number": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
@ -13346,6 +13397,23 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/p-retry": {
"version": "6.2.1",
"resolved": "https://registry.npmjs.org/p-retry/-/p-retry-6.2.1.tgz",
"integrity": "sha512-hEt02O4hUct5wtwg4H4KcWgDdm+l1bOaEy/hWzd8xtXB9BqxTWBBhb+2ImAtH4Cv4rPjV76xN3Zumqk3k3AhhQ==",
"license": "MIT",
"dependencies": {
"@types/retry": "0.12.2",
"is-network-error": "^1.0.0",
"retry": "^0.13.1"
},
"engines": {
"node": ">=16.17"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/p-timeout": {
"version": "6.1.4",
"resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.4.tgz",
@ -14829,7 +14897,6 @@
"version": "0.13.1",
"resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz",
"integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==",
"devOptional": true,
"license": "MIT",
"engines": {
"node": ">= 4"

1
admin/package.json
View File

@ -88,6 +88,7 @@
"@uppy/core": "5.2.0",
"@uppy/dashboard": "5.1.0",
"@uppy/react": "5.1.1",
"@uppy/xhr-upload": "5.2.0",
"@vinejs/vine": "3.0.1",
"@vitejs/plugin-react": "4.7.0",
"autoprefixer": "10.5.0",

1
admin/start/routes.ts
View File

@ -206,6 +206,7 @@ router
router.post('/download-remote', [ZimController, 'downloadRemote'])
router.post('/download-category-tier', [ZimController, 'downloadCategoryTier'])
router.post('/upload', [ZimController, 'upload'])
router.get('/wikipedia', [ZimController, 'getWikipediaState'])
router.post('/wikipedia/select', [ZimController, 'selectWikipedia'])