diff --git a/admin/app/validators/common.ts b/admin/app/validators/common.ts index f98aa85..f9f4e63 100644 --- a/admin/app/validators/common.ts +++ b/admin/app/validators/common.ts @@ -1,4 +1,5 @@ import vine from '@vinejs/vine' +import ipaddr from 'ipaddr.js' /** * Checks whether a URL points to a loopback or link-local address. @@ -47,6 +48,16 @@ export function assertNotPrivateUrl(urlString: string): void { * IAM creds on a misconfigured cloud VM * - non-HTTP schemes (file:, gopher:, etc.) */ +// Canonical cloud instance-metadata addresses. AWS, GCP, Azure, DigitalOcean, +// Oracle Cloud, and Alibaba all expose IMDS at 169.254.169.254 over IPv4; +// AWS additionally exposes it at fd00:ec2::254 over IPv6. +// Compared after `ipaddr.toNormalizedString()`, which expands IPv6 to its +// fully-zero-padded form (e.g. `fd00:ec2::254` → `fd00:ec2:0:0:0:0:0:254`). +const BLOCKED_METADATA_IPV4 = new Set(['169.254.169.254']) +const BLOCKED_METADATA_IPV6 = new Set([ + ipaddr.parse('fd00:ec2::254').toNormalizedString(), +]) + export function assertNotCloudMetadataUrl(urlString: string): void { const parsed = new URL(urlString) @@ -54,13 +65,30 @@ export function assertNotCloudMetadataUrl(urlString: string): void { throw new Error(`URL must use http or https scheme: ${parsed.protocol}`) } + // Node's WHATWG URL parser keeps the brackets on IPv6 literals + // (`http://[::1]/` → hostname `[::1]`), so strip them before parsing. const hostname = parsed.hostname.toLowerCase().replace(/^\[|\]$/g, '') - if ( - hostname === '169.254.169.254' || - hostname === 'fd00:ec2::254' // IPv6 EC2 IMDS - ) { - throw new Error(`URL must not point to the cloud instance metadata endpoint: ${hostname}`) + // If the hostname isn't an IP literal it's a DNS name; allow it. (DNS + // rebinding is out of scope here — would require resolving and re-checking + // at fetch time.) + if (!ipaddr.isValid(hostname)) return + + let addr = ipaddr.parse(hostname) + + // Unwrap IPv4-mapped IPv6 (e.g. ::ffff:169.254.169.254, ::ffff:a9fe:a9fe, + // and the fully-expanded 0:0:0:0:0:ffff:a9fe:a9fe) so the IPv4 check below + // sees the embedded address. + if (addr.kind() === 'ipv6' && (addr as ipaddr.IPv6).isIPv4MappedAddress()) { + addr = (addr as ipaddr.IPv6).toIPv4Address() + } + + const canonical = addr.toNormalizedString() + + const blocked = + addr.kind() === 'ipv4' ? BLOCKED_METADATA_IPV4 : BLOCKED_METADATA_IPV6 + if (blocked.has(canonical)) { + throw new Error(`URL must not point to the cloud instance metadata endpoint: ${canonical}`) } } diff --git a/admin/docs/release-notes.md b/admin/docs/release-notes.md index 1d78438..2d68f22 100644 --- a/admin/docs/release-notes.md +++ b/admin/docs/release-notes.md @@ -3,16 +3,94 @@ ## Version 1.31.1 - April 21, 2026 ### Features -- **AI Assistant**: Added improved support for AMD GPU acceleration for Ollama via ROCm + HSA override. Thanks @chriscrosstalk for the contribution! -- **Content Explorer**: Added support for custom ZIM library sources and pre-seeded ZIM library mirrors in addition to the default Kiwix library. Thanks @chriscrosstalk for the contribution! -- **Content Manager**: Content update sizes and downloads are now properly displayed in Active Downloads with progress bars and friendly names. Thanks @chriscrosstalk for the contribution! -- **Maps**: Map regions can now be extracted and downloaded locally from PMTiles to avoid the need for a full global map download for users who only want specific regions. Thanks @bgauger for the contribution! +- feat(content): custom ZIM library sources with pre-seeded mirrors - #593 +- feat(content-manager): add sortable file size column - #698 +- feat(ai-chat): allow cancelling in-progress model downloads - #701 +- feat(content-updates): show size, surface downloads in Active Downloads - #773 +- feat(maps): regional map downloads via go-pmtiles extract - #780 +- feat(maps): show map coordinates on mouse move - #786 +- feat(AI): re-enable AMD GPU acceleration for Ollama via ROCm + HSA override - #804 +- feat(GPU): auto-remediate nomad_ollama passthrough loss on admin boot - #878 +- feat(KB): per-file ingest state machine (Phase 1 of RFC #883) - #888 +- feat(KB): ratio registry for disk + time estimates (Phase 1B of RFC #883) - #891 +- feat(KB): group admin docs into single row in Stored Files (§9) - #892 +- feat(KB): status pill + last-activity on Processing Queue (§5/§10) - #893 +- feat(KB): Always/Manual ingest policy toggle (§1/§4) - #894 +- feat(KB): conditional warnings A + B on Stored Files (§6) - #895 +- feat(KB): surface embedding-disk estimate in curated tier-change modal (§1) - #897 +- feat(KB): first-chat JIT prompt for ingest policy (Phase 3 task 12) - #899 +- feat(KB): wizard AI policy step (Phase 3 task 13) - #900 +- feat(KB): guardrail modal at 50GB / 10%-free thresholds (§7) - #901 +- feat(easy-setup): split AI into its own conditional step - #908 +- feat(KB): per-file ingest action + state indicator on Stored Files (§5) - #909 +- feat(chat): confirm-on-switch + one-chat-model-at-a-time enforcement - #916 + +## Docs + +- docs: add Community Add-Ons page with field manuals + W3Schools packs - #753 +- docs: add map marker API reference - #783 +- docs: require linked issue for non-trivial PRs - #799 +- docs(map): updated notes on the map pin api - #803 +- docs: link to new WSL2 install guide from README and FAQ - #811 + +## Build / deps + +- build(deps): bump picomatch in /admin - #544 +- build(deps): bump lodash from 4.17.23 to 4.18.1 in /admin - #643 +- build(deps-dev): bump vite from 6.4.1 to 6.4.2 in /admin - #677 +- build(deps): bump axios from 1.13.5 to 1.15.0 in /admin - #708 +- build(deps): bump @adonisjs/http-server from 7.8.0 to 7.8.1 in /admin - #724 +- build(deps): bump follow-redirects from 1.15.11 to 1.16.0 in /admin - #729 +- build(deps): bump protocol-buffers-schema from 3.6.0 to 3.6.1 in /admin - #736 +- build(deps): bump protobufjs from 7.5.4 to 7.5.5 in /admin - #737 ### Bug Fixes -- **API**: Compression is now skipped for Server-Sent Events (SSE) responses to prevent issues with streaming endpoints. Thanks @chriscrosstalk for the fix! -- **Maps**: Fixed logic issues with the global map banner display. Thanks @Gujiassh for the fix! -- **Maps**: The selected map file is now properly deleted after confirming the action in the UI. Thanks @cuyua9 for the fix! -- **System**: Fixed an issue where the a pending update could still be indicated in the UI even after the system was updated successfully. Thanks @jakeaturner for the fix! +- fix(downloads): stage downloads to .tmp to prevent Kiwix loading partial files - #448 +- fix(security): close remaining security audit items 3 & 4 (CWE-918, CWE-209) - #552 +- fix(ai-chat): add null check to model name - #645 +- fix(ai-chat): qwen2.5 loading on every chat message - #649 +- fix(disk-collector): fix storage reporting for NFS mounts - #686 +- fix(rag): add start button in kb modal and ensure restart policy exists - #700 +- fix(admin): only hide global map banner after download - #702 +- fix(maps): wire delete confirmation to API - #732 +- fix: prevent ZIM corrupt file crash and deduplicate Ollama download logs - #741 +- fix(ai): stop local nomad_ollama when remote Ollama is configured - #744 +- fix(rag): repair ZIM embedding pipeline (sync filter, batch gate, DOM walk) - #745 +- fix(zim): accumulate across Kiwix pages to prevent empty Content Explorer - #746 +- fix(qdrant): disable anonymous telemetry by default - #747 +- fix(disk-display): gate NAS Storage label on network filesystem type - #749 +- fix(docker): write /app/version.json from VERSION build-arg - #754 +- fix(rag): pass num_ctx and truncate to Ollama embed call - #763 +- fix(api): accept notes, marker_type, and position on markers endpoints - #770 +- fix(install): warn loudly on non-x86_64 architectures before pulling images - #797 +- fix(stream): skip compression for Server-Sent Events - #798 +- fix(maps): Country Picker UX polish + auto-refresh stored files - #817 +- fix(System): self-heal stale updateAvailable flag after sidecar-driven update - #825 +- fix(settings/update): four UI/UX fixes for the System Update page - #827 +- fix(Maps): send filename instead of full path to delete endpoint - #829 +- fix(Maps): render notes in marker popup when populated - #830 +- fix(AI): vendor-aware AMD HSA override + benchmark discrete-GPU detection - #832 +- fix(System): correct NVIDIA VRAM in Graphics card - #850 +- fix(Downloads): treat missing Content-Type as octet-stream - #859 +- fix(AI): preserve semver tag in DB on AMD Ollama updates - #868 +- fix(AI): rewrite RAG query on first chat follow-up - #869 +- fix(RAG): unbreak multi-batch ZIM ingestion (jobId dedupe) - #872 +- fix(RAG): pace continuation batches when embedding is CPU-only - #873 +- fix(queue): singleton QueueService to stop ioredis connection leak - #877 +- fix(System): correct AMD VRAM in Graphics card + harden log probe - #879 +- fix(RAG): report ZIM ingestion progress in overall-file frame - #880 +- fix(KB): add re-embed and reset & rebuild options to fix broken embeddings - #886 +- fix(ZIM): preserve co-existing Wikipedia corpora on cleanup - #887 +- fix(RAG): anchor continuation-batch initial progress to overall-file frame - #889 +- fix(AI): pre-cap embed input + log fallback reason - #890 +- fix(KB): remove redundant Refresh button from Processing Queue - #896 +- fix(KB): union Stored Files list with state-machine file paths - #898 +- fix(KB): blank-screen on panel open + tooltips on bulk-action buttons - #907 +- fix(KB): TierSelectionModal hook order + register IconLibrary - #917 +- fix(content): show selected tier on cards while downloads are in flight - #918 +- fix(KB): respect Manual ingest policy on post-download dispatch - #919 +- fix(AI): improve remote Ollama url validation to prevent SSRF vuln - #920 +- fix(models): correct inverted belongsTo keys on ChatMessage.session - #921 ### Improvements - **Build**: The Command Center image now uses the VERSION build arg to write `app/version.json` with the current version for improved version tracking and debugging, even in RC environments. Thanks @chriscrosstalk for the contribution! diff --git a/admin/package-lock.json b/admin/package-lock.json index 29f8dd3..a896708 100644 --- a/admin/package-lock.json +++ b/admin/package-lock.json @@ -9,94 +9,95 @@ "version": "0.0.0", "license": "ISC", "dependencies": { - "@adonisjs/auth": "^9.4.0", - "@adonisjs/core": "^6.18.0", - "@adonisjs/cors": "^2.2.1", - "@adonisjs/inertia": "^3.1.1", - "@adonisjs/lucid": "^21.8.2", - "@adonisjs/session": "^7.5.1", - "@adonisjs/shield": "^8.2.0", - "@adonisjs/static": "^1.1.1", - "@adonisjs/transmit": "^2.0.2", - "@adonisjs/transmit-client": "^1.0.0", - "@adonisjs/vite": "^4.0.0", - "@chonkiejs/core": "^0.0.7", - "@headlessui/react": "^2.2.4", - "@inertiajs/react": "^2.0.13", - "@markdoc/markdoc": "^0.5.2", - "@openzim/libzim": "^4.0.0", - "@protomaps/basemaps": "^5.7.0", - "@qdrant/js-client-rest": "^1.16.2", - "@tabler/icons-react": "^3.34.0", - "@tailwindcss/vite": "^4.1.10", - "@tanstack/react-query": "^5.81.5", - "@tanstack/react-query-devtools": "^5.83.0", - "@tanstack/react-virtual": "^3.13.12", - "@uppy/core": "^5.2.0", - "@uppy/dashboard": "^5.1.0", - "@uppy/react": "^5.1.1", - "@vinejs/vine": "^3.0.1", - "@vitejs/plugin-react": "^4.6.0", - "autoprefixer": "^10.4.21", - "axios": "^1.15.0", - "better-sqlite3": "^12.1.1", - "bullmq": "^5.65.1", - "cheerio": "^1.2.0", - "compression": "^1.8.1", - "dockerode": "^4.0.7", - "edge.js": "^6.2.1", - "fast-xml-parser": "^5.5.7", - "fuse.js": "^7.1.0", - "jszip": "^3.10.1", - "luxon": "^3.6.1", - "maplibre-gl": "^4.7.1", - "mysql2": "^3.14.1", - "ollama": "^0.6.3", - "openai": "^6.27.0", - "pdf-parse": "^2.4.5", - "pdf2pic": "^3.2.0", - "pino-pretty": "^13.0.0", - "pmtiles": "^4.4.0", - "postcss": "^8.5.6", - "react": "^19.1.0", - "react-adonis-transmit": "^1.0.1", - "react-dom": "^19.1.0", - "react-map-gl": "^8.1.0", - "react-markdown": "^10.1.0", - "reflect-metadata": "^0.2.2", - "remark-gfm": "^4.0.1", - "sharp": "^0.34.5", - "stopword": "^3.1.5", - "systeminformation": "^5.31.0", - "tailwindcss": "^4.2.1", - "tar": "^7.5.11", - "tesseract.js": "^7.0.0", - "url-join": "^5.0.0", - "yaml": "^2.8.3" + "@adonisjs/auth": "9.6.0", + "@adonisjs/core": "6.19.3", + "@adonisjs/cors": "2.2.1", + "@adonisjs/inertia": "3.1.1", + "@adonisjs/lucid": "21.8.2", + "@adonisjs/session": "7.7.1", + "@adonisjs/shield": "8.2.0", + "@adonisjs/static": "1.1.1", + "@adonisjs/transmit": "2.0.2", + "@adonisjs/transmit-client": "1.1.0", + "@adonisjs/vite": "4.0.0", + "@chonkiejs/core": "0.0.7", + "@headlessui/react": "2.2.9", + "@inertiajs/react": "2.3.13", + "@markdoc/markdoc": "0.5.4", + "@openzim/libzim": "4.0.0", + "@protomaps/basemaps": "5.7.0", + "@qdrant/js-client-rest": "1.16.2", + "@tabler/icons-react": "3.36.1", + "@tailwindcss/vite": "4.1.18", + "@tanstack/react-query": "5.90.20", + "@tanstack/react-query-devtools": "5.91.3", + "@tanstack/react-virtual": "3.13.18", + "@uppy/core": "5.2.0", + "@uppy/dashboard": "5.1.0", + "@uppy/react": "5.1.1", + "@vinejs/vine": "3.0.1", + "@vitejs/plugin-react": "4.7.0", + "autoprefixer": "10.4.24", + "axios": "1.15.0", + "better-sqlite3": "12.6.2", + "bullmq": "5.67.2", + "cheerio": "1.2.0", + "compression": "1.8.1", + "dockerode": "4.0.9", + "edge.js": "6.4.0", + "fast-xml-parser": "5.5.9", + "fuse.js": "7.1.0", + "ipaddr.js": "^2.4.0", + "jszip": "3.10.1", + "luxon": "3.7.2", + "maplibre-gl": "4.7.1", + "mysql2": "3.16.2", + "ollama": "0.6.3", + "openai": "6.27.0", + "pdf-parse": "2.4.5", + "pdf2pic": "3.2.0", + "pino-pretty": "13.1.3", + "pmtiles": "4.4.0", + "postcss": "8.5.6", + "react": "19.2.4", + "react-adonis-transmit": "1.0.1", + "react-dom": "19.2.4", + "react-map-gl": "8.1.0", + "react-markdown": "10.1.0", + "reflect-metadata": "0.2.2", + "remark-gfm": "4.0.1", + "sharp": "0.34.5", + "stopword": "3.1.5", + "systeminformation": "5.31.0", + "tailwindcss": "4.2.2", + "tar": "7.5.11", + "tesseract.js": "7.0.0", + "url-join": "5.0.0", + "yaml": "2.8.3" }, "devDependencies": { - "@adonisjs/assembler": "^7.8.2", - "@adonisjs/eslint-config": "^2.0.0", - "@adonisjs/prettier-config": "^1.4.4", - "@adonisjs/tsconfig": "^1.4.0", - "@japa/assert": "^4.0.1", - "@japa/plugin-adonisjs": "^4.0.0", - "@japa/runner": "^4.2.0", + "@adonisjs/assembler": "7.8.2", + "@adonisjs/eslint-config": "2.1.2", + "@adonisjs/prettier-config": "1.4.5", + "@adonisjs/tsconfig": "1.4.1", + "@japa/assert": "4.2.0", + "@japa/plugin-adonisjs": "4.0.0", + "@japa/runner": "4.5.0", "@swc/core": "1.11.24", - "@tanstack/eslint-plugin-query": "^5.81.2", - "@types/compression": "^1.8.1", - "@types/dockerode": "^4.0.1", - "@types/luxon": "^3.6.2", - "@types/node": "^22.15.18", - "@types/react": "^19.1.8", - "@types/react-dom": "^19.1.6", - "@types/stopword": "^2.0.3", - "eslint": "^9.26.0", - "hot-hook": "^0.4.0", - "prettier": "^3.5.3", - "ts-node-maintained": "^10.9.5", - "typescript": "~5.8.3", - "vite": "^6.4.2" + "@tanstack/eslint-plugin-query": "5.91.4", + "@types/compression": "1.8.1", + "@types/dockerode": "4.0.1", + "@types/luxon": "3.7.1", + "@types/node": "22.19.7", + "@types/react": "19.2.10", + "@types/react-dom": "19.2.3", + "@types/stopword": "2.0.3", + "eslint": "9.39.2", + "hot-hook": "0.4.0", + "prettier": "3.8.1", + "ts-node-maintained": "10.9.6", + "typescript": "5.8.3", + "vite": "6.4.2" } }, "node_modules/@adobe/css-tools": { @@ -10099,12 +10100,12 @@ } }, "node_modules/ipaddr.js": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", - "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.4.0.tgz", + "integrity": "sha512-9VGk3HGanVE6JoZXHiCpnGy5X0jYDnN4EA4lntFPj+1vIWlFhIylq2CrrCOJH9EAhc5CYhq18F2Av2tgoAPsYQ==", "license": "MIT", "engines": { - "node": ">= 0.10" + "node": ">= 10" } }, "node_modules/is-alphabetical": { @@ -13790,6 +13791,15 @@ "node": ">= 0.10" } }, + "node_modules/proxy-addr/node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, "node_modules/proxy-from-env": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz", diff --git a/admin/package.json b/admin/package.json index b273fd0..48c7108 100644 --- a/admin/package.json +++ b/admin/package.json @@ -100,6 +100,7 @@ "edge.js": "6.4.0", "fast-xml-parser": "5.5.9", "fuse.js": "7.1.0", + "ipaddr.js": "^2.4.0", "jszip": "3.10.1", "luxon": "3.7.2", "maplibre-gl": "4.7.1", diff --git a/admin/tests/unit/cloud_metadata_url.spec.ts b/admin/tests/unit/cloud_metadata_url.spec.ts new file mode 100644 index 0000000..4db7c0a --- /dev/null +++ b/admin/tests/unit/cloud_metadata_url.spec.ts @@ -0,0 +1,57 @@ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { assertNotCloudMetadataUrl } from '../../app/validators/common.js' + +const expectBlocked = (url: string) => { + assert.throws(() => assertNotCloudMetadataUrl(url), /instance metadata|http or https/) +} + +const expectAllowed = (url: string) => { + assert.doesNotThrow(() => assertNotCloudMetadataUrl(url)) +} + +test('blocks plain IPv4 IMDS', () => { + expectBlocked('http://169.254.169.254/latest/meta-data/') +}) + +test('blocks IPv6 EC2 IMDS', () => { + expectBlocked('http://[fd00:ec2::254]/') +}) + +test('blocks fully-expanded IPv6 EC2 IMDS', () => { + expectBlocked('http://[fd00:ec2:0:0:0:0:0:254]/') +}) + +test('blocks IPv4-mapped IPv6 form of IMDS (dotted)', () => { + expectBlocked('http://[::ffff:169.254.169.254]/') +}) + +test('blocks IPv4-mapped IPv6 form of IMDS (hex)', () => { + expectBlocked('http://[::ffff:a9fe:a9fe]/') +}) + +test('blocks fully-expanded IPv4-mapped IPv6 form of IMDS', () => { + expectBlocked('http://[0:0:0:0:0:ffff:a9fe:a9fe]/') +}) + +test('blocks non-http(s) schemes', () => { + expectBlocked('file:///etc/passwd') + expectBlocked('gopher://169.254.169.254/') +}) + +test('allows LAN / loopback / RFC1918 hosts (intentional for remote-ollama use case)', () => { + expectAllowed('http://127.0.0.1:11434/') + expectAllowed('http://192.168.1.10:11434/') + expectAllowed('http://10.0.0.5:11434/') + expectAllowed('http://[::1]:11434/') +}) + +test('allows DNS hostnames', () => { + expectAllowed('http://ollama.lan:11434/') + expectAllowed('https://api.example.com/v1') +}) + +test('allows other link-local IPv4 addresses (not the metadata IP)', () => { + expectAllowed('http://169.254.1.1/') +})