docs(release): finalize v1.33.0 release notes [skip ci]

This reverts commit d8cca183f9.

.dockerignore

@@ -2,4 +2,7 @@
 .env.*
 .git
 node_modules
-*.log
+*.log
+admin/storage
+admin/node_modules
+admin/build

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,193 @@
+name: Bug Report
+description: Report a bug or issue with Project N.O.M.A.D.
+title: "[Bug]: "
+labels: ["bug", "needs-triage"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to report a bug! Please fill out the information below to help us diagnose and fix the issue.
+        
+        **Before submitting:**
+        - Search existing issues to avoid duplicates
+        - Ensure you're running the latest version of N.O.M.A.D.
+        - Redact any personal or sensitive information from logs/configs
+        - Please don't submit issues related to running N.O.M.A.D. on Unraid or another NAS - we don't have plans to support these kinds of platforms at this time
+
+  - type: dropdown
+    id: issue-category
+    attributes:
+      label: Issue Category
+      description: What area is this issue related to?
+      options:
+        - Installation/Setup
+        - AI Assistant (Ollama)
+        - Knowledge Base/RAG (Document Upload)
+        - Docker/Container Issues
+        - GPU Configuration
+        - Content Downloads (ZIM, Maps, Collections)
+        - Service Management (Start/Stop/Update)
+        - System Performance/Resources
+        - UI/Frontend Issue
+        - Other
+    validations:
+      required: true
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Bug Description
+      description: Provide a clear and concise description of what the bug is
+      placeholder: What happened? What did you expect to happen?
+    validations:
+      required: true
+
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Steps to Reproduce
+      description: How can we reproduce this issue?
+      placeholder: |
+        1. Go to '...'
+        2. Click on '...'
+        3. See error
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected-behavior
+    attributes:
+      label: Expected Behavior
+      description: What did you expect to happen?
+      placeholder: Describe the expected outcome
+    validations:
+      required: true
+
+  - type: textarea
+    id: actual-behavior
+    attributes:
+      label: Actual Behavior
+      description: What actually happened?
+      placeholder: Describe what actually occurred, including any error messages
+    validations:
+      required: true
+
+  - type: input
+    id: nomad-version
+    attributes:
+      label: N.O.M.A.D. Version
+      description: What version of N.O.M.A.D. are you running? (Check Settings > Update or run `docker ps` and check nomad_admin image tag)
+      placeholder: "e.g., 1.29.0"
+    validations:
+      required: true
+
+  - type: dropdown
+    id: os
+    attributes:
+      label: Operating System
+      description: What OS are you running N.O.M.A.D. on?
+      options:
+        - Ubuntu 24.04
+        - Ubuntu 22.04
+        - Ubuntu 20.04
+        - Debian 13 (Trixie)
+        - Debian 12 (Bookworm)
+        - Debian 11 (Bullseye)
+        - Other Debian-based
+        - Other (not yet officially supported)
+    validations:
+      required: true
+
+  - type: input
+    id: docker-version
+    attributes:
+      label: Docker Version
+      description: What version of Docker are you running? (`docker --version`)
+      placeholder: "e.g., Docker version 24.0.7"
+
+  - type: dropdown
+    id: gpu-present
+    attributes:
+      label: Do you have a dedicated GPU?
+      options:
+        - "Yes"
+        - "No"
+        - "Not sure"
+    validations:
+      required: true
+
+  - type: input
+    id: gpu-model
+    attributes:
+      label: GPU Model (if applicable)
+      description: What GPU model do you have? (Check Settings > System or run `nvidia-smi` if NVIDIA GPU)
+      placeholder: "e.g., NVIDIA GeForce RTX 3060"
+
+  - type: textarea
+    id: system-specs
+    attributes:
+      label: System Specifications
+      description: Provide relevant system specs (CPU, RAM, available disk space)
+      placeholder: |
+        CPU: 
+        RAM: 
+        Available Disk Space: 
+        GPU (if any): 
+
+  - type: textarea
+    id: service-status
+    attributes:
+      label: Service Status (if relevant)
+      description: If this is a service-related issue, what's the status of relevant services? (Check Settings > Apps or run `docker ps`)
+      placeholder: |
+        Paste output from `docker ps` or describe service states from the UI
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant Logs
+      description: |
+        Include any relevant logs or error messages. **Please redact any personal/sensitive information.**
+        
+        Useful commands for collecting logs:
+        - N.O.M.A.D. management app: `docker logs nomad_admin`
+        - Ollama: `docker logs nomad_ollama`
+        - Qdrant: `docker logs nomad_qdrant`
+        - Specific service: `docker logs nomad_<service-name>`
+      placeholder: Paste relevant log output here
+      render: shell
+
+  - type: textarea
+    id: browser-console
+    attributes:
+      label: Browser Console Errors (if UI issue)
+      description: If this is a UI issue, include any errors from your browser's developer console (F12)
+      placeholder: Paste browser console errors here
+      render: javascript
+
+  - type: textarea
+    id: screenshots
+    attributes:
+      label: Screenshots
+      description: If applicable, add screenshots to help explain your problem (drag and drop images here)
+
+  - type: textarea
+    id: additional-context
+    attributes:
+      label: Additional Context
+      description: Add any other context about the problem here (network setup, custom configurations, recent changes, etc.)
+
+  - type: checkboxes
+    id: terms
+    attributes:
+      label: Pre-submission Checklist
+      description: Please confirm the following before submitting
+      options:
+        - label: I have searched for existing issues that might be related to this bug
+          required: true
+        - label: I am running the latest version of Project N.O.M.A.D. (or have noted my version above)
+          required: true
+        - label: I have redacted any personal or sensitive information from logs and screenshots
+          required: true
+        - label: This issue is NOT related to running N.O.M.A.D. on an unsupported/non-Debian-based OS
+          required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,17 @@
+blank_issues_enabled: false
+contact_links:
+  - name: 💬 Discord Community
+    url: https://discord.com/invite/crosstalksolutions
+    about: Join our Discord community for general questions, support, and discussions
+  - name: 📖 Documentation
+    url: https://projectnomad.us
+    about: Check the official documentation and guides
+  - name: 🏆 Community Leaderboard
+    url: https://benchmark.projectnomad.us
+    about: View the N.O.M.A.D. benchmark leaderboard
+  - name: 🤝 Contributing Guide
+    url: https://github.com/Crosstalk-Solutions/project-nomad/blob/main/CONTRIBUTING.md
+    about: Learn how to contribute to Project N.O.M.A.D.
+  - name: 📅 Roadmap
+    url: https://roadmap.projectnomad.us
+    about: See our public roadmap, vote on features, and suggest new ones

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,150 @@
+name: Feature Request
+description: Suggest a new feature or enhancement for Project N.O.M.A.D.
+title: "[Feature]: "
+labels: ["enhancement", "needs-discussion"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for your interest in improving Project N.O.M.A.D.! Before you submit a feature request, consider checking our [roadmap](https://roadmap.projectnomad.us) to see if it's already planned or in progress. You're welcome to suggest new ideas there if you don't plan on opening PRs yourself.
+
+        
+        **Please note:** Feature requests are not guaranteed to be implemented. All requests are evaluated based on alignment with the project's goals, feasibility, and community demand.
+        
+        **Before submitting:**
+        - Search existing feature requests and our [roadmap](https://roadmap.projectnomad.us) to avoid duplicates
+        - Consider if this aligns with N.O.M.A.D.'s mission: offline-first knowledge and education
+        - Consider the technical feasibility of the feature: N.O.M.A.D. is designed to be containerized and run on a wide range of hardware, so features that require heavy resources (aside from GPU-intensive tasks) or complex host configurations may be less likely to be implemented
+        - Consider the scope of the feature: Small, focused enhancements that can be implemented incrementally are more likely to be implemented than large, broad features that would require significant development effort or have an unclear path forward
+        - If you're able to contribute code, testing, or documentation, that significantly increases the chances of your feature being implemented
+
+  - type: dropdown
+    id: feature-category
+    attributes:
+      label: Feature Category
+      description: What area does this feature relate to?
+      options:
+        - New Service/Tool Integration
+        - AI Assistant Enhancement
+        - Knowledge Base/RAG Improvement
+        - Content Management (ZIM, Maps, Collections)
+        - UI/UX Improvement
+        - System Management
+        - Performance Optimization
+        - Documentation
+        - Security
+        - Other
+    validations:
+      required: true
+
+  - type: textarea
+    id: problem
+    attributes:
+      label: Problem Statement
+      description: What problem does this feature solve? Is your feature request related to a pain point?
+      placeholder: I find it frustrating when... / It would be helpful if... / Users struggle with...
+    validations:
+      required: true
+
+  - type: textarea
+    id: solution
+    attributes:
+      label: Proposed Solution
+      description: Describe the feature or enhancement you'd like to see
+      placeholder: Add a feature that... / Change the behavior to... / Integrate with...
+    validations:
+      required: true
+
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Alternative Solutions
+      description: Have you considered any alternative solutions or workarounds?
+      placeholder: I've tried... / Another approach could be... / A workaround is...
+
+  - type: textarea
+    id: use-case
+    attributes:
+      label: Use Case
+      description: Describe a specific scenario where this feature would be valuable
+      placeholder: |
+        As a [type of user], when I [do something], I want to [accomplish something] so that [benefit].
+        
+        Example: Because I have a dedicated GPU, I want to be able to see in the UI if GPU support is enabled so that I can optimize performance and troubleshoot issues more easily.
+
+  - type: dropdown
+    id: user-type
+    attributes:
+      label: Who would benefit from this feature?
+      description: What type of users would find this most valuable?
+      multiple: true
+      options:
+        - Individual/Home Users
+        - Families
+        - Teachers/Educators
+        - Students
+        - Survivalists/Preppers
+        - Developers/Contributors
+        - Organizations
+        - All Users
+    validations:
+      required: true
+
+  - type: dropdown
+    id: priority
+    attributes:
+      label: How important is this feature to you?
+      options:
+        - Critical - Blocking my use of N.O.M.A.D.
+        - High - Would significantly improve my experience
+        - Medium - Would be nice to have
+        - Low - Minor convenience
+    validations:
+      required: true
+
+  - type: textarea
+    id: implementation-ideas
+    attributes:
+      label: Implementation Ideas (Optional)
+      description: If you have technical suggestions for how this could be implemented, share them here
+      placeholder: This could potentially use... / It might integrate with... / A possible approach is...
+
+  - type: textarea
+    id: examples
+    attributes:
+      label: Examples or References
+      description: Are there similar features in other applications? Include links, screenshots, or descriptions
+      placeholder: Similar to how [app name] does... / See this example at [URL]
+
+  - type: dropdown
+    id: willing-to-contribute
+    attributes:
+      label: Would you be willing to help implement this?
+      description: Contributing increases the likelihood of implementation
+      options:
+        - "Yes - I can write the code"
+        - "Yes - I can help test"
+        - "Yes - I can help with documentation"
+        - "Maybe - with guidance"
+        - "No - I don't have the skills/time"
+    validations:
+      required: true
+
+  - type: textarea
+    id: additional-context
+    attributes:
+      label: Additional Context
+      description: Add any other context, mockups, diagrams, or information about the feature request
+
+  - type: checkboxes
+    id: checklist
+    attributes:
+      label: Pre-submission Checklist
+      description: Please confirm the following before submitting
+      options:
+        - label: I have searched for existing feature requests that might be similar
+          required: true
+        - label: This feature aligns with N.O.M.A.D.'s mission of offline-first knowledge and education
+          required: true
+        - label: I understand that feature requests are not guaranteed to be implemented
+          required: true

.github/dependabot.yaml

@@ -0,0 +1,7 @@
+version: 2
+updates:
+  - package-ecosystem: "npm"
+    directory: "/admin"
+    schedule:
+      interval: "weekly"
+    target-branch: "rc"

.github/scripts/finalize-release-notes.sh

@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+#
+# finalize-release-notes.sh
+#
+# Stamps the "## Unreleased" section in a release-notes file with a version
+# and date, and extracts the section content for use in GitHub releases / email.
+# Also includes all commits since the last release for complete transparency.
+#
+# Usage:  finalize-release-notes.sh <version> <file-path>
+#
+# Exit codes:
+#   0 - Success: section stamped and extracted
+#   1 - No "## Unreleased" section found (skip gracefully)
+#   2 - Unreleased section exists but is empty (skip gracefully)
+
+set -euo pipefail
+
+VERSION="${1:?Usage: finalize-release-notes.sh <version> <file-path>}"
+FILE="${2:?Usage: finalize-release-notes.sh <version> <file-path>}"
+
+if [[ ! -f "$FILE" ]]; then
+  echo "Error: File not found: $FILE" >&2
+  exit 1
+fi
+
+# Find the line number of the ## Unreleased header (case-insensitive)
+HEADER_LINE=$(grep -inm1 '^## unreleased' "$FILE" | cut -d: -f1)
+
+if [[ -z "$HEADER_LINE" ]]; then
+  echo "No '## Unreleased' section found. Skipping."
+  exit 1
+fi
+
+TOTAL_LINES=$(wc -l < "$FILE")
+
+# Find the next section header (## Version ...) or --- separator after the Unreleased header
+NEXT_SECTION_LINE=""
+if [[ $HEADER_LINE -lt $TOTAL_LINES ]]; then
+  NEXT_SECTION_LINE=$(tail -n +"$((HEADER_LINE + 1))" "$FILE" \
+    | grep -nm1 '^## \|^---$' \
+    | cut -d: -f1)
+fi
+
+if [[ -n "$NEXT_SECTION_LINE" ]]; then
+  # NEXT_SECTION_LINE is relative to HEADER_LINE+1, convert to absolute
+  END_LINE=$((HEADER_LINE + NEXT_SECTION_LINE - 1))
+else
+  # Section runs to end of file
+  END_LINE=$TOTAL_LINES
+fi
+
+# Extract content between header and next section (exclusive of both boundaries)
+CONTENT_START=$((HEADER_LINE + 1))
+CONTENT_END=$END_LINE
+
+# Extract the section body (between header line and the next boundary)
+SECTION_BODY=$(sed -n "${CONTENT_START},${CONTENT_END}p" "$FILE" | sed '/^$/N;/^\n$/d')
+
+# Check for actual content: strip blank lines and lines that are only markdown headers (###...)
+TRIMMED=$(echo "$SECTION_BODY" | sed '/^[[:space:]]*$/d')
+HAS_CONTENT=$(echo "$SECTION_BODY" | sed '/^[[:space:]]*$/d' | grep -v '^###' || true)
+
+if [[ -z "$TRIMMED" || -z "$HAS_CONTENT" ]]; then
+  echo "Unreleased section is empty. Skipping."
+  exit 2
+fi
+
+# Format the date as "Month Day, Year"
+DATE_STAMP=$(date +'%B %-d, %Y')
+NEW_HEADER="## Version ${VERSION} - ${DATE_STAMP}"
+
+# Build the replacement: swap the header line, keep everything else intact
+{
+  # Lines before the Unreleased header
+  if [[ $HEADER_LINE -gt 1 ]]; then
+    head -n "$((HEADER_LINE - 1))" "$FILE"
+  fi
+  # New versioned header
+  echo "$NEW_HEADER"
+  # Content between header and next section
+  sed -n "${CONTENT_START},${CONTENT_END}p" "$FILE"
+  # Rest of the file after the section
+  if [[ $END_LINE -lt $TOTAL_LINES ]]; then
+    tail -n +"$((END_LINE + 1))" "$FILE"
+  fi
+} > "${FILE}.tmp"
+
+mv "${FILE}.tmp" "$FILE"
+
+# Get commits since the last release
+LAST_TAG=$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null || echo "")
+COMMIT_LIST=""
+
+if [[ -n "$LAST_TAG" ]]; then
+  echo "Fetching commits since ${LAST_TAG}..."
+  # Get commits between last tag and HEAD, excluding merge commits and skip ci commits
+  COMMIT_LIST=$(git log "${LAST_TAG}..HEAD" \
+    --no-merges \
+    --pretty=format:"- %s ([%h](https://github.com/${GITHUB_REPOSITORY}/commit/%H))" \
+    --grep="\[skip ci\]" --invert-grep \
+    || echo "")
+else
+  echo "No previous tag found, fetching all commits..."
+  COMMIT_LIST=$(git log \
+    --no-merges \
+    --pretty=format:"- %s ([%h](https://github.com/${GITHUB_REPOSITORY}/commit/%H))" \
+    --grep="\[skip ci\]" --invert-grep \
+    || echo "")
+fi
+
+# Write the extracted section content (for GitHub release body / future email)
+{
+  echo "$NEW_HEADER"
+  echo ""
+  if [[ -n "$TRIMMED" ]]; then
+    echo "$TRIMMED"
+    echo ""
+  fi
+  
+  # Add commit history if available
+  if [[ -n "$COMMIT_LIST" ]]; then
+    echo "---"
+    echo ""
+    echo "### 📝 All Changes"
+    echo ""
+    echo "$COMMIT_LIST"
+  fi
+} > "${FILE}.section"
+
+echo "Finalized release notes for v${VERSION}"
+echo "  Updated: ${FILE}"
+echo "  Extracted: ${FILE}.section"
+exit 0

.github/workflows/build-admin-on-pr.yml

@@ -0,0 +1,25 @@
+name: Build Admin
+
+on: pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: '24'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+        working-directory: ./admin
+
+      - name: Run build
+        run: npm run build
+        working-directory: ./admin

.github/workflows/build-disk-collector.yml

@@ -0,0 +1,51 @@
+name: Build Disk Collector Image
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Semantic version to label the Docker image under (no "v" prefix, e.g. "1.2.3")'
+        required: true
+        type: string
+      tag_latest:
+        description: 'Also tag this image as :latest?'
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  check_authorization:
+    name: Check authorization to publish new Docker image
+    runs-on: ubuntu-latest
+    outputs:
+      isAuthorized: ${{ steps.check-auth.outputs.is_authorized }}
+    steps:
+      - name: check-auth
+        id: check-auth
+        run: echo "is_authorized=${{ contains(secrets.DEPLOYMENT_AUTHORIZED_USERS, github.triggering_actor) }}" >> $GITHUB_OUTPUT
+  build:
+    name: Build disk-collector image
+    needs: check_authorization
+    if: needs.check_authorization.outputs.isAuthorized == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v7
+        with:
+          context: install/sidecar-disk-collector
+          push: true
+          tags: |
+            ghcr.io/crosstalk-solutions/project-nomad-disk-collector:${{ inputs.version }}
+            ghcr.io/crosstalk-solutions/project-nomad-disk-collector:v${{ inputs.version }}
+            ${{ inputs.tag_latest && 'ghcr.io/crosstalk-solutions/project-nomad-disk-collector:latest' || '' }}

.github/workflows/build-primary-image.yml

@@ -0,0 +1,54 @@
+name: Build Primary Docker Image
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Semantic version to label the Docker image under (no "v" prefix, e.g. "1.2.3")'
+        required: true
+        type: string
+      tag_latest:
+        description: 'Also tag this image as :latest? (Keep false for RC and beta releases)'
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  check_authorization:
+    name: Check authorization to publish new Docker image
+    runs-on: ubuntu-latest
+    outputs:
+      isAuthorized: ${{ steps.check-auth.outputs.is_authorized }}
+    steps:
+      - name: check-auth
+        id: check-auth
+        run: echo "is_authorized=${{ contains(secrets.DEPLOYMENT_AUTHORIZED_USERS, github.triggering_actor) }}" >> $GITHUB_OUTPUT      
+  build:
+    name: Build Docker image
+    needs: check_authorization
+    if: needs.check_authorization.outputs.isAuthorized == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v7
+        with:
+          push: true
+          tags: |
+            ghcr.io/crosstalk-solutions/project-nomad:${{ inputs.version }}
+            ghcr.io/crosstalk-solutions/project-nomad:v${{ inputs.version }}
+            ${{ inputs.tag_latest && 'ghcr.io/crosstalk-solutions/project-nomad:latest' || '' }}
+          build-args: |
+            VERSION=${{ inputs.version }}
+            BUILD_DATE=${{ github.event.workflow_run.created_at }}
+            VCS_REF=${{ github.sha }}

.github/workflows/build-sidecar-updater.yml

@@ -1,12 +1,17 @@
-name: Build Docker Image
+name: Build Sidecar Updater Image
 
 on:
   workflow_dispatch:
     inputs:
       version:
-        description: 'Semantic version to label the Docker image under'
+        description: 'Semantic version to label the Docker image under (no "v" prefix, e.g. "1.2.3")'
         required: true
         type: string
+      tag_latest:
+        description: 'Also tag this image as :latest?'
+        required: false
+        type: boolean
+        default: false
 
 jobs:
   check_authorization:
@@ -17,9 +22,9 @@ jobs:
     steps:
       - name: check-auth
         id: check-auth
-        run: echo "is_authorized=${{ contains(secrets.DEPLOYMENT_AUTHORIZED_USERS, github.triggering_actor) }}" >> $GITHUB_OUTPUT      
+        run: echo "is_authorized=${{ contains(secrets.DEPLOYMENT_AUTHORIZED_USERS, github.triggering_actor) }}" >> $GITHUB_OUTPUT
   build:
-    name: Build Docker image
+    name: Build sidecar-updater image
     needs: check_authorization
     if: needs.check_authorization.outputs.isAuthorized == 'true'
     runs-on: ubuntu-latest
@@ -28,7 +33,7 @@ jobs:
       packages: write
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       - name: Log in to GitHub Container Registry
         uses: docker/login-action@v2
         with:
@@ -38,7 +43,9 @@ jobs:
       - name: Build and push
         uses: docker/build-push-action@v5
         with:
+          context: install/sidecar-updater
           push: true
           tags: |
-            ghcr.io/crosstalk-solutions/project-nomad:${{ inputs.version }}
-            ghcr.io/crosstalk-solutions/project-nomad:latest
+            ghcr.io/crosstalk-solutions/project-nomad-sidecar-updater:${{ inputs.version }}
+            ghcr.io/crosstalk-solutions/project-nomad-sidecar-updater:v${{ inputs.version }}
+            ${{ inputs.tag_latest && 'ghcr.io/crosstalk-solutions/project-nomad-sidecar-updater:latest' || '' }}

.github/workflows/release.yml

@@ -22,16 +22,72 @@ jobs:
       newVersion: ${{ steps.semver.outputs.new_release_version }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0
           persist-credentials: false
+      - name: Sync tags
+        run: git fetch --tags --force
       - name: semantic-release
-        uses: cycjimmy/semantic-release-action@v3
+        uses: cycjimmy/semantic-release-action@v6
         id: semver
         env:
           GITHUB_TOKEN: ${{ secrets.COSMISTACKBOT_ACCESS_TOKEN }}
           GIT_AUTHOR_NAME: cosmistack-bot
           GIT_AUTHOR_EMAIL: dev@cosmistack.com
           GIT_COMMITTER_NAME: cosmistack-bot
-          GIT_COMMITTER_EMAIL: dev@cosmistack.com
+          GIT_COMMITTER_EMAIL: dev@cosmistack.com
+
+      - name: Finalize release notes
+        # Skip for pre-releases (versions containing a hyphen, e.g. 1.27.0-rc.1)
+        if: |
+          steps.semver.outputs.new_release_published == 'true' &&
+          !contains(steps.semver.outputs.new_release_version, '-')
+        id: finalize-notes
+        env:
+          GITHUB_REPOSITORY: ${{ github.repository }}
+        run: |
+          git pull origin main
+          chmod +x .github/scripts/finalize-release-notes.sh
+          EXIT_CODE=0
+          .github/scripts/finalize-release-notes.sh \
+            "${{ steps.semver.outputs.new_release_version }}" \
+            admin/docs/release-notes.md || EXIT_CODE=$?
+          if [[ "$EXIT_CODE" -eq 0 ]]; then
+            echo "has_notes=true" >> $GITHUB_OUTPUT
+          else
+            echo "has_notes=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Commit finalized release notes
+        if: |
+          steps.semver.outputs.new_release_published == 'true' &&
+          steps.finalize-notes.outputs.has_notes == 'true' &&
+          !contains(steps.semver.outputs.new_release_version, '-')
+        run: |
+          git config user.name "cosmistack-bot"
+          git config user.email "dev@cosmistack.com"
+          git remote set-url origin https://x-access-token:${{ secrets.COSMISTACKBOT_ACCESS_TOKEN }}@github.com/${{ github.repository }}.git
+          git add admin/docs/release-notes.md
+          git commit -m "docs(release): finalize v${{ steps.semver.outputs.new_release_version }} release notes [skip ci]"
+          git push origin main
+
+      - name: Update GitHub release body
+        if: |
+          steps.semver.outputs.new_release_published == 'true' &&
+          steps.finalize-notes.outputs.has_notes == 'true' &&
+          !contains(steps.semver.outputs.new_release_version, '-')
+        env:
+          GH_TOKEN: ${{ secrets.COSMISTACKBOT_ACCESS_TOKEN }}
+        run: |
+          gh release edit "v${{ steps.semver.outputs.new_release_version }}" \
+            --notes-file admin/docs/release-notes.md.section
+
+      # Future: Send release notes email
+      # - name: Send release notes email
+      #   if: steps.semver.outputs.new_release_published == 'true' && steps.finalize-notes.outputs.has_notes == 'true'
+      #   run: |
+      #     curl -X POST "https://api.projectnomad.us/api/v1/newsletter/release" \
+      #       -H "Authorization: Bearer ${{ secrets.NOMAD_API_KEY }}" \
+      #       -H "Content-Type: application/json" \
+      #       -d "{\"version\": \"${{ steps.semver.outputs.new_release_version }}\", \"body\": $(cat admin/docs/release-notes.md.section | jq -Rs .)}"

.github/workflows/validate-collection-urls.yml

@@ -0,0 +1,58 @@
+name: Validate Collection URLs
+
+on:
+  push:
+    paths:
+      - 'collections/**.json'
+  pull_request:
+    paths:
+      - 'collections/**.json'
+
+jobs:
+  validate-urls:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Extract and validate URLs
+        run: |
+          FAILED=0
+          CHECKED=0
+          FAILED_URLS=""
+
+          # Recursively extract all non-null string URLs from every JSON file in collections/
+          URLS=$(jq -r '.. | .url? | select(type == "string")' collections/*.json | sort -u)
+
+          while IFS= read -r url; do
+            [ -z "$url" ] && continue
+            CHECKED=$((CHECKED + 1))
+            printf "Checking: %s ... " "$url"
+
+            # Use Range: bytes=0-0 to avoid downloading the full file.
+            # --max-filesize 1 aborts early if the server ignores the Range header
+            # and returns 200 with the full body. The HTTP status is still captured.
+            HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \
+              --range 0-0 \
+              --max-filesize 1 \
+              --max-time 30 \
+              --location \
+              "$url")
+
+            if [ "$HTTP_CODE" = "200" ] || [ "$HTTP_CODE" = "206" ]; then
+              echo "OK ($HTTP_CODE)"
+            else
+              echo "FAILED ($HTTP_CODE)"
+              FAILED=$((FAILED + 1))
+              FAILED_URLS="$FAILED_URLS\n  - $url (HTTP $HTTP_CODE)"
+            fi
+          done <<< "$URLS"
+
+          echo ""
+          echo "Checked $CHECKED URLs, $FAILED failed."
+
+          if [ "$FAILED" -gt 0 ]; then
+            echo ""
+            echo "Broken URLs:"
+            printf "%b\n" "$FAILED_URLS"
+            exit 1
+          fi

.releaserc.json

@@ -1,5 +1,8 @@
 {
-  "branches": ["master"],
+  "branches": [
+    "main",
+    { "name": "rc", "prerelease": "rc" }
+  ],
   "plugins": [
     "@semantic-release/commit-analyzer",
     "@semantic-release/release-notes-generator",

CODE_OF_CONDUCT.md

@@ -0,0 +1,128 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

CONTRIBUTING.md

@@ -0,0 +1,175 @@
+# Contributing to Project N.O.M.A.D.
+
+Thank you for your interest in contributing to Project N.O.M.A.D.! Community contributions are what keep this project growing and improving. Please read this guide fully before getting started — it will save you (and the maintainers) a lot of time.
+
+> **Note:** Acceptance of contributions is not guaranteed. All pull requests are evaluated based on quality, relevance, and alignment with the project's goals. The maintainers of Project N.O.M.A.D. ("Nomad") reserve the right accept, deny, or modify any pull request at their sole discretion.
+
+---
+
+## Table of Contents
+
+- [Code of Conduct](#code-of-conduct)
+- [Before You Start](#before-you-start)
+- [Getting Started](#getting-started)
+- [Development Workflow](#development-workflow)
+- [Commit Messages](#commit-messages)
+- [Release Notes](#release-notes)
+- [Versioning](#versioning)
+- [Submitting a Pull Request](#submitting-a-pull-request)
+- [Feedback & Community](#feedback--community)
+
+---
+
+## Code of Conduct
+
+Please read and review our full [Code of Conduct](https://github.com/Crosstalk-Solutions/project-nomad/blob/main/CODE_OF_CONDUCT.md) before contributing. In short: please be respectful and considerate in all interactions with maintainers and other contributors.
+
+We are committed to providing a welcoming environment for everyone. Disrespectful or abusive behavior will not be tolerated. 
+
+---
+
+## Before You Start
+
+**Open an issue first.** Before writing any code for a non-trivial change, you must [open an issue](../../issues/new) to discuss your proposed change. This helps avoid duplicate work and ensures your contribution aligns with the project's direction. **Pull requests submitted without a corresponding issue may be closed at the maintainers' discretion.**
+
+**Trivial fixes are exempt** and may be submitted directly as a PR. Examples:
+- Typo and grammar corrections
+- Documentation clarifications
+- Small one-line bug fixes with an obvious cause
+
+If you're not sure whether your change qualifies as trivial, open an issue first.
+
+When opening an issue:
+- Use a clear, descriptive title
+- Describe the problem you're solving or the feature you want to add
+- If it's a bug, include steps to reproduce it and as much detail about your environment as possible
+- Ensure you redact any personal or sensitive information in any logs, configs, etc.
+
+---
+
+## Getting Started with Contributing
+**Please note**: this is the Getting Started guide for developing and contributing to Nomad, NOT [installing Nomad](https://github.com/Crosstalk-Solutions/project-nomad/blob/main/README.md) for regular use! 
+
+### Prerequisites
+
+- A Debian-based OS (Ubuntu recommended)
+- `sudo`/root privileges
+- Docker installed and running
+- A stable internet connection (required for dependency downloads)
+- Node.js (for frontend/admin work)
+
+### Fork & Clone
+
+1. Click **Fork** at the top right of this repository
+2. Clone your fork locally:
+   ```bash
+   git clone https://github.com/YOUR_USERNAME/project-nomad.git
+   cd project-nomad
+   ```
+3. Add the upstream remote so you can stay in sync:
+   ```bash
+   git remote add upstream https://github.com/Crosstalk-Solutions/project-nomad.git
+   ```
+
+### Avoid Installing a Release Version Locally
+Because Nomad relies heavily on Docker, we actually recommend against installing a release version of the project on the same local machine where you are developing. This can lead to conflicts with ports, volumes, and other resources. Instead, you can run your development version in a separate Docker environment while keeping your local machine clean. It certainly __can__ be done, but it adds complexity to your setup and workflow. If you choose to install a release version locally, please ensure you have a clear strategy for managing potential conflicts and resource usage.
+
+---
+
+## Development Workflow
+
+1. **Sync with upstream** before starting any new work. We prefer rebasing over merge commits to keep a clean, linear git history as much as possible (this also makes it easier for maintainers to review and merge your changes). To sync with upstream:
+   ```bash
+   git fetch upstream
+   git checkout dev
+   git rebase upstream/dev
+   ```
+
+2. **Create a feature branch** off `dev` with a descriptive name:
+   ```bash
+   git checkout -b fix/issue-123
+   # or
+   git checkout -b feature/add-new-tool
+   ```
+
+3. **Make your changes.** Follow existing code style and conventions. Test your changes locally against a running N.O.M.A.D. instance before submitting.
+
+4. **Add release notes** (see [Release Notes](#release-notes) below).
+
+5. **Commit your changes** using [Conventional Commits](#commit-messages).
+
+6. **Push your branch** and open a pull request.
+
+---
+
+## Commit Messages
+
+This project uses [Conventional Commits](https://www.conventionalcommits.org/). All commit messages must follow this format:
+
+```
+<type>(<scope>): <description>
+```
+
+**Common types:**
+
+| Type | When to use |
+|------|-------------|
+| `feat` | A new user-facing feature |
+| `fix` | A bug fix |
+| `docs` | Documentation changes only |
+| `refactor` | Code change that isn't a fix or feature and does not affect functionality |
+| `chore` | Build process, dependency updates, tooling |
+| `test` | Adding or updating tests |
+
+**Scope** is optional but encouraged — use it to indicate the area of the codebase affected (e.g., `api`, `ui`, `maps`).
+
+**Examples:**
+```
+feat(ui): add dark mode toggle to Command Center
+fix(api): resolve container status not updating after restart
+docs: update hardware requirements in README
+chore(deps): bump docker-compose to v2.24
+```
+
+---
+
+## Release Notes
+
+Human-readable release notes live in [`admin/docs/release-notes.md`](admin/docs/release-notes.md) and are displayed directly in the Command Center UI.
+
+If your PR is merged in, the maintainers will update the release notes with a summary of your contribution and credit you as the author. You do not need to add this yourself in the PR (please don't, as it may cause merge conflicts), but you can include a suggested note in the PR description if you like.
+
+---
+
+## Versioning
+
+This project uses [Semantic Versioning](https://semver.org/). Versions are managed in the root `package.json` and updated automatically by `semantic-release`. The `project-nomad` Docker image uses this version. The `admin/package.json` version stays at `0.0.0` and should not be changed manually.
+
+---
+
+## Submitting a Pull Request
+
+1. Push your branch to your fork:
+   ```bash
+   git push origin your-branch-name
+   ```
+2. Open a pull request against the `dev` branch of this repository
+3. In the PR description:
+   - Summarize what your changes do and why
+   - Reference the related issue (e.g., `Closes #123`) — required for non-trivial changes
+   - Note any relevant testing steps or environment details
+4. Be responsive to feedback — maintainers may request changes. Pull requests with no activity for an extended period may be closed.
+
+---
+
+## Feedback & Community
+
+Have questions or want to discuss ideas before opening an issue? Join the community:
+
+- **Discord:** [Join the Crosstalk Solutions server](https://discord.com/invite/crosstalksolutions) — the best place to get help, share your builds, and talk with other N.O.M.A.D. users
+- **Website:** [www.projectnomad.us](https://www.projectnomad.us)
+- **Benchmark Leaderboard:** [benchmark.projectnomad.us](https://benchmark.projectnomad.us)
+
+---
+
+*Project N.O.M.A.D. is licensed under the [Apache License 2.0](LICENSE).*

Dockerfile

@@ -1,7 +1,15 @@
-FROM node:22.16.0-alpine3.22 AS base
+FROM node:22-slim AS base
 
-# Install bash & curl for entrypoint script compatibility
-RUN apk add --no-cache bash curl
+# Install bash & curl for entrypoint script compatibility, graphicsmagick for pdf2pic, and vips-dev & build-base for sharp 
+RUN apt-get update && apt-get install -y \
+      bash \
+      curl \
+      openssl \
+      graphicsmagick \
+      libvips-dev \
+      build-essential \
+      pciutils \
+      && rm -rf /var/lib/apt/lists/*
 
 # All deps stage
 FROM base AS deps
@@ -24,12 +32,66 @@ RUN node ace build
 
 # Production stage
 FROM base
+ARG VERSION=dev
+ARG BUILD_DATE
+ARG VCS_REF
+ARG TARGETARCH
+
+# go-pmtiles (regional map extracts). Pinned so the CLI's stdout format stays
+# in sync with parseDryRunOutput().
+ARG PMTILES_VERSION=1.30.2
+# Upstream releases don't ship a checksums file, so pin per-arch SHA256 here.
+# When bumping PMTILES_VERSION, regenerate these with:
+#   curl -fsSL <release-url> | sha256sum
+ARG PMTILES_SHA256_AMD64=2cd3aa18868297fc88425038f794efdc0995e0275f4ca16fa496dd79e245a40c
+ARG PMTILES_SHA256_ARM64=804cdf071834e1156af554c1a26cc42b56b9cde5a2db9c6e3653d16fb846d5fa
+RUN set -eux; \
+    case "${TARGETARCH:-amd64}" in \
+      amd64) PMTILES_ARCH=x86_64; PMTILES_SHA256="${PMTILES_SHA256_AMD64}" ;; \
+      arm64) PMTILES_ARCH=arm64;  PMTILES_SHA256="${PMTILES_SHA256_ARM64}" ;; \
+      *) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \
+    esac; \
+    TARBALL="go-pmtiles_${PMTILES_VERSION}_Linux_${PMTILES_ARCH}.tar.gz"; \
+    cd /tmp; \
+    curl -fsSL -o "$TARBALL" \
+      "https://github.com/protomaps/go-pmtiles/releases/download/v${PMTILES_VERSION}/${TARBALL}"; \
+    echo "${PMTILES_SHA256}  ${TARBALL}" | sha256sum -c -; \
+    tar -xzf "$TARBALL" -C /usr/local/bin pmtiles; \
+    rm -f "$TARBALL"; \
+    chmod +x /usr/local/bin/pmtiles; \
+    /usr/local/bin/pmtiles version
+
+# Labels
+LABEL org.opencontainers.image.title="Project N.O.M.A.D" \
+      org.opencontainers.image.description="The Project N.O.M.A.D Official Docker image" \
+      org.opencontainers.image.version="${VERSION}" \
+      org.opencontainers.image.created="${BUILD_DATE}" \
+      org.opencontainers.image.revision="${VCS_REF}" \
+      org.opencontainers.image.vendor="Crosstalk Solutions, LLC" \
+      org.opencontainers.image.documentation="https://github.com/CrosstalkSolutions/project-nomad/blob/main/README.md" \
+      org.opencontainers.image.source="https://github.com/CrosstalkSolutions/project-nomad" \
+      org.opencontainers.image.licenses="Apache-2.0"
+
 ENV NODE_ENV=production
 WORKDIR /app
 COPY --from=production-deps /app/node_modules /app/node_modules
 COPY --from=build /app/build /app
-# Copy root package.json for version info
-COPY package.json /app/version.json
+# Generate version.json from the VERSION build-arg so the image tag is the
+# single source of truth (previously copied root package.json, which drifted
+# from the tag when semantic-release did not commit the bump back).
+RUN echo "{\"version\":\"${VERSION}\"}" > /app/version.json
+
+# Copy docs and README for access within the container
 COPY admin/docs /app/docs
+COPY README.md /app/README.md
+
+# Empty Calibre library, seeded into storage/books on Calibre-Web install
+# (see DockerService._runPreinstallActions__CalibreWeb)
+COPY install/calibre-empty-library/metadata.db /app/assets/calibre/metadata.db
+
+# Copy entrypoint script and ensure it's executable
+COPY install/entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
 EXPOSE 8080
-CMD ["node", "./bin/server.js"]
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

FAQ.md

@@ -0,0 +1,108 @@
+# Frequently Asked Questions (FAQ)
+
+Find answers to some of the most common questions about Project N.O.M.A.D.
+
+## Can I customize the port(s) that NOMAD uses?
+
+Yes, you can customize the ports that NOMAD's core services (Command Center, MySQL, Redis) use. Please refer to the [Advanced Installation](README.md#advanced-installation) section of the README for more details on how to do this.
+
+Note: As of 3/24/2026, only the core services defined in the `docker-compose.yml` file currently support port customization - the installable applications (e.g. Ollama, Kiwix, etc.) do not yet support this, but we have multiple PR's in the works to add this feature for all installable applications in a future release.
+
+## Can I customize the storage location for NOMAD's data?
+
+Yes, you can customize the storage location for NOMAD's content by modifying the `docker-compose.yml` file to adjust the appropriate bind mounts to point to your desired storage location on your host machine. Please refer to the [Advanced Installation](README.md#advanced-installation) section of the README for more details on how to do this.
+
+## Can I store NOMAD's data on an external drive or network storage?
+
+Short answer: yes, but we can't do it for you (and we recommend a local drive for best performance).
+
+Long answer: Custom storage paths, mount points, and external drives (like iSCSI or SMB/NFS volumes) **are possible**, but this will be up to your individual configuration on the host before NOMAD starts, and then passed in via the compose.yml as this is a *host-level concern*, not a NOMAD-level concern (see above for details). NOMAD itself can't configure this for you, nor could we support all possible configurations in the install script.
+
+## Can I run NOMAD on MAC, WSL2, or a non-Debian-based Distro?
+
+**WSL2 on Windows** is community-supported via the [WSL2 install guide](https://www.projectnomad.us/install/wsl2) — covers two install paths (native Docker and Docker Desktop) with all known gotchas documented and empirical performance numbers comparing WSL2 to bare-metal.
+
+**macOS and other non-Debian Linux distros** aren't officially supported. See [Why does NOMAD require a Debian-based OS?](#why-does-nomad-require-a-debian-based-os) for details.
+
+## Why does NOMAD require a Debian-based OS?
+
+Project N.O.M.A.D. is currently designed to run on Debian-based Linux distributions (with Ubuntu being the recommended distro) because our installation scripts and Docker configurations are optimized for this environment. While it's technically possible to run the Docker containers on other operating systems that support Docker, we have not tested or optimized the installation process for non-Debian-based systems, so we cannot guarantee a smooth experience on those platforms at this time.
+
+Support for other operating systems will come in the future, but because our development resources are limited as a free and open-source project, we needed to prioritize our efforts and focus on a narrower set of supported platforms for the initial release. We chose Debian-based Linux as our starting point because it's widely used, easy to spin up, and provides a stable environment for running Docker containers.
+
+For Windows users, the [WSL2 install guide](https://www.projectnomad.us/install/wsl2) provides a community-supported path. Community members have also published guides for other platforms (e.g. macOS) in our Discord community and [Github Discussions](https://github.com/Crosstalk-Solutions/project-nomad/discussions), so if you're interested in running N.O.M.A.D. on a non-Debian-based system, we recommend checking there for any available resources or guides. However, keep in mind that if you choose to run N.O.M.A.D. on a non-Debian-based system, you may encounter issues that we won't be able to provide support for, and you may need to have a higher level of technical expertise to troubleshoot and resolve any problems that arise.
+
+## Can I run NOMAD on a Raspberry Pi or other ARM-based device?
+Project N.O.M.A.D. is currently designed to run on x86-64 architecture, and we have not yet tested or optimized it for ARM-based devices like the Raspberry Pi (and have not published any official images for ARM architecture).
+
+Support for ARM-based devices is on our roadmap, but our initial focus was on x86-64 hardware due to its widespread use and compatibility with a wide range of applications.
+
+Community members have forked and published their own ARM-compatible images and installation guides for running N.O.M.A.D. on Raspberry Pi and other ARM-based devices in our Discord community and [Github Discussions](https://github.com/Crosstalk-Solutions/project-nomad/discussions), but these are not officially supported by the core development team, and we cannot guarantee their functionality or provide support for any issues that arise when using these community-created resources.
+
+## What are the hardware requirements for running NOMAD?
+
+Project N.O.M.A.D. itself is quite lightweight and can run on even modest x86-64 hardware, but the tools and resources you choose to install with N.O.M.A.D. will determine the specs required for your unique deployment. Please see the [Hardware Guide](https://www.projectnomad.us/hardware) for detailed build recommendations at various price points.
+
+## Does NOMAD support languages other than English?
+
+As of March 2026, Project N.O.M.A.D.'s UI is only available in English, and the majority of the tools and resources available through N.O.M.A.D. are also primarily in English. However, we have multi-language support on our roadmap for a future release, and we are actively working on adding support for additional languages both in the UI and in the available tools/resources. If you're interested in contributing to this effort, please check out our [CONTRIBUTING.md](CONTRIBUTING.md) file for guidelines on how to get involved.
+
+## What technologies is NOMAD built with?
+
+Project N.O.M.A.D. is built using a combination of technologies, including:
+- **Docker:** for containerization of the Command Center and its dependencies
+- **Node.js & TypeScript:** for the backend of the Command Center, particularly the [AdonisJS](https://adonisjs.com/) framework
+- **React:** for the frontend of the Command Center, utilizing [Vite](https://vitejs.dev/) and [Inertia.js](https://inertiajs.com/) under the hood
+- **MySQL:** for the Command Center's database
+- **Redis:** for various caching, background jobs, "cron" tasks, and other internal processes within the Command Center
+
+NOMAD makes use of the Docker-outside-of-Docker ("DooD") pattern, which allows the Command Center to manage and orchestrate other Docker containers on the host machine without needing to run Docker itself inside a container. This approach provides better performance and compatibility with a wider range of host environments while still allowing for powerful container management capabilities through the Command Center's UI.
+
+## Can I run NOMAD if I have existing Docker containers on my machine?
+Yes, you can safely run Project N.O.M.A.D. on a machine that already has existing Docker containers. NOMAD is designed to coexist with other Docker containers and will not interfere with them as long as there are no port conflicts or resource constraints.
+
+All of NOMAD's containers are prefixed with `nomad_` in their names, so they can be easily identified and managed separately from any other containers you may have running. Just make sure to review the ports that NOMAD's core services (Command Center, MySQL, Redis) use during installation and adjust them if necessary to avoid conflicts with your existing containers.
+
+## Why does NOMAD require access to the Docker socket?
+
+See [What technologies is NOMAD built with?](#what-technologies-is-nomad-built-with)
+
+## Can I use any AI models?
+NOMAD by default uses Ollama inside of a docker container to run LLM Models for the AI Assistant. So if you find a model on HuggingFace for example, you won't be able to use that model in NOMAD. The list of available models in the AI Assistant settings (/settings/models) may not show all of the models you are looking for. If you found a model from https://ollama.com/search that you'd like to try and its not in the settings page, you can use a curl command to download the model.  
+`curl -X POST -H "Content-Type: application/json" -d '{"model":"MODEL_NAME_HERE"}' http://localhost:8080/api/ollama/models` replacing MODEL_NAME_HERE with the model name from whats in the ollama website.
+
+## Do I have to install the AI features in NOMAD?
+
+No, the AI features in NOMAD (Ollama, Qdrant, custom RAG pipeline, etc.) are all optional and not required to use the core functionality of NOMAD.
+
+## Is NOMAD actually free? Are there any hidden costs?
+Yes, Project N.O.M.A.D. is completely free and open-source software licensed under the Apache License 2.0. There are no hidden costs or fees associated with using NOMAD itself, and we don't have any plans to introduce "premium" features or paid tiers.
+
+Aside from the cost of the hardware you choose to run it on, there are no costs associated with using NOMAD.
+
+## Do you sell hardware or pre-built devices with NOMAD pre-installed?
+
+No, we do not sell hardware or pre-built devices with NOMAD pre-installed at this time. Project N.O.M.A.D. is a free and open-source software project, and we provide detailed installation instructions and hardware recommendations for users to set up their own NOMAD instances on compatible hardware of their choice. The tradeoff to this DIY approach is some additional setup time and technical know-how required on the user's end, but it also allows for greater flexibility and customization in terms of hardware selection and configuration to best suit each user's unique needs, budget, and preferences.
+
+## How quickly are issues resolved when reported?
+
+We strive to address and resolve issues as quickly as possible, but please keep in mind that Project N.O.M.A.D. is a free and open-source project maintained by a small team of volunteers. We prioritize issues based on their severity, impact on users, and the resources required to resolve them. Critical issues that affect a large number of users are typically addressed more quickly, while less severe issues may take longer to resolve. Aside from the development efforts needed to address the issue, we do our best to conduct thorough testing and validation to ensure that any fix we implement doesn't introduce new issues or regressions, which also adds to the time it takes to resolve an issue.
+
+We also encourage community involvement in troubleshooting and resolving issues, so if you encounter a problem, please consider checking our Discord community and Github Discussions for potential solutions or workarounds while we work on an official fix.
+
+## How often are new features added or updates released?
+
+We aim to release updates and new features on a regular basis, but the exact timing can vary based on the complexity of the features being developed, the resources available to our volunteer development team, and the feedback and needs of our community. We typically release smaller "patch" versions more frequently to address bugs and make minor improvements, while larger feature releases may take more time to develop and test before they're ready for release.
+
+## I opened a PR to contribute a new feature or fix a bug. How long does it usually take for PRs to be reviewed and merged?
+We appreciate all contributions to the project and strive to review and merge pull requests (PRs) as quickly as possible. The time it takes for a PR to be reviewed and merged can vary based on several factors, including the complexity of the changes, the current workload of our maintainers, and the need for any additional testing or revisions.
+
+Because NOMAD is still a young project, some PRs (particularly those for new features) may take longer to review and merge as we prioritize building out the core functionality and ensuring stability before adding new features. However, we do our best to provide timely feedback on all PRs and keep contributors informed about the status of their contributions.
+
+## I have a question that isn't answered here. Where can I ask for help?
+
+If you have a question that isn't answered in this FAQ, please feel free to ask for help in our Discord community (https://discord.com/invite/crosstalksolutions) or on our Github Discussions page (https://github.com/Crosstalk-Solutions/project-nomad/discussions).
+
+## I have a suggestion for a new feature or improvement. How can I share it?
+
+We welcome and encourage suggestions for new features and improvements! We highly encourage sharing your ideas (or upvoting existing suggestions) on our public roadmap at https://roadmap.projectnomad.us, where we track new feature requests. This is the best way to ensure that your suggestion is seen by the development team and the community, and it also allows other community members to upvote and show support for your idea, which can help prioritize it for future development.

LICENSE

@@ -0,0 +1,190 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to the Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by the Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding any notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   Copyright 2024-2026 Crosstalk Solutions LLC
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,40 +1,79 @@
-# *NOTE: Project N.O.M.A.D. is still in active development and should not be considered stable!*
+<div align="center">
+<img src="admin/public/project_nomad_logo.webp" width="200" height="200"/>
+
+# Project N.O.M.A.D.
+### Node for Offline Media, Archives, and Data
+
+**Knowledge That Never Goes Offline**
+
+[![Website](https://img.shields.io/badge/Website-projectnomad.us-blue)](https://www.projectnomad.us)
+[![Discord](https://img.shields.io/badge/Discord-Join%20Community-5865F2)](https://discord.com/invite/crosstalksolutions)
+[![Benchmark](https://img.shields.io/badge/Benchmark-Leaderboard-green)](https://benchmark.projectnomad.us)
 
-<div style="width: 100;text-align: center;margin-bottom: 25px;">
-<img src="https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/master/admin/public/project_nomad_logo.png" width="200" height="200"/>
 </div>
 
-# Project N.O.M.A.D. (Node for Offline Media, Archives, and Data)
-Project N.O.M.A.D., is a self-contained, offline survival computer packed with critical tools, knowledge, and AI to keep you informed and empowered—anytime, anywhere.
+---
+
+Project N.O.M.A.D. is a self-contained, offline-first knowledge and education server packed with critical tools, knowledge, and AI to keep you informed and empowered — anytime, anywhere.
 
 ## Installation & Quickstart
 Project N.O.M.A.D. can be installed on any Debian-based operating system (we recommend Ubuntu). Installation is completely terminal-based, and all tools and resources are designed to be accessed through the browser, so there's no need for a desktop environment if you'd rather setup N.O.M.A.D. as a "server" and access it through other clients.
 
 *Note: sudo/root privileges are required to run the install script*
 
+### Quick Install (Debian-based OS Only)
 ```bash
-curl -fsSL https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/master/install/install_nomad.sh -o install_nomad.sh
-```
-
-```bash
+sudo apt-get update && \
+sudo apt-get install -y curl && \
+curl -fsSL https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/install/install_nomad.sh \
+  -o install_nomad.sh && \
 sudo bash install_nomad.sh
 ```
 
 Project N.O.M.A.D. is now installed on your device! Open a browser and navigate to `http://localhost:8080` (or `http://DEVICE_IP:8080`) to start exploring!
 
-## How It Works
-From a technical standpoint, N.O.M.A.D. is primarily a management UI ("Command Center") and API that orchestrates a goodie basket of containerized offline archive tools and resources such as 
-[Kiwix](https://kiwix.org/), [OpenStreetMap](https://www.openstreetmap.org/), [Ollama](https://ollama.com/), [OpenWebUI](https://openwebui.com/), and more.
+For a complete step-by-step walkthrough (including Ubuntu installation), see the [Installation Guide](https://www.projectnomad.us/install). For Windows users, see the [WSL2 install guide](https://www.projectnomad.us/install/wsl2) — community-supported path covering native Docker and Docker Desktop install routes.
 
-By abstracting the installation of each of these awesome tools, N.O.M.A.D. makes getting your offline survival computer up and running a breeze! N.O.M.A.D. also includes some additional built-in handy tools, such as a ZIM library managment interface, calculators, and more.
+### Advanced Installation
+For more control over the installation process, copy and paste the [Docker Compose template](https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/install/management_compose.yaml) into a `docker-compose.yml` file and customize it to your liking (be sure to replace any placeholders with your actual values). Then, run `docker compose up -d` to start the Command Center and its dependencies. Note: this method is recommended for advanced users only, as it requires familiarity with Docker and manual configuration before starting.
+
+## How It Works
+N.O.M.A.D. is a management UI ("Command Center") and API that orchestrates a collection of containerized tools and resources via [Docker](https://www.docker.com/). It handles installation, configuration, and updates for everything — so you don't have to.
+
+**Built-in capabilities include:**
+- **AI Chat with Knowledge Base** — local AI chat powered by [Ollama](https://ollama.com/) or you can use OpenAI API compatible software such as LM Studio or llama.cpp, with document upload and semantic search (RAG via [Qdrant](https://qdrant.tech/))
+- **Information Library** — offline Wikipedia, medical references, ebooks, and more via [Kiwix](https://kiwix.org/)
+- **Education Platform** — Khan Academy courses with progress tracking via [Kolibri](https://learningequality.org/kolibri/)
+- **Offline Maps** — downloadable regional maps via [ProtoMaps](https://protomaps.com)
+- **Data Tools** — encryption, encoding, and analysis via [CyberChef](https://gchq.github.io/CyberChef/)
+- **Notes** — local note-taking via [FlatNotes](https://github.com/dullage/flatnotes)
+- **System Benchmark** — hardware scoring with a [community leaderboard](https://benchmark.projectnomad.us)
+- **Supply Depot** — a one-click app catalog (PDF tools, file browser, e-book library, password manager, and more) plus the ability to run your own custom Docker containers
+- **Automatic Updates** — opt-in, hands-off updates for the core software, installed apps, and offline content, on a schedule you control
+- **Easy Setup Wizard** — guided first-time configuration with curated content collections
+
+N.O.M.A.D. also includes built-in tools like a Wikipedia content selector, ZIM library manager, and content explorer.
+
+## What's Included
+
+| Capability | Powered By | What You Get |
+|-----------|-----------|-------------|
+| Information Library | Kiwix | Offline Wikipedia, medical references, survival guides, ebooks |
+| AI Assistant | Ollama + Qdrant | Built-in chat with document upload and semantic search |
+| Education Platform | Kolibri | Khan Academy courses, progress tracking, multi-user support |
+| Offline Maps | ProtoMaps | Downloadable regional maps for offline viewing and search |
+| Data Tools | CyberChef | Encryption, encoding, hashing, and data analysis |
+| Notes | FlatNotes | Local note-taking with markdown support |
+| System Benchmark | Built-in | Hardware scoring, Builder Tags, and community leaderboard |
+| Supply Depot | Built-in | One-click app catalog + bring-your-own custom Docker containers |
 
 ## Device Requirements
 While many similar offline survival computers are designed to be run on bare-minimum, lightweight hardware, Project N.O.M.A.D. is quite the opposite. To install and run the
 available AI tools, we highly encourage the use of a beefy, GPU-backed device to make the most of your install.
 
-At it's core, however, N.O.M.A.D. is still very lightweight. For a barebones installation of the management application itself, the following minimal specs are required:
+At its core, however, N.O.M.A.D. is still very lightweight. For a barebones installation of the management application itself, the following minimal specs are required:
 
-*Note: Project N.O.M.A.D. is not sponsored by any hardware manufacturer and is designed to be as hardware-agnostic as possible. The harware listed below is for example/comparison use only*
+*Note: Project N.O.M.A.D. is not sponsored by any hardware manufacturer and is designed to be as hardware-agnostic as possible. The hardware listed below is for example/comparison use only*
 
 #### Minimum Specs
 - Processor: 2 GHz dual-core processor or better
@@ -43,30 +82,92 @@ At it's core, however, N.O.M.A.D. is still very lightweight. For a barebones ins
 - OS: Debian-based (Ubuntu recommended)
 - Stable internet connection (required during install only)
 
-To run LLM's and other included AI tools:
+To run LLMs and other included AI tools:
 
 #### Optimal Specs
 - Processor: AMD Ryzen 7 or Intel Core i7 or better
 - RAM: 32 GB system memory
-- Graphics: NVIDIA RTX 3060 or better (more VRAM = run larger models)
+- Graphics: NVIDIA RTX 3060 or AMD equivalent or better (more VRAM = run larger models)
 - Storage: At least 250 GB free disk space (preferably on SSD)
 - OS: Debian-based (Ubuntu recommended)
 - Stable internet connection (required during install only)
 
-Again, Project N.O.M.A.D. itself is quite lightweight - it's the tools and resources you choose to install with N.O.M.A.D. that will determine the specs required for your unique deployment
+**For detailed build recommendations at three price points ($150–$1,000+), see the [Hardware Guide](https://www.projectnomad.us/hardware).**
+
+Again, Project N.O.M.A.D. itself is quite lightweight — it's the tools and resources you choose to install with N.O.M.A.D. that will determine the specs required for your unique deployment
+
+#### Running AI models on a different host
+By default, N.O.M.A.D.'s installer will attempt to setup Ollama on the host when the AI Assistant is installed. However, if you would like to run the AI model on a different host, you can go to the settings of the AI assistant and input a URL for either an ollama or OpenAI-compatible API server (such as LM Studio).  
+Note that if you use Ollama on a different host, you must start the server with this option: `OLLAMA_HOST=0.0.0.0`.  
+Ollama is the preferred way to use the AI assistant, as it has features such as model download that OpenAI API does not support. So when using LM Studio, for example, you will have to use LM Studio to download models.
+You are responsible for the setup of Ollama/OpenAI server on the other host.
+
+## Frequently Asked Questions (FAQ)
+For answers to common questions about Project N.O.M.A.D., please see our [FAQ](FAQ.md) page.
 
 ## About Internet Usage & Privacy
 Project N.O.M.A.D. is designed for offline usage. An internet connection is only required during the initial installation (to download dependencies) and if you (the user) decide to download additional tools and resources at a later time. Otherwise, N.O.M.A.D. does not require an internet connection and has ZERO built-in telemetry.
 
-To test internet connectivity, N.O.M.A.D. attempts to make a request to Cloudflare's utility endpoint, `https://1.1.1.1/cdn-cgi/trace` and checks for a successful response.
+To test internet connectivity, N.O.M.A.D. first attempts to make a request to Cloudflare's utility endpoint, `https://1.1.1.1/cdn-cgi/trace`. If that endpoint is unreachable (for example, because your network blocks `1.1.1.1`), it falls back to other endpoints the application already contacts (the GitHub API and the Project N.O.M.A.D. API) and considers the connection online if any of them respond.
+
+You can override the endpoint used for this check in two ways. The connectivity test URL can be configured from the UI under **Settings → Advanced** (stored locally on your instance), or you can set the `INTERNET_STATUS_TEST_URL` environment variable. When set, the environment variable always takes precedence over the UI-configured value. If neither is set, the built-in defaults above are used.
 
 ## About Security
-By design, Project N.O.M.A.D. is intended to be open and available without hurdles - it includes no authentication. If you decide to connect your device to a local network after install (e.g. for allowing other devices to access it's resources), you can block/open ports to control which services are exposed.
+By design, Project N.O.M.A.D. is intended to be open and available without hurdles — it includes no authentication. If you decide to connect your device to a local network after install (e.g. for allowing other devices to access its resources), you can block/open ports to control which services are exposed.
 
-## Versioning
-This project uses semantic versioning. The version is managed in the root `package.json` 
-and automatically updated by semantic-release. For simplicity's sake, the "project-nomad" container
-uses the same version defined there instead of the version in `admin/package.json` (stays at 0.0.0), as it's the only container derived from the code.
+**Will authentication be added in the future?** Maybe. It's not currently a priority, but if there's enough demand for it, we may consider building in an optional authentication layer in a future release to support use cases where multiple users need access to the same instance but with different permission levels (e.g. family use with parental controls, classroom use with teacher/admin accounts, etc.). We have a suggestion for this on our public roadmap, so if this is something you'd like to see, please upvote it here: https://roadmap.projectnomad.us/posts/1/user-authentication-please-build-in-user-auth-with-admin-user-roles
+
+For now, we recommend using network-level controls to manage access if you're planning to expose your N.O.M.A.D. instance to other devices on a local network. N.O.M.A.D. is not designed to be exposed directly to the internet, and we strongly advise against doing so unless you really know what you're doing, have taken appropriate security measures, and understand the risks involved.
+
+## Contributing
+Contributions are welcome and appreciated! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines on how to contribute to the project.
+
+### Testing Auto-Updates (Dry Run)
+
+The Command Center can automatically install **minor/patch** updates of itself during a configurable window, after a cool-off period, and only when pre-flight checks pass (sufficient disk for the new image, no downloads or app installs in progress). Major versions always require a manual update.
+
+Because exercising this logic with real version bumps is impractical, an Ace command runs the **entire decision pipeline without ever triggering an update**. Run it from the `admin/` directory:
+
+```bash
+# 1) Deterministic scenario suite — no network, DB, or Docker required.
+#    Proves every branch (major-only, cool-off, prerelease/draft, window wrap, …)
+#    and exits non-zero on failure, so it's safe to wire into CI.
+node ace auto-update:dry-run --scenarios
+
+# 2) Simulate "what would happen if I were running 1.32.0 right now?"
+#    against the LIVE GitHub releases feed and real pre-flight checks:
+node ace auto-update:dry-run --current=1.32.0 --force-enabled
+
+# 3) Fully offline simulation with a canned release list and a fixed clock:
+node ace auto-update:dry-run --current=1.32.0 --force-enabled \
+  --releases-file=./fixtures/releases.json --now=2026-06-04T21:00:00Z \
+  --window-start=20:00 --window-end=23:00 --cooloff=72 --skip-preflight
+```
+
+It prints the resolved decision — current version, whether the clock is inside the window, the eligible target (if any), and pre-flight blockers — ending in a clear verdict such as `WOULD UPDATE → v1.33.2` or `WOULD NOT UPDATE (outside-window): …`. **No real update is ever requested.**
+
+| Flag | Description |
+|------|-------------|
+| `--scenarios` | Run the built-in deterministic scenario suite and exit |
+| `--current=<version>` | Simulate this currently-running version (e.g. `1.32.0`) |
+| `--force-enabled` | Treat auto-update as enabled, ignoring the saved setting |
+| `--cooloff=<hours>` | Override the cool-off period |
+| `--window-start=<HH:MM>` / `--window-end=<HH:MM>` | Override the update window |
+| `--now=<ISO timestamp>` | Simulate the clock at a specific time |
+| `--releases-file=<path>` | Use a local JSON releases array instead of fetching GitHub (offline) |
+| `--skip-preflight` | Bypass the Docker/disk/queue pre-flight checks |
+
+## Community & Resources
+
+- **Website:** [www.projectnomad.us](https://www.projectnomad.us) - Learn more about the project
+- **Discord:** [Join the Community](https://discord.com/invite/crosstalksolutions) - Get help, share your builds, and connect with other NOMAD users
+- **Benchmark Leaderboard:** [benchmark.projectnomad.us](https://benchmark.projectnomad.us) - See how your hardware stacks up against other NOMAD builds
+- **FAQ:** [FAQ.md](FAQ.md) - Find answers to frequently asked questions
+- **Community Add-Ons:** [admin/docs/community-add-ons.md](admin/docs/community-add-ons.md) - Third-party content packs built by the community
+
+## License
+
+Project N.O.M.A.D. is licensed under the [Apache License 2.0](LICENSE).
 
 ## Helper Scripts
 Once installed, Project N.O.M.A.D. has a few helper scripts should you ever need to troubleshoot issues or perform maintenance that can't be done through the Command Center. All of these scripts are found in Project N.O.M.A.D.'s install directory, `/opt/project-nomad`
@@ -81,7 +182,7 @@ sudo bash /opt/project-nomad/start_nomad.sh
 
 ###### Stop Script - Stops all installed project containers
 ```bash
-sudo bash /opt/project-nomad/start_nomad.sh
+sudo bash /opt/project-nomad/stop_nomad.sh
 ```
 ###
 
@@ -92,9 +193,5 @@ sudo bash /opt/project-nomad/update_nomad.sh
 
 ###### Uninstall Script - Need to start fresh? Use the uninstall script to make your life easy. Note: this cannot be undone!
 ```bash
-curl -fsSL https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/master/install/uninstall_nomad.sh -o uninstall_nomad.sh
+curl -fsSL https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/install/uninstall_nomad.sh -o uninstall_nomad.sh && sudo bash uninstall_nomad.sh
 ```
-
-```bash
-sudo bash uninstall_nomad.sh
-```

admin/.env.example

@@ -1,6 +1,10 @@
 PORT=8080
 HOST=localhost
 LOG_LEVEL=info
+# Optional: override the URL used to test internet connectivity.
+# Defaults to https://1.1.1.1/cdn-cgi/trace with fallbacks to hosts the app
+# already contacts. Leave unset to use the defaults.
+# INTERNET_STATUS_TEST_URL=https://1.1.1.1/cdn-cgi/trace
 APP_KEY=some_random_key
 NODE_ENV=development
 SESSION_DRIVER=cookie
@@ -12,6 +16,9 @@ DB_PASSWORD=password
 DB_SSL=false
 REDIS_HOST=localhost
 REDIS_PORT=6379
+# Optional: Redis logical database index (0-15). Defaults to 0 if unset.
+# Set this when sharing a Redis instance across services to avoid key collisions.
+# REDIS_DB=0
 # Storage path for NOMAD content (ZIM files, maps, etc.)
 # On Windows dev, use an absolute path like: C:/nomad-storage
 # On Linux production, use: /opt/project-nomad/storage

admin/README.md

@@ -1,5 +0,0 @@
-
-## Docker container
-```
-docker run --rm -it -p 8080:8080 jturnercosmistack/projectnomad:admin-latest -e PORT=8080 -e HOST=0.0.0.0 -e APP_KEY=secretlongpasswordsecret -e LOG_LEVEL=debug -e DRIVE_DISK=fs
-```

admin/adonisrc.ts

@@ -53,7 +53,11 @@ export default defineConfig({
     () => import('@adonisjs/lucid/database_provider'),
     () => import('@adonisjs/inertia/inertia_provider'),
     () => import('@adonisjs/transmit/transmit_provider'),
-    () => import('#providers/map_static_provider')
+    () => import('#providers/map_static_provider'),
+    () => import('#providers/kiwix_migration_provider'),
+    () => import('#providers/qdrant_restart_policy_provider'),
+    () => import('#providers/version_check_provider'),
+    () => import('#providers/gpu_passthrough_remediation_provider'),
   ],
 
   /*
@@ -105,6 +109,10 @@ export default defineConfig({
       pattern: 'resources/views/**/*.edge',
       reloadServer: false,
     },
+    {
+      pattern: 'resources/geodata/**/*.geojson',
+      reloadServer: false,
+    },
     {
       pattern: 'public/**',
       reloadServer: false,

admin/app/controllers/benchmark_controller.ts

@@ -0,0 +1,278 @@
+import { inject } from '@adonisjs/core'
+import type { HttpContext } from '@adonisjs/core/http'
+import { BenchmarkService } from '#services/benchmark_service'
+import { runBenchmarkValidator, submitBenchmarkValidator } from '#validators/benchmark'
+import { RunBenchmarkJob } from '#jobs/run_benchmark_job'
+import type { BenchmarkType } from '../../types/benchmark.js'
+import { randomUUID } from 'node:crypto'
+import logger from '@adonisjs/core/services/logger'
+
+@inject()
+export default class BenchmarkController {
+  constructor(private benchmarkService: BenchmarkService) {}
+
+  /**
+   * Start a benchmark run (async via job queue, or sync if specified)
+   */
+  async run({ request, response }: HttpContext) {
+    const payload = await request.validateUsing(runBenchmarkValidator)
+    const benchmarkType: BenchmarkType = payload.benchmark_type || 'full'
+    const runSync = request.input('sync') === 'true' || request.input('sync') === true
+
+    // Check if a benchmark is already running
+    const status = this.benchmarkService.getStatus()
+    if (status.status !== 'idle') {
+      return response.status(409).send({
+        success: false,
+        error: 'A benchmark is already running',
+        current_benchmark_id: status.benchmarkId,
+      })
+    }
+
+    // Run synchronously if requested (useful for local dev without Redis)
+    if (runSync) {
+      try {
+        let result
+        switch (benchmarkType) {
+          case 'full':
+            result = await this.benchmarkService.runFullBenchmark()
+            break
+          case 'system':
+            result = await this.benchmarkService.runSystemBenchmarks()
+            break
+          case 'ai':
+            result = await this.benchmarkService.runAIBenchmark()
+            break
+          default:
+            result = await this.benchmarkService.runFullBenchmark()
+        }
+        return response.send({
+          success: true,
+          benchmark_id: result.benchmark_id,
+          nomad_score: result.nomad_score,
+          result,
+        })
+      } catch (error) {
+        logger.error({ err: error }, '[BenchmarkController] Benchmark run failed')
+        return response.status(500).send({
+          success: false,
+          error: 'An internal error occurred while running the benchmark.',
+        })
+      }
+    }
+
+    // Generate benchmark ID and dispatch job (async)
+    const benchmarkId = randomUUID()
+    const { job, created } = await RunBenchmarkJob.dispatch({
+      benchmark_id: benchmarkId,
+      benchmark_type: benchmarkType,
+      include_ai: benchmarkType === 'full' || benchmarkType === 'ai',
+    })
+
+    return response.status(201).send({
+      success: true,
+      job_id: job?.id || benchmarkId,
+      benchmark_id: benchmarkId,
+      message: created
+        ? `${benchmarkType} benchmark started`
+        : 'Benchmark job already exists',
+    })
+  }
+
+  /**
+   * Run a system-only benchmark (CPU, memory, disk)
+   */
+  async runSystem({ response }: HttpContext) {
+    const status = this.benchmarkService.getStatus()
+    if (status.status !== 'idle') {
+      return response.status(409).send({
+        success: false,
+        error: 'A benchmark is already running',
+      })
+    }
+
+    const benchmarkId = randomUUID()
+    await RunBenchmarkJob.dispatch({
+      benchmark_id: benchmarkId,
+      benchmark_type: 'system',
+      include_ai: false,
+    })
+
+    return response.status(201).send({
+      success: true,
+      benchmark_id: benchmarkId,
+      message: 'System benchmark started',
+    })
+  }
+
+  /**
+   * Run an AI-only benchmark
+   */
+  async runAI({ response }: HttpContext) {
+    const status = this.benchmarkService.getStatus()
+    if (status.status !== 'idle') {
+      return response.status(409).send({
+        success: false,
+        error: 'A benchmark is already running',
+      })
+    }
+
+    const benchmarkId = randomUUID()
+    await RunBenchmarkJob.dispatch({
+      benchmark_id: benchmarkId,
+      benchmark_type: 'ai',
+      include_ai: true,
+    })
+
+    return response.status(201).send({
+      success: true,
+      benchmark_id: benchmarkId,
+      message: 'AI benchmark started',
+    })
+  }
+
+  /**
+   * Get all benchmark results
+   */
+  async results({}: HttpContext) {
+    const results = await this.benchmarkService.getAllResults()
+    return {
+      results,
+      total: results.length,
+    }
+  }
+
+  /**
+   * Get the latest benchmark result
+   */
+  async latest({}: HttpContext) {
+    const result = await this.benchmarkService.getLatestResult()
+    if (!result) {
+      return { result: null }
+    }
+    return { result }
+  }
+
+  /**
+   * Get a specific benchmark result by ID
+   */
+  async show({ params, response }: HttpContext) {
+    const result = await this.benchmarkService.getResultById(params.id)
+    if (!result) {
+      return response.status(404).send({
+        error: 'Benchmark result not found',
+      })
+    }
+    return { result }
+  }
+
+  /**
+   * Submit benchmark results to central repository
+   */
+  async submit({ request, response }: HttpContext) {
+    const payload = await request.validateUsing(submitBenchmarkValidator)
+    const anonymous = request.input('anonymous') === true || request.input('anonymous') === 'true'
+
+    try {
+      const submitResult = await this.benchmarkService.submitToRepository(payload.benchmark_id, anonymous)
+      return response.send({
+        success: true,
+        repository_id: submitResult.repository_id,
+        percentile: submitResult.percentile,
+      })
+    } catch (error) {
+      // Pass through the status code from the service if available, otherwise default to 400
+      const statusCode = (error as any).statusCode || 400
+      logger.error({ err: error }, '[BenchmarkController] Benchmark submit failed')
+      return response.status(statusCode).send({
+        success: false,
+        error: 'Failed to submit benchmark results.',
+      })
+    }
+  }
+
+  /**
+   * Update builder tag for a benchmark result
+   */
+  async updateBuilderTag({ request, response }: HttpContext) {
+    const benchmarkId = request.input('benchmark_id')
+    const builderTag = request.input('builder_tag')
+
+    if (!benchmarkId) {
+      return response.status(400).send({
+        success: false,
+        error: 'benchmark_id is required',
+      })
+    }
+
+    const result = await this.benchmarkService.getResultById(benchmarkId)
+    if (!result) {
+      return response.status(404).send({
+        success: false,
+        error: 'Benchmark result not found',
+      })
+    }
+
+    // Validate builder tag format if provided
+    if (builderTag) {
+      const tagPattern = /^[A-Za-z]+-[A-Za-z]+-\d{4}$/
+      if (!tagPattern.test(builderTag)) {
+        return response.status(400).send({
+          success: false,
+          error: 'Invalid builder tag format. Expected: Word-Word-0000',
+        })
+      }
+    }
+
+    result.builder_tag = builderTag || null
+    await result.save()
+
+    return response.send({
+      success: true,
+      builder_tag: result.builder_tag,
+    })
+  }
+
+  /**
+   * Get comparison stats from central repository
+   */
+  async comparison({}: HttpContext) {
+    const stats = await this.benchmarkService.getComparisonStats()
+    return { stats }
+  }
+
+  /**
+   * Get current benchmark status
+   */
+  async status({}: HttpContext) {
+    return this.benchmarkService.getStatus()
+  }
+
+  /**
+   * Get benchmark settings
+   */
+  async settings({}: HttpContext) {
+    const { default: BenchmarkSetting } = await import('#models/benchmark_setting')
+    return await BenchmarkSetting.getAllSettings()
+  }
+
+  /**
+   * Update benchmark settings
+   */
+  async updateSettings({ request, response }: HttpContext) {
+    const { default: BenchmarkSetting } = await import('#models/benchmark_setting')
+    const body = request.body()
+
+    if (body.allow_anonymous_submission !== undefined) {
+      await BenchmarkSetting.setValue(
+        'allow_anonymous_submission',
+        body.allow_anonymous_submission ? 'true' : 'false'
+      )
+    }
+
+    return response.send({
+      success: true,
+      settings: await BenchmarkSetting.getAllSettings(),
+    })
+  }
+}

admin/app/controllers/chats_controller.ts

@@ -0,0 +1,120 @@
+import { inject } from '@adonisjs/core'
+import type { HttpContext } from '@adonisjs/core/http'
+import { ChatService } from '#services/chat_service'
+import { createSessionSchema, updateSessionSchema, addMessageSchema } from '#validators/chat'
+import KVStore from '#models/kv_store'
+import { SystemService } from '#services/system_service'
+import { SERVICE_NAMES } from '../../constants/service_names.js'
+import logger from '@adonisjs/core/services/logger'
+
+@inject()
+export default class ChatsController {
+  constructor(private chatService: ChatService, private systemService: SystemService) {}
+
+  async inertia({ inertia, response }: HttpContext) {
+    const aiAssistantInstalled = await this.systemService.checkServiceInstalled(SERVICE_NAMES.OLLAMA)
+    if (!aiAssistantInstalled) {
+      return response.status(404).json({ error: 'AI Assistant service not installed' })
+    }
+    
+    const chatSuggestionsEnabled = await KVStore.getValue('chat.suggestionsEnabled')
+    return inertia.render('chat', {
+      settings: {
+        chatSuggestionsEnabled: chatSuggestionsEnabled ?? false,
+      },
+    })
+  }
+
+  async index({}: HttpContext) {
+    return await this.chatService.getAllSessions()
+  }
+
+  async show({ params, response }: HttpContext) {
+    const sessionId = parseInt(params.id)
+    const session = await this.chatService.getSession(sessionId)
+
+    if (!session) {
+      return response.status(404).json({ error: 'Session not found' })
+    }
+
+    return session
+  }
+
+  async store({ request, response }: HttpContext) {
+    try {
+      const data = await request.validateUsing(createSessionSchema)
+      const session = await this.chatService.createSession(data.title, data.model)
+      return response.status(201).json(session)
+    } catch (error) {
+      logger.error({ err: error }, '[ChatsController] Failed to create session')
+      return response.status(500).json({
+        error: 'Failed to create session',
+      })
+    }
+  }
+
+  async suggestions({ response }: HttpContext) {
+    try {
+      const suggestions = await this.chatService.getChatSuggestions()
+      return response.status(200).json({ suggestions })
+    } catch (error) {
+      logger.error({ err: error }, '[ChatsController] Failed to get suggestions')
+      return response.status(500).json({
+        error: 'Failed to get suggestions',
+      })
+    }
+  }
+
+  async update({ params, request, response }: HttpContext) {
+    try {
+      const sessionId = parseInt(params.id)
+      const data = await request.validateUsing(updateSessionSchema)
+      const session = await this.chatService.updateSession(sessionId, data)
+      return session
+    } catch (error) {
+      logger.error({ err: error }, '[ChatsController] Failed to update session')
+      return response.status(500).json({
+        error: 'Failed to update session',
+      })
+    }
+  }
+
+  async destroy({ params, response }: HttpContext) {
+    try {
+      const sessionId = parseInt(params.id)
+      await this.chatService.deleteSession(sessionId)
+      return response.status(204)
+    } catch (error) {
+      logger.error({ err: error }, '[ChatsController] Failed to delete session')
+      return response.status(500).json({
+        error: 'Failed to delete session',
+      })
+    }
+  }
+
+  async addMessage({ params, request, response }: HttpContext) {
+    try {
+      const sessionId = parseInt(params.id)
+      const data = await request.validateUsing(addMessageSchema)
+      const message = await this.chatService.addMessage(sessionId, data.role, data.content)
+      return response.status(201).json(message)
+    } catch (error) {
+      logger.error({ err: error }, '[ChatsController] Failed to add message')
+      return response.status(500).json({
+        error: 'Failed to add message',
+      })
+    }
+  }
+
+  async destroyAll({ response }: HttpContext) {
+    try {
+      const result = await this.chatService.deleteAllSessions()
+      return response.status(200).json(result)
+    } catch (error) {
+      logger.error({ err: error }, '[ChatsController] Failed to delete all sessions')
+      return response.status(500).json({
+        error: 'Failed to delete all sessions',
+      })
+    }
+  }
+}

admin/app/controllers/collection_updates_controller.ts

@@ -0,0 +1,30 @@
+import { CollectionUpdateService } from '#services/collection_update_service'
+import {
+  assertNotPrivateUrl,
+  applyContentUpdateValidator,
+  applyAllContentUpdatesValidator,
+} from '#validators/common'
+import type { HttpContext } from '@adonisjs/core/http'
+
+export default class CollectionUpdatesController {
+  async checkForUpdates({}: HttpContext) {
+    const service = new CollectionUpdateService()
+    return await service.checkForUpdates()
+  }
+
+  async applyUpdate({ request }: HttpContext) {
+    const update = await request.validateUsing(applyContentUpdateValidator)
+    assertNotPrivateUrl(update.download_url)
+    const service = new CollectionUpdateService()
+    return await service.applyUpdate(update)
+  }
+
+  async applyAllUpdates({ request }: HttpContext) {
+    const { updates } = await request.validateUsing(applyAllContentUpdatesValidator)
+    for (const update of updates) {
+      assertNotPrivateUrl(update.download_url)
+    }
+    const service = new CollectionUpdateService()
+    return await service.applyAllUpdates(updates)
+  }
+}

admin/app/controllers/downloads_controller.ts

@@ -15,4 +15,13 @@ export default class DownloadsController {
     const payload = await request.validateUsing(downloadJobsByFiletypeSchema)
     return this.downloadService.listDownloadJobs(payload.params.filetype)
   }
+
+  async removeJob({ params }: HttpContext) {
+    await this.downloadService.removeFailedJob(params.jobId)
+    return { success: true }
+  }
+
+  async cancelJob({ params }: HttpContext) {
+    return this.downloadService.cancelJob(params.jobId)
+  }
 }

admin/app/controllers/easy_setup_controller.ts

@@ -1,5 +1,7 @@
 import { SystemService } from '#services/system_service'
 import { ZimService } from '#services/zim_service'
+import { CollectionManifestService } from '#services/collection_manifest_service'
+import KVStore from '#models/kv_store'
 import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
 
@@ -11,10 +13,14 @@ export default class EasySetupController {
   ) {}
 
   async index({ inertia }: HttpContext) {
-    const services = await this.systemService.getServices({ installedOnly: false })
+    const [services, remoteOllamaUrl] = await Promise.all([
+      this.systemService.getServices({ installedOnly: false }),
+      KVStore.getValue('ai.remoteOllamaUrl'),
+    ])
     return inertia.render('easy-setup/index', {
       system: {
         services: services,
+        remoteOllamaUrl: remoteOllamaUrl ?? '',
       },
     })
   }
@@ -26,4 +32,22 @@ export default class EasySetupController {
   async listCuratedCategories({}: HttpContext) {
     return await this.zimService.listCuratedCategories()
   }
+
+  async refreshManifests({}: HttpContext) {
+    const manifestService = new CollectionManifestService()
+    const [zimChanged, mapsChanged, wikiChanged] = await Promise.all([
+      manifestService.fetchAndCacheSpec('zim_categories'),
+      manifestService.fetchAndCacheSpec('maps'),
+      manifestService.fetchAndCacheSpec('wikipedia'),
+    ])
+
+    return {
+      success: true,
+      changed: {
+        zim_categories: zimChanged,
+        maps: mapsChanged,
+        wikipedia: wikiChanged,
+      },
+    }
+  }
 }

admin/app/controllers/maps_controller.ts

@@ -1,19 +1,24 @@
 import { MapService } from '#services/map_service'
+import MapMarker from '#models/map_marker'
 import {
+  assertNotPrivateUrl,
   downloadCollectionValidator,
   filenameParamValidator,
+  mapExtractPreflightValidator,
+  mapExtractValidator,
   remoteDownloadValidator,
   remoteDownloadValidatorOptional,
 } from '#validators/common'
 import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
+import vine from '@vinejs/vine'
 
 @inject()
 export default class MapsController {
   constructor(private mapService: MapService) {}
 
   async index({ inertia }: HttpContext) {
-    const baseAssetsCheck = await this.mapService.checkBaseAssetsExist()
+    const baseAssetsCheck = await this.mapService.ensureBaseAssets()
     const regionFiles = await this.mapService.listRegions()
     return inertia.render('maps', {
       maps: {
@@ -23,19 +28,16 @@ export default class MapsController {
     })
   }
 
-  async checkBaseAssets({}: HttpContext) {
-    const exists = await this.mapService.checkBaseAssetsExist()
-    return { exists }
-  }
-
   async downloadBaseAssets({ request }: HttpContext) {
     const payload = await request.validateUsing(remoteDownloadValidatorOptional)
+    if (payload.url) assertNotPrivateUrl(payload.url)
     await this.mapService.downloadBaseAssets(payload.url)
     return { success: true }
   }
 
   async downloadRemote({ request }: HttpContext) {
     const payload = await request.validateUsing(remoteDownloadValidator)
+    assertNotPrivateUrl(payload.url)
     const filename = await this.mapService.downloadRemote(payload.url)
     return {
       message: 'Download started successfully',
@@ -57,6 +59,7 @@ export default class MapsController {
   // For providing a "preflight" check in the UI before actually starting a background download
   async downloadRemotePreflight({ request }: HttpContext) {
     const payload = await request.validateUsing(remoteDownloadValidator)
+    assertNotPrivateUrl(payload.url)
     const info = await this.mapService.downloadRemotePreflight(payload.url)
     return info
   }
@@ -74,8 +77,57 @@ export default class MapsController {
     return await this.mapService.listRegions()
   }
 
-  async styles({ response }: HttpContext) {
-    const styles = await this.mapService.generateStylesJSON()
+  async globalMapInfo({}: HttpContext) {
+    return await this.mapService.getGlobalMapInfo()
+  }
+
+  async downloadGlobalMap({}: HttpContext) {
+    const result = await this.mapService.downloadGlobalMap()
+    return {
+      message: 'Download started successfully',
+      ...result,
+    }
+  }
+
+  async listCountries({}: HttpContext) {
+    return { countries: await this.mapService.listCountries() }
+  }
+
+  async listCountryGroups({}: HttpContext) {
+    return { groups: await this.mapService.listCountryGroups() }
+  }
+
+  async extractPreflight({ request }: HttpContext) {
+    const payload = await request.validateUsing(mapExtractPreflightValidator)
+    return await this.mapService.extractPreflight(payload)
+  }
+
+  async extractRegion({ request }: HttpContext) {
+    const payload = await request.validateUsing(mapExtractValidator)
+    const result = await this.mapService.extractRegion(payload)
+    return {
+      message: 'Extract started successfully',
+      ...result,
+    }
+  }
+
+  async styles({ request, response }: HttpContext) {
+    // Automatically ensure base assets are present before generating styles
+    const baseAssetsExist = await this.mapService.ensureBaseAssets()
+    if (!baseAssetsExist) {
+      return response.status(500).send({
+        message:
+          'Base map assets are missing and could not be downloaded. Please check your connection and try again.',
+      })
+    }
+
+    const forwardedProto = request.headers()['x-forwarded-proto'];
+
+    const protocol: string = forwardedProto
+      ? (typeof forwardedProto === 'string' ? forwardedProto : request.protocol())
+      : request.protocol();
+
+    const styles = await this.mapService.generateStylesJSON(request.host(), protocol)
     return response.json(styles)
   }
 
@@ -97,4 +149,72 @@ export default class MapsController {
       message: 'Map file deleted successfully',
     }
   }
+
+  // --- Map Markers ---
+
+  async listMarkers({}: HttpContext) {
+    return await MapMarker.query().orderBy('created_at', 'asc')
+  }
+
+  async createMarker({ request }: HttpContext) {
+    const payload = await request.validateUsing(
+      vine.compile(
+        vine.object({
+          name: vine.string().trim().minLength(1).maxLength(255),
+          longitude: vine.number().min(-180).max(180),
+          latitude: vine.number().min(-90).max(90),
+          color: vine.string().trim().maxLength(20).optional(),
+          notes: vine.string().trim().nullable().optional(),
+          marker_type: vine.string().trim().maxLength(20).optional(),
+        })
+      )
+    )
+    const marker = await MapMarker.create({
+      name: payload.name,
+      longitude: payload.longitude,
+      latitude: payload.latitude,
+      color: payload.color ?? 'orange',
+      notes: payload.notes ?? null,
+      marker_type: payload.marker_type ?? 'pin',
+    })
+    return marker
+  }
+
+  async updateMarker({ request, response }: HttpContext) {
+    const { id } = request.params()
+    const marker = await MapMarker.find(id)
+    if (!marker) {
+      return response.status(404).send({ message: 'Marker not found' })
+    }
+    const payload = await request.validateUsing(
+      vine.compile(
+        vine.object({
+          name: vine.string().trim().minLength(1).maxLength(255).optional(),
+          color: vine.string().trim().maxLength(20).optional(),
+          longitude: vine.number().min(-180).max(180).optional(),
+          latitude: vine.number().min(-90).max(90).optional(),
+          notes: vine.string().trim().nullable().optional(),
+          marker_type: vine.string().trim().maxLength(20).optional(),
+        })
+      )
+    )
+    if (payload.name !== undefined) marker.name = payload.name
+    if (payload.color !== undefined) marker.color = payload.color
+    if (payload.longitude !== undefined) marker.longitude = payload.longitude
+    if (payload.latitude !== undefined) marker.latitude = payload.latitude
+    if (payload.notes !== undefined) marker.notes = payload.notes
+    if (payload.marker_type !== undefined) marker.marker_type = payload.marker_type
+    await marker.save()
+    return marker
+  }
+
+  async deleteMarker({ request, response }: HttpContext) {
+    const { id } = request.params()
+    const marker = await MapMarker.find(id)
+    if (!marker) {
+      return response.status(404).send({ message: 'Marker not found' })
+    }
+    await marker.delete()
+    return { message: 'Marker deleted' }
+  }
 }

admin/app/controllers/ollama_controller.ts

@@ -0,0 +1,458 @@
+import { ChatService } from '#services/chat_service'
+import { DockerService } from '#services/docker_service'
+import { OllamaService } from '#services/ollama_service'
+import { RagService } from '#services/rag_service'
+import Service from '#models/service'
+import KVStore from '#models/kv_store'
+import { modelNameSchema } from '#validators/download'
+import { chatSchema, getAvailableModelsSchema, unloadChatModelsSchema } from '#validators/ollama'
+import { assertNotCloudMetadataUrl } from '#validators/common'
+import { inject } from '@adonisjs/core'
+import type { HttpContext } from '@adonisjs/core/http'
+import { RAG_CONTEXT_LIMITS, SYSTEM_PROMPTS } from '../../constants/ollama.js'
+import { SERVICE_NAMES } from '../../constants/service_names.js'
+import logger from '@adonisjs/core/services/logger'
+type Message = { role: 'system' | 'user' | 'assistant'; content: string }
+
+@inject()
+export default class OllamaController {
+  constructor(
+    private chatService: ChatService,
+    private dockerService: DockerService,
+    private ollamaService: OllamaService,
+    private ragService: RagService
+  ) { }
+
+  async availableModels({ request }: HttpContext) {
+    const reqData = await request.validateUsing(getAvailableModelsSchema)
+    return await this.ollamaService.getAvailableModels({
+      sort: reqData.sort,
+      recommendedOnly: reqData.recommendedOnly,
+      query: reqData.query || null,
+      limit: reqData.limit || 15,
+      force: reqData.force,
+    })
+  }
+
+  /**
+   * Send Ollama `keep_alive: 0` hints to every currently-loaded chat model
+   * except the embedding model and (optionally) a target model to preserve.
+   * Used by the chat UI to enforce the "one chat model at a time" invariant
+   * on model-switch, session-switch, and page-load. Best-effort: a failure
+   * here should not block the calling flow.
+   */
+  async unloadChatModels({ request, response }: HttpContext) {
+    const { targetModel } = await request.validateUsing(unloadChatModelsSchema)
+    const unloaded = await this.ollamaService.unloadAllChatModelsExcept(targetModel ?? null)
+    return response.status(200).json({ unloaded })
+  }
+
+  async chat({ request, response }: HttpContext) {
+    const reqData = await request.validateUsing(chatSchema)
+
+    // Flush SSE headers immediately so the client connection is open while
+    // pre-processing (query rewriting, RAG lookup) runs in the background.
+    if (reqData.stream) {
+      response.response.setHeader('Content-Type', 'text/event-stream')
+      response.response.setHeader('Cache-Control', 'no-cache')
+      response.response.setHeader('Connection', 'keep-alive')
+      response.response.flushHeaders()
+    }
+
+    try {
+      // If there are no system messages in the chat inject system prompts
+      const hasSystemMessage = reqData.messages.some((msg) => msg.role === 'system')
+      if (!hasSystemMessage) {
+        const systemPrompt = {
+          role: 'system' as const,
+          content: SYSTEM_PROMPTS.default,
+        }
+        logger.debug('[OllamaController] Injecting system prompt')
+        reqData.messages.unshift(systemPrompt)
+      }
+
+      // Query rewriting for better RAG retrieval with manageable context
+      // Will return user's latest message if no rewriting is needed
+      const rewrittenQuery = await this.rewriteQueryWithContext(reqData.messages, reqData.model)
+
+      logger.debug(`[OllamaController] Rewritten query for RAG: "${rewrittenQuery}"`)
+      if (rewrittenQuery) {
+        const relevantDocs = await this.ragService.searchSimilarDocuments(
+          rewrittenQuery,
+          5, // Top 5 most relevant chunks
+          0.3 // Minimum similarity score of 0.3
+        )
+
+        logger.debug(`[RAG] Retrieved ${relevantDocs.length} relevant documents for query: "${rewrittenQuery}"`)
+
+        // If relevant context is found, inject as a system message with adaptive limits
+        if (relevantDocs.length > 0) {
+          // Determine context budget based on model size
+          const { maxResults, maxTokens } = this.getContextLimitsForModel(reqData.model)
+          let trimmedDocs = relevantDocs.slice(0, maxResults)
+
+          // Apply token cap if set (estimate ~3.5 chars per token)
+          // Always include the first (most relevant) result — the cap only gates subsequent results
+          if (maxTokens > 0) {
+            const charCap = maxTokens * 3.5
+            let totalChars = 0
+            trimmedDocs = trimmedDocs.filter((doc, idx) => {
+              totalChars += doc.text.length
+              return idx === 0 || totalChars <= charCap
+            })
+          }
+
+          logger.debug(
+            `[RAG] Injecting ${trimmedDocs.length}/${relevantDocs.length} results (model: ${reqData.model}, maxResults: ${maxResults}, maxTokens: ${maxTokens || 'unlimited'})`
+          )
+
+          // Label each context block with its source title when available (a neutral,
+          // honest provenance signal) but never the raw relevance score — nomic cosine
+          // scores for genuinely relevant passages sit ~0.4-0.6, and surfacing e.g.
+          // "42%" primes the model to distrust correct context. Scores stay in the logs
+          // above for debugging.
+          const contextText = trimmedDocs
+            .map((doc, idx) => {
+              const title = doc.metadata?.full_title || doc.metadata?.article_title
+              const label = title ? `[Context ${idx + 1} — ${title}]` : `[Context ${idx + 1}]`
+              return `${label}\n${doc.text}`
+            })
+            .join('\n\n')
+
+          const systemMessage = {
+            role: 'system' as const,
+            content: SYSTEM_PROMPTS.rag_context(contextText),
+          }
+
+          // Insert system message at the beginning (after any existing system messages)
+          const firstNonSystemIndex = reqData.messages.findIndex((msg) => msg.role !== 'system')
+          const insertIndex = firstNonSystemIndex === -1 ? 0 : firstNonSystemIndex
+          reqData.messages.splice(insertIndex, 0, systemMessage)
+        }
+      }
+
+      // If system messages are large (e.g. due to RAG context), request a context window big
+      // enough to fit them. Ollama respects num_ctx per-request; LM Studio ignores it gracefully.
+      const systemChars = reqData.messages
+        .filter((m) => m.role === 'system')
+        .reduce((sum, m) => sum + m.content.length, 0)
+      const estimatedSystemTokens = Math.ceil(systemChars / 3.5)
+      let numCtx: number | undefined
+      if (estimatedSystemTokens > 3000) {
+        const needed = estimatedSystemTokens + 2048 // leave room for conversation + response
+        numCtx = [8192, 16384, 32768, 65536].find((n) => n >= needed) ?? 65536
+        logger.debug(`[OllamaController] Large system prompt (~${estimatedSystemTokens} tokens), requesting num_ctx: ${numCtx}`)
+      }
+
+      // Check if the model supports "thinking" capability for enhanced response generation
+      // If gpt-oss model, it requires a text param for "think" https://docs.ollama.com/api/chat
+      const thinkingCapability = await this.ollamaService.checkModelHasThinking(reqData.model)
+      const think: boolean | 'medium' = thinkingCapability ? (reqData.model.startsWith('gpt-oss') ? 'medium' : true) : false
+
+      // Separate sessionId from the Ollama request payload — Ollama rejects unknown fields
+      const { sessionId, ...ollamaRequest } = reqData
+
+      // Save user message to DB before streaming if sessionId provided
+      let userContent: string | null = null
+      if (sessionId) {
+        const lastUserMsg = [...reqData.messages].reverse().find((m) => m.role === 'user')
+        if (lastUserMsg) {
+          userContent = lastUserMsg.content
+          await this.chatService.addMessage(sessionId, 'user', userContent)
+        }
+      }
+
+      if (reqData.stream) {
+        logger.debug(`[OllamaController] Initiating streaming response for model: "${reqData.model}" with think: ${think}`)
+        // Headers already flushed above
+        const stream = await this.ollamaService.chatStream({ ...ollamaRequest, think, numCtx })
+        let fullContent = ''
+        for await (const chunk of stream) {
+          if (chunk.message?.content) {
+            fullContent += chunk.message.content
+          }
+          response.response.write(`data: ${JSON.stringify(chunk)}\n\n`)
+        }
+        response.response.end()
+
+        // Save assistant message and optionally generate title
+        if (sessionId && fullContent) {
+          await this.chatService.addMessage(sessionId, 'assistant', fullContent)
+          const messageCount = await this.chatService.getMessageCount(sessionId)
+          if (messageCount <= 2 && userContent) {
+            this.chatService.generateTitle(sessionId, userContent, fullContent, reqData.model).catch((err) => {
+              logger.error(`[OllamaController] Title generation failed: ${err instanceof Error ? err.message : err}`)
+            })
+          }
+        }
+        return
+      }
+
+      // Non-streaming (legacy) path
+      const result = await this.ollamaService.chat({ ...ollamaRequest, think, numCtx })
+
+      if (sessionId && result?.message?.content) {
+        await this.chatService.addMessage(sessionId, 'assistant', result.message.content)
+        const messageCount = await this.chatService.getMessageCount(sessionId)
+        if (messageCount <= 2 && userContent) {
+          this.chatService.generateTitle(sessionId, userContent, result.message.content, reqData.model).catch((err) => {
+            logger.error(`[OllamaController] Title generation failed: ${err instanceof Error ? err.message : err}`)
+          })
+        }
+      }
+
+      return result
+    } catch (error) {
+      if (reqData.stream) {
+        response.response.write(`data: ${JSON.stringify({ error: true })}\n\n`)
+        response.response.end()
+        return
+      }
+      throw error
+    }
+  }
+
+  async remoteStatus() {
+    const remoteUrl = await KVStore.getValue('ai.remoteOllamaUrl')
+    if (!remoteUrl) {
+      return { configured: false, connected: false }
+    }
+    try {
+      const testResponse = await fetch(`${remoteUrl.replace(/\/$/, '')}/v1/models`, {
+        signal: AbortSignal.timeout(3000),
+      })
+      return { configured: true, connected: testResponse.ok }
+    } catch {
+      return { configured: true, connected: false }
+    }
+  }
+
+  async configureRemote({ request, response }: HttpContext) {
+    const remoteUrl: string | null = request.input('remoteUrl', null)
+
+    const ollamaService = await Service.query().where('service_name', SERVICE_NAMES.OLLAMA).first()
+    if (!ollamaService) {
+      return response.status(404).send({ success: false, message: 'Ollama service record not found.' })
+    }
+
+    // Clear path: null or empty URL removes remote config. If a local nomad_ollama container
+    // still exists (user had previously installed AI Assistant locally), restart it and keep
+    // the service marked installed. Otherwise fall back to uninstalled.
+    if (!remoteUrl || remoteUrl.trim() === '') {
+      await KVStore.clearValue('ai.remoteOllamaUrl')
+      const hasLocalContainer = await this._startLocalOllamaContainerIfExists()
+      ollamaService.installed = hasLocalContainer
+      ollamaService.installation_status = 'idle'
+      await ollamaService.save()
+      return {
+        success: true,
+        message: hasLocalContainer
+          ? 'Remote Ollama cleared. Local Ollama container restored.'
+          : 'Remote Ollama configuration cleared.',
+      }
+    }
+
+    try {
+      assertNotCloudMetadataUrl(remoteUrl)
+    } catch (err) {
+      return response.status(400).send({
+        success: false,
+        message: err instanceof Error ? err.message : 'Invalid URL.',
+      })
+    }
+
+    // Test connectivity via OpenAI-compatible /v1/models endpoint (works with Ollama, LM Studio, llama.cpp, etc.)
+    try {
+      const testResponse = await fetch(`${remoteUrl.replace(/\/$/, '')}/v1/models`, {
+        signal: AbortSignal.timeout(5000),
+      })
+      if (!testResponse.ok) {
+        return response.status(400).send({
+          success: false,
+          message: `Could not connect to ${remoteUrl} (HTTP ${testResponse.status}). Make sure the server is running and accessible. For Ollama, start it with OLLAMA_HOST=0.0.0.0.`,
+        })
+      }
+    } catch (error) {
+      return response.status(400).send({
+        success: false,
+        message: `Could not connect to ${remoteUrl}. Make sure the server is running and reachable. For Ollama, start it with OLLAMA_HOST=0.0.0.0.`,
+      })
+    }
+
+    // Save remote URL and mark service as installed
+    await KVStore.setValue('ai.remoteOllamaUrl', remoteUrl.trim())
+    ollamaService.installed = true
+    ollamaService.installation_status = 'idle'
+    await ollamaService.save()
+
+    // Stop the local nomad_ollama container (if running) so it doesn't compete with the
+    // remote host for GPU / port 11434. Preserves the container and its models volume.
+    await this._stopLocalOllamaContainer()
+
+    // Install Qdrant if not already installed (fire-and-forget)
+    const qdrantService = await Service.query().where('service_name', SERVICE_NAMES.QDRANT).first()
+    if (qdrantService && !qdrantService.installed) {
+      this.dockerService.createContainerPreflight(SERVICE_NAMES.QDRANT).catch((error) => {
+        logger.error('[OllamaController] Failed to start Qdrant preflight:', error)
+      })
+    }
+
+    // Mirror post-install side effects: disable suggestions, trigger docs discovery
+    await KVStore.setValue('chat.suggestionsEnabled', false)
+    this.ragService.discoverNomadDocs().catch((error) => {
+      logger.error('[OllamaController] Failed to discover Nomad docs:', error)
+    })
+
+    return { success: true, message: 'Remote Ollama configured.' }
+  }
+
+  private async _stopLocalOllamaContainer(): Promise<void> {
+    try {
+      const containers = await this.dockerService.docker.listContainers({ all: true })
+      const ollamaContainer = containers.find((c) =>
+        c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`)
+      )
+      if (!ollamaContainer || ollamaContainer.State !== 'running') {
+        return
+      }
+      await this.dockerService.docker.getContainer(ollamaContainer.Id).stop()
+      this.dockerService.invalidateServicesStatusCache()
+      logger.info('[OllamaController] Stopped local nomad_ollama (remote Ollama configured)')
+    } catch (error: any) {
+      logger.error(
+        { err: error },
+        '[OllamaController] Failed to stop local nomad_ollama; remote Ollama is still active'
+      )
+    }
+  }
+
+  private async _startLocalOllamaContainerIfExists(): Promise<boolean> {
+    try {
+      const containers = await this.dockerService.docker.listContainers({ all: true })
+      const ollamaContainer = containers.find((c) =>
+        c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`)
+      )
+      if (!ollamaContainer) {
+        return false
+      }
+      if (ollamaContainer.State !== 'running') {
+        await this.dockerService.docker.getContainer(ollamaContainer.Id).start()
+        this.dockerService.invalidateServicesStatusCache()
+        logger.info('[OllamaController] Started local nomad_ollama (remote Ollama cleared)')
+      }
+      return true
+    } catch (error: any) {
+      logger.error(
+        { err: error },
+        '[OllamaController] Failed to start local nomad_ollama on remote clear'
+      )
+      return false
+    }
+  }
+
+  async deleteModel({ request }: HttpContext) {
+    const reqData = await request.validateUsing(modelNameSchema)
+    await this.ollamaService.deleteModel(reqData.model)
+    return {
+      success: true,
+      message: `Model deleted: ${reqData.model}`,
+    }
+  }
+
+  async dispatchModelDownload({ request }: HttpContext) {
+    const reqData = await request.validateUsing(modelNameSchema)
+    await this.ollamaService.dispatchModelDownload(reqData.model)
+    return {
+      success: true,
+      message: `Download job dispatched for model: ${reqData.model}`,
+    }
+  }
+
+  async installedModels({ }: HttpContext) {
+    return await this.ollamaService.getModels()
+  }
+
+  /**
+   * Determines RAG context limits based on model size extracted from the model name.
+   * Parses size indicators like "1b", "3b", "8b", "70b" from model names/tags.
+   */
+  private getContextLimitsForModel(modelName: string): { maxResults: number; maxTokens: number } {
+    // Extract parameter count from model name (e.g., "llama3.2:3b", "qwen2.5:1.5b", "gemma:7b")
+    const sizeMatch = modelName.match(/(\d+\.?\d*)[bB]/)
+    const paramBillions = sizeMatch ? parseFloat(sizeMatch[1]) : 8 // default to 8B if unknown
+
+    for (const tier of RAG_CONTEXT_LIMITS) {
+      if (paramBillions <= tier.maxParams) {
+        return { maxResults: tier.maxResults, maxTokens: tier.maxTokens }
+      }
+    }
+
+    // Fallback: no limits
+    return { maxResults: 5, maxTokens: 0 }
+  }
+
+  private async rewriteQueryWithContext(
+    messages: Message[],
+    model: string
+  ): Promise<string | null> {
+    const lastUserMessage = [...messages].reverse().find(msg => msg.role === 'user')
+
+    try {
+      // Skip the entire RAG pipeline if there are no documents to search
+      const hasDocuments = await this.ragService.hasDocuments()
+      if (!hasDocuments) {
+        return null
+      }
+
+      // Get recent conversation history (last 6 messages for 3 turns)
+      const recentMessages = messages.slice(-6)
+
+      // Skip rewriting on the very first turn — with only one user message
+      // there is no prior context to fold in, so the rewrite would just echo
+      // the message back at the cost of an extra LLM round-trip. From the
+      // first follow-up onward we need the rewrite so the RAG query carries
+      // entities and topics from earlier turns ("the bars" → "Hershey's bars
+      // chocolate poisoning dog"); without it, embeddings match nothing and
+      // the assistant loses the thread.
+      const userMessages = recentMessages.filter(msg => msg.role === 'user')
+      if (userMessages.length < 2) {
+        return lastUserMessage?.content || null
+      }
+
+      const conversationContext = recentMessages
+        .map(msg => {
+          const role = msg.role === 'user' ? 'User' : 'Assistant'
+          // Truncate assistant messages to first 200 chars to keep context manageable
+          const content = msg.role === 'assistant'
+            ? msg.content.slice(0, 200) + (msg.content.length > 200 ? '...' : '')
+            : msg.content
+          return `${role}: "${content}"`
+        })
+        .join('\n')
+
+      const response = await this.ollamaService.chat({
+        model,
+        messages: [
+          {
+            role: 'system',
+            content: SYSTEM_PROMPTS.query_rewrite,
+          },
+          {
+            role: 'user',
+            content: `Conversation:\n${conversationContext}\n\nRewritten Query:`,
+          },
+        ],
+      })
+
+      const rewrittenQuery = response.message.content.trim()
+      logger.info(`[RAG] Query rewritten: "${rewrittenQuery}"`)
+      return rewrittenQuery
+    } catch (error) {
+      logger.error(
+        `[RAG] Query rewriting failed: ${error instanceof Error ? error.message : error}`
+      )
+      // Fallback to last user message if rewriting fails
+      return lastUserMessage?.content || null
+    }
+  }
+}

admin/app/controllers/openwebui_controller.ts

@@ -1,40 +0,0 @@
-import { OpenWebUIService } from '#services/openwebui_service'
-import { modelNameSchema } from '#validators/download'
-import { getAvailableModelsSchema } from '#validators/openwebui'
-import { inject } from '@adonisjs/core'
-import type { HttpContext } from '@adonisjs/core/http'
-
-@inject()
-export default class OpenWebUIController {
-  constructor(private openWebUIService: OpenWebUIService) {}
-
-  async models({ request }: HttpContext) {
-    const reqData = await request.validateUsing(getAvailableModelsSchema)
-    return await this.openWebUIService.getAvailableModels({
-      sort: reqData.sort,
-      recommendedOnly: reqData.recommendedOnly,
-    })
-  }
-
-  async installedModels({}: HttpContext) {
-    return await this.openWebUIService.getInstalledModels()
-  }
-
-  async deleteModel({ request }: HttpContext) {
-    const reqData = await request.validateUsing(modelNameSchema)
-    await this.openWebUIService.deleteModel(reqData.model)
-    return {
-      success: true,
-      message: `Model deleted: ${reqData.model}`,
-    }
-  }
-
-  async dispatchModelDownload({ request }: HttpContext) {
-    const reqData = await request.validateUsing(modelNameSchema)
-    await this.openWebUIService.dispatchModelDownload(reqData.model)
-    return {
-      success: true,
-      message: `Download job dispatched for model: ${reqData.model}`,
-    }
-  }
-}

admin/app/controllers/rag_controller.ts

@@ -0,0 +1,192 @@
+import { RagService } from '#services/rag_service'
+import { EmbedFileJob } from '#jobs/embed_file_job'
+import KbRatioRegistry from '#models/kb_ratio_registry'
+import { inject } from '@adonisjs/core'
+import type { HttpContext } from '@adonisjs/core/http'
+import app from '@adonisjs/core/services/app'
+import { randomBytes } from 'node:crypto'
+import { sanitizeFilename } from '../utils/fs.js'
+import { basename } from 'node:path'
+import { deleteFileSchema, embedFileSchema, estimateBatchSchema, fileSourceSchema, getJobStatusSchema } from '#validators/rag'
+import logger from '@adonisjs/core/services/logger'
+
+@inject()
+export default class RagController {
+  constructor(private ragService: RagService) { }
+
+  public async upload({ request, response }: HttpContext) {
+    const uploadedFile = request.file('file')
+    if (!uploadedFile) {
+      return response.status(400).json({ error: 'No file uploaded' })
+    }
+
+    const randomSuffix = randomBytes(6).toString('hex')
+    const sanitizedName = sanitizeFilename(uploadedFile.clientName)
+
+    const fileName = `${sanitizedName}-${randomSuffix}.${uploadedFile.extname || 'txt'}`
+    const fullPath = app.makePath(RagService.UPLOADS_STORAGE_PATH, fileName)
+
+    await uploadedFile.move(app.makePath(RagService.UPLOADS_STORAGE_PATH), {
+      name: fileName,
+    })
+
+    // Dispatch background job for embedding
+    const result = await EmbedFileJob.dispatch({
+      filePath: fullPath,
+      fileName,
+    })
+
+    return response.status(202).json({
+      message: result.message,
+      jobId: result.jobId,
+      fileName,
+      filePath: `/${RagService.UPLOADS_STORAGE_PATH}/${fileName}`,
+      alreadyProcessing: !result.created,
+    })
+  }
+
+  public async getActiveJobs({ response }: HttpContext) {
+    const jobs = await EmbedFileJob.listActiveJobs()
+    return response.status(200).json(jobs)
+  }
+
+  public async getJobStatus({ request, response }: HttpContext) {
+    const reqData = await request.validateUsing(getJobStatusSchema)
+
+    const fullPath = app.makePath(RagService.UPLOADS_STORAGE_PATH, reqData.filePath)
+    const status = await EmbedFileJob.getStatus(fullPath)
+
+    if (!status.exists) {
+      return response.status(404).json({ error: 'Job not found for this file' })
+    }
+
+    return response.status(200).json(status)
+  }
+
+  public async getStoredFiles({ response }: HttpContext) {
+    const files = await this.ragService.getStoredFiles()
+    return response.status(200).json({ files })
+  }
+
+  public async getFileWarnings({ response }: HttpContext) {
+    const result = await this.ragService.computeFileWarnings()
+    return response.status(200).json(result)
+  }
+
+  public async deleteFile({ request, response }: HttpContext) {
+    const { source } = await request.validateUsing(deleteFileSchema)
+    const result = await this.ragService.deleteFileBySource(source)
+    if (!result.success) {
+      return response.status(500).json({ error: result.message })
+    }
+    return response.status(200).json({ message: result.message })
+  }
+
+  public async embedFile({ request, response }: HttpContext) {
+    const { source, force } = await request.validateUsing(embedFileSchema)
+    const result = await this.ragService.embedSingleFile(source, force ?? false)
+    if (!result.success) {
+      const status = {
+        not_found: 404,
+        inflight: 409,
+        delete_failed: 500,
+        dispatch_failed: 500,
+      }[result.code]
+      return response.status(status).json({ error: result.message, code: result.code })
+    }
+    return response.status(202).json({ message: result.message })
+  }
+
+  public async getFailedJobs({ response }: HttpContext) {
+    const jobs = await EmbedFileJob.listFailedJobs()
+    return response.status(200).json(jobs)
+  }
+
+  public async cleanupFailedJobs({ response }: HttpContext) {
+    const result = await EmbedFileJob.cleanupFailedJobs()
+    return response.status(200).json({
+      message: `Cleaned up ${result.cleaned} failed job${result.cleaned !== 1 ? 's' : ''}${result.filesDeleted > 0 ? `, deleted ${result.filesDeleted} file${result.filesDeleted !== 1 ? 's' : ''}` : ''}.`,
+      ...result,
+    })
+  }
+
+  public async cancelAllJobs({ response }: HttpContext) {
+    const result = await EmbedFileJob.cancelAllJobs()
+    return response.status(200).json({
+      message: `Cancelled ${result.cancelled} job${result.cancelled !== 1 ? 's' : ''}${result.filesDeleted > 0 ? `, deleted ${result.filesDeleted} file${result.filesDeleted !== 1 ? 's' : ''}` : ''}.`,
+      ...result,
+    })
+  }
+
+  public async policyPromptState({ response }: HttpContext) {
+    const result = await this.ragService.getPolicyPromptState()
+    return response.status(200).json(result)
+  }
+
+  public async scanAndSync({ response }: HttpContext) {
+    try {
+      const syncResult = await this.ragService.scanAndSyncStorage()
+      return response.status(200).json(syncResult)
+    } catch (error) {
+      logger.error({ err: error }, '[RagController] Error scanning and syncing storage')
+      return response.status(500).json({ error: 'Error scanning and syncing storage' })
+    }
+  }
+
+  public async reembedAll({ response }: HttpContext) {
+    try {
+      const result = await this.ragService.reembedAll()
+      return response.status(200).json(result)
+    } catch (error) {
+      logger.error({ err: error }, '[RagController] Error during re-embed all')
+      return response.status(500).json({ error: 'Error during re-embed all' })
+    }
+  }
+
+  public async resetAndRebuild({ response }: HttpContext) {
+    try {
+      const result = await this.ragService.resetAndRebuild()
+      return response.status(200).json(result)
+    } catch (error) {
+      logger.error({ err: error }, '[RagController] Error during reset and rebuild')
+      return response.status(500).json({ error: 'Error during reset and rebuild' })
+    }
+  }
+
+  public async health({ response }: HttpContext) {
+    const result = await this.ragService.checkQdrantHealth()
+    return response.status(200).json(result)
+  }
+
+  public async estimateBatch({ request, response }: HttpContext) {
+    const { files } = await request.validateUsing(estimateBatchSchema)
+    // The registry matches on basename prefixes; if a caller passes a full path
+    // (e.g. /app/storage/zim/wikipedia_en_simple_…), strip directories first so
+    // patterns like `wikipedia_en_simple_` still match.
+    const normalized = files.map((f) => ({
+      filename: basename(f.filename),
+      sizeBytes: f.sizeBytes,
+    }))
+    const result = await KbRatioRegistry.estimateBatch(normalized)
+    return response.status(200).json(result)
+  }
+
+  public async getFileContent({ request, response }: HttpContext) {
+    const { source } = await request.validateUsing(fileSourceSchema)
+    const result = await this.ragService.readFileContent(source)
+    if (!result) {
+      return response.status(404).json({ error: 'File not found or not viewable' })
+    }
+    return response.status(200).json(result)
+  }
+
+  public async downloadFile({ request, response }: HttpContext) {
+    const { source } = await request.validateUsing(fileSourceSchema)
+    const filePath = await this.ragService.resolveDownloadPath(source)
+    if (!filePath) {
+      return response.status(404).json({ error: 'File not found' })
+    }
+    const fileName = filePath.split(/[/\\]/).at(-1) ?? 'download'
+    return response.attachment(filePath, fileName)
+  }
+}

admin/app/controllers/settings_controller.ts

@@ -1,77 +1,142 @@
-import { MapService } from '#services/map_service';
-import { OpenWebUIService } from '#services/openwebui_service';
-import { SystemService } from '#services/system_service';
-import { inject } from '@adonisjs/core';
+import KVStore from '#models/kv_store'
+import { BenchmarkService } from '#services/benchmark_service'
+import { MapService } from '#services/map_service'
+import { OllamaService } from '#services/ollama_service'
+import { SystemService } from '#services/system_service'
+import { getSettingSchema, updateSettingSchema, validateSettingValue } from '#validators/settings'
+import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
+import env from '#start/env'
 
 @inject()
 export default class SettingsController {
-    constructor(
-        private systemService: SystemService,
-        private mapService: MapService,
-        private openWebUIService: OpenWebUIService
-    ) { }
+  constructor(
+    private systemService: SystemService,
+    private mapService: MapService,
+    private benchmarkService: BenchmarkService,
+    private ollamaService: OllamaService
+  ) {}
 
-    async system({ inertia }: HttpContext) {
-        const systemInfo = await this.systemService.getSystemInfo();
-        return inertia.render('settings/system', {
-            system: {
-                info: systemInfo
-            }
-        });
-    }
+  async system({ inertia }: HttpContext) {
+    const systemInfo = await this.systemService.getSystemInfo()
+    return inertia.render('settings/system', {
+      system: {
+        info: systemInfo,
+      },
+    })
+  }
 
-    async apps({ inertia }: HttpContext) {
-        const services = await this.systemService.getServices({ installedOnly: false });
-        return inertia.render('settings/apps', {
-            system: {
-                services
-            }
-        });
-    }
-    
-    async legal({ inertia }: HttpContext) {
-        return inertia.render('settings/legal');
-    }
+  async apps({ inertia }: HttpContext) {
+    const services = await this.systemService.getServices({ installedOnly: false })
+    return inertia.render('settings/apps', {
+      system: {
+        services,
+      },
+    })
+  }
 
-    async maps({ inertia }: HttpContext) {
-        const baseAssetsCheck = await this.mapService.checkBaseAssetsExist();
-        const regionFiles = await this.mapService.listRegions();
-        return inertia.render('settings/maps', {
-            maps: {
-                baseAssetsExist: baseAssetsCheck,
-                regionFiles: regionFiles.files
-            }
-        });
-    }
+  async legal({ inertia }: HttpContext) {
+    return inertia.render('settings/legal')
+  }
 
-    async models({ inertia }: HttpContext) {
-        const availableModels = await this.openWebUIService.getAvailableModels();
-        const installedModels = await this.openWebUIService.getInstalledModels();
-        return inertia.render('settings/models', {
-            models: {
-                availableModels: availableModels || [],
-                installedModels: installedModels || []
-            }
-        });
-    }
+  async support({ inertia }: HttpContext) {
+    return inertia.render('settings/support')
+  }
 
-    async update({ inertia }: HttpContext) {
-        const updateInfo = await this.systemService.checkLatestVersion();
-        return inertia.render('settings/update', {
-            system: {
-                updateAvailable: updateInfo.updateAvailable,
-                latestVersion: updateInfo.latestVersion,
-                currentVersion: updateInfo.currentVersion
-            }
-        });
-    }
+  async maps({ inertia }: HttpContext) {
+    const baseAssetsCheck = await this.mapService.ensureBaseAssets()
+    const regionFiles = await this.mapService.listRegions()
+    return inertia.render('settings/maps', {
+      maps: {
+        baseAssetsExist: baseAssetsCheck,
+        regionFiles: regionFiles.files,
+      },
+    })
+  }
 
-    async zim({ inertia }: HttpContext) {
-        return inertia.render('settings/zim/index')
-    }
+  async models({ inertia }: HttpContext) {
+    const availableModels = await this.ollamaService.getAvailableModels({
+      sort: 'pulls',
+      recommendedOnly: false,
+      query: null,
+      limit: 15,
+    })
+    const installedModels = await this.ollamaService.getModels().catch(() => [])
+    const chatSuggestionsEnabled = await KVStore.getValue('chat.suggestionsEnabled')
+    const aiAssistantCustomName = await KVStore.getValue('ai.assistantCustomName')
+    const remoteOllamaUrl = await KVStore.getValue('ai.remoteOllamaUrl')
+    const ollamaFlashAttention = await KVStore.getValue('ai.ollamaFlashAttention')
+    return inertia.render('settings/models', {
+      models: {
+        availableModels: availableModels?.models || [],
+        installedModels: installedModels || [],
+        settings: {
+          chatSuggestionsEnabled: chatSuggestionsEnabled ?? false,
+          aiAssistantCustomName: aiAssistantCustomName ?? '',
+          remoteOllamaUrl: remoteOllamaUrl ?? '',
+          ollamaFlashAttention: ollamaFlashAttention ?? true,
+        },
+      },
+    })
+  }
 
-    async zimRemote({ inertia }: HttpContext) {
-        return inertia.render('settings/zim/remote-explorer');
+  async update({ inertia }: HttpContext) {
+    const updateInfo = await this.systemService.checkLatestVersion()
+    return inertia.render('settings/update', {
+      system: {
+        updateAvailable: updateInfo.updateAvailable,
+        latestVersion: updateInfo.latestVersion,
+        currentVersion: updateInfo.currentVersion,
+      },
+    })
+  }
+
+  async zim({ inertia }: HttpContext) {
+    return inertia.render('settings/zim/index')
+  }
+
+  async zimRemote({ inertia }: HttpContext) {
+    return inertia.render('settings/zim/remote-explorer')
+  }
+
+  async benchmark({ inertia }: HttpContext) {
+    const latestResult = await this.benchmarkService.getLatestResult()
+    const status = this.benchmarkService.getStatus()
+    return inertia.render('settings/benchmark', {
+      benchmark: {
+        latestResult,
+        status: status.status,
+        currentBenchmarkId: status.benchmarkId,
+      },
+    })
+  }
+
+  async advanced({ inertia }: HttpContext) {
+    // When the env var is set it always takes precedence over the stored value,
+    // so surface that to the UI to disable the field and explain the override.
+    const envOverride = Boolean(env.get('INTERNET_STATUS_TEST_URL')?.trim())
+    const internetStatusTestUrl = await KVStore.getValue('system.internetStatusTestUrl')
+    return inertia.render('settings/advanced', {
+      advanced: {
+        internetStatusTestUrl: internetStatusTestUrl ?? '',
+        internetStatusTestUrlEnvOverride: envOverride,
+      },
+    })
+  }
+
+  async getSetting({ request, response }: HttpContext) {
+    const { key } = await getSettingSchema.validate({ key: request.qs().key });
+    const value = await KVStore.getValue(key);
+    return response.status(200).send({ key, value });
+  }
+
+  async updateSetting({ request, response }: HttpContext) {
+    const reqData = await request.validateUsing(updateSettingSchema)
+    const valueError = validateSettingValue(reqData.key, reqData.value)
+    if (valueError) {
+      return response.status(422).send({ success: false, message: valueError })
     }
-}
+    await this.systemService.updateSetting(reqData.key, reqData.value)
+    return response.status(200).send({ success: true, message: 'Setting updated successfully' })
+  }
+}

admin/app/controllers/supply_depot_controller.ts

@@ -0,0 +1,13 @@
+import { SystemService } from '#services/system_service'
+import { inject } from '@adonisjs/core'
+import type { HttpContext } from '@adonisjs/core/http'
+
+@inject()
+export default class SupplyDepotController {
+  constructor(private systemService: SystemService) {}
+
+  async index({ inertia }: HttpContext) {
+    const services = await this.systemService.getServices({ installedOnly: false })
+    return inertia.render('supply-depot', { system: { services } })
+  }
+}

admin/app/controllers/system_controller.ts

@@ -1,16 +1,47 @@
 import { DockerService } from '#services/docker_service';
 import { SystemService } from '#services/system_service'
 import { SystemUpdateService } from '#services/system_update_service'
-import { affectServiceValidator, installServiceValidator } from '#validators/system';
+import { ContainerRegistryService } from '#services/container_registry_service'
+import { AutoUpdateService } from '#services/auto_update_service'
+import { AppAutoUpdateService } from '#services/app_auto_update_service'
+import { ContentAutoUpdateService } from '#services/content_auto_update_service'
+import { DownloadService } from '#services/download_service'
+import { QueueService } from '#services/queue_service'
+import { CheckServiceUpdatesJob } from '#jobs/check_service_updates_job'
+import {
+  affectServiceValidator,
+  checkLatestVersionValidator,
+  customAppValidator,
+  deleteCustomAppValidator,
+  installServiceValidator,
+  preflightCustomValidator,
+  preflightValidator,
+  serviceLogsValidator,
+  subscribeToReleaseNotesValidator,
+  uninstallServiceValidator,
+  updateCustomAppValidator,
+  updateServiceValidator,
+  setServiceAutoUpdateValidator,
+  setServiceCustomUrlValidator,
+  normalizeCustomUrl,
+} from '#validators/system'
+import {
+  DEFAULT_CPUS,
+  DEFAULT_MEMORY_MB,
+  evaluateCustomApp,
+} from '#services/custom_app_guard'
 import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
+import logger from '@adonisjs/core/services/logger'
+import Service from '#models/service'
 
 @inject()
 export default class SystemController {
     constructor(
         private systemService: SystemService,
         private dockerService: DockerService,
-        private systemUpdateService: SystemUpdateService
+        private systemUpdateService: SystemUpdateService,
+        private containerRegistryService: ContainerRegistryService
     ) { }
 
     async getInternetStatus({ }: HttpContext) {
@@ -32,7 +63,7 @@ export default class SystemController {
         if (result.success) {
             response.send({ success: true, message: result.message });
         } else {
-            response.status(400).send({ error: result.message });
+            response.status(400).send({ success: false, message: result.message });
         }
     }
 
@@ -46,8 +77,9 @@ export default class SystemController {
         response.send({ success: result.success, message: result.message });
     }
 
-    async checkLatestVersion({ }: HttpContext) {
-        return await this.systemService.checkLatestVersion();
+    async checkLatestVersion({ request }: HttpContext) {
+        const payload = await request.validateUsing(checkLatestVersionValidator)
+        return await this.systemService.checkLatestVersion(payload.force);
     }
 
     async forceReinstallService({ request, response }: HttpContext) {
@@ -102,4 +134,674 @@ export default class SystemController {
         const logs = this.systemUpdateService.getUpdateLogs();
         response.send({ logs });
     }
+
+    async getAutoUpdateStatus({ response }: HttpContext) {
+        // Construct inline reusing already-injected singletons + the QueueService
+        // singleton (its constructor is private to prevent Redis connection leaks,
+        // so we must not let the container new a fresh one).
+        const autoUpdateService = new AutoUpdateService(
+            this.dockerService,
+            new DownloadService(QueueService.getInstance()),
+            this.systemService,
+            this.systemUpdateService,
+            this.containerRegistryService
+        )
+
+        try {
+            const status = await autoUpdateService.getStatus()
+            response.send(status)
+        } catch (error) {
+            logger.error({ err: error }, '[SystemController] Failed to get auto-update status')
+            response.status(500).send({ error: 'Failed to retrieve auto-update status' })
+        }
+    }
+
+    async getAppAutoUpdateStatus({ response }: HttpContext) {
+        // Constructed inline reusing already-injected singletons + the QueueService
+        // singleton (its constructor is private to prevent Redis connection leaks),
+        // mirroring getAutoUpdateStatus. Apps need no SystemUpdateService (no sidecar).
+        const appAutoUpdateService = new AppAutoUpdateService(
+            this.dockerService,
+            new DownloadService(QueueService.getInstance()),
+            this.systemService,
+            this.containerRegistryService
+        )
+
+        try {
+            const status = await appAutoUpdateService.getStatus()
+            response.send(status)
+        } catch (error) {
+            logger.error({ err: error }, '[SystemController] Failed to get app auto-update status')
+            response.status(500).send({ error: 'Failed to retrieve app auto-update status' })
+        }
+    }
+
+    async getContentAutoUpdateStatus({ response }: HttpContext) {
+        // Mirrors getAppAutoUpdateStatus. Content auto-update needs only the
+        // DownloadService (for the active-download pre-flight); the catalog and
+        // collection-update services default-construct inside the service.
+        const contentAutoUpdateService = new ContentAutoUpdateService(
+            new DownloadService(QueueService.getInstance())
+        )
+
+        try {
+            const status = await contentAutoUpdateService.getStatus()
+            response.send(status)
+        } catch (error) {
+            logger.error({ err: error }, '[SystemController] Failed to get content auto-update status')
+            response.status(500).send({ error: 'Failed to retrieve content auto-update status' })
+        }
+    }
+
+    async setServiceAutoUpdate({ request, response }: HttpContext) {
+        const payload = await request.validateUsing(setServiceAutoUpdateValidator)
+        const service = await Service.query().where('service_name', payload.service_name).first()
+        if (!service) {
+            return response.status(404).send({ error: `Service ${payload.service_name} not found` })
+        }
+
+        service.auto_update_enabled = payload.enabled
+        // Re-enabling clears any prior self-disable so the app gets a fresh start.
+        if (payload.enabled) {
+            service.auto_update_consecutive_failures = 0
+            service.auto_update_disabled_reason = null
+        }
+        await service.save()
+
+        return response.send({ success: true, message: 'App auto-update preference updated' })
+    }
+
+
+    async subscribeToReleaseNotes({ request }: HttpContext) {
+        const reqData = await request.validateUsing(subscribeToReleaseNotesValidator);
+        return await this.systemService.subscribeToReleaseNotes(reqData.email);
+    }
+
+    async getDebugInfo({}: HttpContext) {
+        const debugInfo = await this.systemService.getDebugInfo()
+        return { debugInfo }
+    }
+
+    async checkServiceUpdates({ response }: HttpContext) {
+        await CheckServiceUpdatesJob.dispatch()
+        response.send({ success: true, message: 'Service update check dispatched' })
+    }
+
+    async getAvailableVersions({ params, response }: HttpContext) {
+        const serviceName = params.name
+        const service = await (await import('#models/service')).default
+            .query()
+            .where('service_name', serviceName)
+            .where('installed', true)
+            .first()
+
+        if (!service) {
+            return response.status(404).send({ error: `Service ${serviceName} not found or not installed` })
+        }
+
+        try {
+            const hostArch = await this.getHostArch()
+            const updates = await this.containerRegistryService.getAvailableUpdates(
+                service.container_image,
+                hostArch,
+                service.source_repo
+            )
+            response.send({ versions: updates })
+        } catch (error) {
+            logger.error({ err: error }, `[SystemController] Failed to fetch versions for ${serviceName}`)
+            response.status(500).send({ error: 'Failed to fetch available versions for this service.' })
+        }
+    }
+
+    async updateService({ request, response }: HttpContext) {
+        const payload = await request.validateUsing(updateServiceValidator)
+        const result = await this.dockerService.updateContainer(
+            payload.service_name,
+            payload.target_version
+        )
+
+        if (result.success) {
+            response.send({ success: true, message: result.message })
+        } else {
+            response.status(400).send({ error: result.message })
+        }
+    }
+
+    private async getHostArch(): Promise<string> {
+        try {
+            const info = await this.dockerService.docker.info()
+            const arch = info.Architecture || ''
+            const archMap: Record<string, string> = {
+                x86_64: 'amd64',
+                aarch64: 'arm64',
+                armv7l: 'arm',
+                amd64: 'amd64',
+                arm64: 'arm64',
+            }
+            return archMap[arch] || arch.toLowerCase()
+        } catch {
+            return 'amd64'
+        }
+    }
+
+    /**
+     * Pre-install preflight check: reports port conflicts and resource warnings for a service.
+     * Results are advisory — the UI shows warnings but allows the user to force-proceed.
+     */
+    async preflightCheck({ request, response }: HttpContext) {
+        const payload = await request.validateUsing(preflightValidator)
+
+        const service = await Service.query().where('service_name', payload.service_name).first()
+        if (!service) {
+            return response.status(404).send({ error: `Service ${payload.service_name} not found` })
+        }
+
+        // Extract host ports from container_config — the MySQL driver may return JSON columns
+        // as an already-parsed object rather than a string, so guard before calling JSON.parse.
+        const rawConfig = service.container_config
+        const config = rawConfig
+            ? typeof rawConfig === 'object'
+                ? rawConfig
+                : JSON.parse(rawConfig as string)
+            : null
+        const portBindings: Record<string, [{ HostPort: string }]> =
+            config?.HostConfig?.PortBindings ?? {}
+        const hostPorts = Object.values(portBindings)
+            .flat()
+            .map((b) => parseInt(b.HostPort, 10))
+            .filter((p) => !isNaN(p))
+
+        // Parse resource requirements from metadata (same object-guard as container_config)
+        let minMemoryMB = 256
+        let minDiskMB = 512
+        try {
+            const rawMeta = service.metadata
+            const meta = rawMeta
+                ? typeof rawMeta === 'object'
+                    ? rawMeta
+                    : JSON.parse(rawMeta as string)
+                : null
+            if (meta?.minMemoryMB) minMemoryMB = meta.minMemoryMB
+            if (meta?.minDiskMB) minDiskMB = meta.minDiskMB
+        } catch {}
+
+        const [{ conflicts: portConflicts }, resourceWarnings] = await Promise.all([
+            this.dockerService.checkPortConflicts(hostPorts),
+            this.systemService.checkResourceWarnings(minMemoryMB, minDiskMB),
+        ])
+
+        return response.send({ portConflicts, resourceWarnings })
+    }
+
+    /** Return the next suggested host port for a custom app (8600+ range). */
+    async suggestCustomPort({ response }: HttpContext) {
+        const port = await this.systemService.getNextSuggestedCustomPort()
+        return response.send({ port })
+    }
+
+    /**
+     * Service-less preflight for the custom-app form: given host ports, volumes and an image,
+     * report port conflicts, host resource warnings, overridable guard warnings (risky bind
+     * mounts / untrusted or moving-tag images), and hard blocks (docker socket, system dirs,
+     * malformed image). Lets the form give live feedback before a Service record exists.
+     */
+    async preflightCustomApp({ request, response }: HttpContext) {
+        const payload = await request.validateUsing(preflightCustomValidator)
+        const [{ conflicts }, resourceWarnings] = await Promise.all([
+            this.dockerService.checkPortConflicts(payload.ports ?? []),
+            this.systemService.checkResourceWarnings(256, 512),
+        ])
+        // When editing, the app's own container legitimately holds its ports — don't flag those.
+        const portConflicts = payload.exclude_service
+            ? conflicts.filter((c) => c.usedBy !== payload.exclude_service)
+            : conflicts
+        const guard = evaluateCustomApp({ image: payload.image, volumes: payload.volumes })
+        return response.send({
+            portConflicts,
+            resourceWarnings: [...resourceWarnings, ...guard.warnings],
+            blocked: guard.blocked,
+        })
+    }
+
+    /** Create and immediately begin installing a custom app container. */
+    async createCustomApp({ request, response }: HttpContext) {
+        const payload = await request.validateUsing(customAppValidator)
+
+        // Derive a stable service_name from the friendly name
+        const slug = payload.friendly_name
+            .toLowerCase()
+            .replace(/[^a-z0-9]+/g, '_')
+            .replace(/^_+|_+$/g, '')
+        const serviceName = `nomad_custom_${slug}`
+
+        const existing = await Service.query().where('service_name', serviceName).first()
+        if (existing) {
+            return response.status(409).send({
+                success: false,
+                message: `A custom app named "${payload.friendly_name}" already exists. Choose a different name.`,
+            })
+        }
+
+        // Reject duplicate host ports within the request — Docker would otherwise fail at
+        // start time with an opaque "port is already allocated" error.
+        const hostPorts = (payload.ports ?? []).map((p) => p.host)
+        const duplicateHostPorts = [...new Set(hostPorts.filter((p, i) => hostPorts.indexOf(p) !== i))]
+        if (duplicateHostPorts.length) {
+            return response.status(422).send({
+                success: false,
+                message: `Duplicate host port(s): ${duplicateHostPorts.join(', ')}. Each host port can map to only one container.`,
+            })
+        }
+
+        // Security guardrails: hard-block dangerous bind mounts / malformed images regardless of
+        // force; surface overridable warnings (risky paths, untrusted/moving-tag images) unless forced.
+        const guard = evaluateCustomApp({ image: payload.image, volumes: payload.volumes })
+        if (guard.blocked.length) {
+            return response.status(422).send({
+                success: false,
+                message: guard.blocked.join(' '),
+                blocked: guard.blocked,
+            })
+        }
+        if (!payload.force && guard.warnings.length) {
+            return response.status(409).send({
+                success: false,
+                message: guard.warnings.join(' '),
+                warnings: guard.warnings,
+            })
+        }
+
+        // Advisory preflight: surface port conflicts before creating the record so a failed
+        // install doesn't leave a phantom card. The user can re-submit with force=true to override.
+        if (!payload.force && hostPorts.length) {
+            const { conflicts } = await this.dockerService.checkPortConflicts(hostPorts)
+            if (conflicts.length) {
+                return response.status(409).send({
+                    success: false,
+                    message: `Port conflict: ${conflicts
+                        .map((c) => `${c.port} (in use by ${c.usedBy})`)
+                        .join(', ')}.`,
+                    portConflicts: conflicts,
+                })
+            }
+        }
+
+        const { containerConfig, uiLocation } = this.buildCustomContainerConfig(payload)
+
+        await Service.create({
+            service_name: serviceName,
+            friendly_name: payload.friendly_name,
+            container_image: payload.image,
+            container_config: JSON.stringify(containerConfig),
+            ui_location: uiLocation,
+            icon: payload.icon || 'IconBrandDocker',
+            installed: false,
+            installation_status: 'idle',
+            is_dependency_service: false,
+            is_custom: true,
+            category: payload.category ?? 'custom',
+            depends_on: null,
+        })
+
+        const result = await this.dockerService.createContainerPreflight(serviceName)
+        if (result.success) {
+            return response.send({ success: true, message: result.message, service_name: serviceName })
+        }
+        return response.status(400).send({ success: false, message: result.message })
+    }
+
+    /** Delete a custom app: stop + remove its container, then delete the DB record. */
+    async deleteCustomApp({ request, response }: HttpContext) {
+        const payload = await request.validateUsing(deleteCustomAppValidator)
+
+        const service = await Service.query().where('service_name', payload.service_name).first()
+        if (!service) {
+            return response.status(404).send({ error: `Service ${payload.service_name} not found` })
+        }
+        if (!service.is_custom) {
+            return response.status(403).send({ error: 'Only custom apps can be deleted.' })
+        }
+
+        await this.dockerService.removeCustomAppContainer(payload.service_name, payload.remove_image ?? false)
+        await service.delete()
+
+        return response.send({ success: true, message: `Custom app ${payload.service_name} deleted` })
+    }
+
+    /** Uninstall a curated catalog app: stop + remove its container (optionally its image) and
+     * return the card to the available catalog. App data under the storage path stays on disk,
+     * so a later reinstall picks it back up. Custom apps are removed via deleteCustomApp instead,
+     * which also drops their DB record. */
+    async uninstallService({ request, response }: HttpContext) {
+        const payload = await request.validateUsing(uninstallServiceValidator)
+
+        const service = await Service.query().where('service_name', payload.service_name).first()
+        if (!service) {
+            return response.status(404).send({ error: `Service ${payload.service_name} not found` })
+        }
+        if (service.is_custom) {
+            return response.status(403).send({ error: 'Custom apps are removed via delete.' })
+        }
+        if (service.is_dependency_service) {
+            return response.status(403).send({ error: 'Dependency services cannot be uninstalled directly.' })
+        }
+        if (!service.installed) {
+            return response.status(409).send({ error: `Service ${payload.service_name} is not installed` })
+        }
+
+        const result = await this.dockerService.uninstallService(
+            payload.service_name,
+            payload.remove_image ?? false
+        )
+        if (!result.success) {
+            return response.status(500).send({ success: false, message: result.message })
+        }
+        return response.send({ success: true, message: result.message })
+    }
+
+    /** Set or clear an app's custom launch URL (works for curated and custom apps). Purely a
+     * metadata change — no container is touched. An empty/invalid value clears the override, after
+     * which the default host + port link is used again. */
+    async setServiceCustomUrl({ request, response }: HttpContext) {
+        const payload = await request.validateUsing(setServiceCustomUrlValidator)
+
+        const service = await Service.query().where('service_name', payload.service_name).first()
+        if (!service) {
+            return response.status(404).send({ success: false, message: `Service ${payload.service_name} not found` })
+        }
+        // Hidden dependency services (e.g. Qdrant) aren't user-launchable, so they have no link to set.
+        if (service.is_dependency_service) {
+            return response.status(403).send({ success: false, message: 'This service cannot be configured.' })
+        }
+
+        // Reject a non-empty value that isn't a valid http(s) URL; an empty value clears the override.
+        const normalized = normalizeCustomUrl(payload.custom_url)
+        if (payload.custom_url && payload.custom_url.trim() && !normalized) {
+            return response.status(422).send({
+                success: false,
+                message: 'Custom URL must be a valid http(s) address (e.g. https://jellyfin.myhomelab.net).',
+            })
+        }
+
+        service.custom_url = normalized
+        await service.save()
+
+        return response.send({ success: true, custom_url: service.custom_url })
+    }
+
+    /** Re-pull a custom app's image and recreate its container in place (preserving volumes). */
+    async updateCustomApp_pullLatest({ request, response }: HttpContext) {
+        const payload = await request.validateUsing(installServiceValidator)
+
+        const service = await Service.query().where('service_name', payload.service_name).first()
+        if (!service) {
+            return response.status(404).send({ success: false, message: `Service ${payload.service_name} not found` })
+        }
+        if (!service.is_custom) {
+            return response.status(403).send({ success: false, message: 'Only custom apps can be updated this way.' })
+        }
+
+        const result = await this.dockerService.recreateCustomAppContainer(payload.service_name, {
+            forcePull: true,
+        })
+        if (result.success) {
+            return response.send({ success: true, message: result.message })
+        }
+        return response.status(400).send({ success: false, message: result.message })
+    }
+
+    /** Return the last N lines of a service container's logs. */
+    async getServiceLogs({ params, request, response }: HttpContext) {
+        // Scope to managed services only — otherwise any sibling container's logs (admin app,
+        // database) would be readable by name on this unauthenticated API surface.
+        const service = await Service.query().where('service_name', params.name).first()
+        if (!service) {
+            return response.status(404).send({ success: false, message: `Service ${params.name} not found` })
+        }
+        const { tail } = await request.validateUsing(serviceLogsValidator)
+        const result = await this.dockerService.getContainerLogs(params.name, tail ?? 200)
+        if (!result.success) {
+            return response.status(404).send({ success: false, message: result.message })
+        }
+        return response.send({ success: true, logs: result.logs })
+    }
+
+    /** Return a one-shot CPU/memory usage snapshot for a running service container. */
+    async getServiceStats({ params, response }: HttpContext) {
+        // Scope to managed services only (see getServiceLogs).
+        const service = await Service.query().where('service_name', params.name).first()
+        if (!service) {
+            return response.status(404).send({ success: false, message: `Service ${params.name} not found` })
+        }
+        const result = await this.dockerService.getContainerStats(params.name)
+        if (!result.success) {
+            return response.status(404).send({ success: false, message: result.message })
+        }
+        return response.send({ success: true, running: result.running ?? false, stats: result.stats ?? null })
+    }
+
+    /** Return an app's current configuration in the editable form-shape. */
+    async getCustomApp({ params, response }: HttpContext) {
+        const service = await Service.query().where('service_name', params.name).first()
+        if (!service) {
+            return response.status(404).send({ error: `Service ${params.name} not found` })
+        }
+        // Custom and curated apps are both editable; hidden dependency services (e.g. Qdrant) are not.
+        if (service.is_dependency_service) {
+            return response.status(403).send({ error: 'This service cannot be edited.' })
+        }
+        return response.send({ success: true, app: this.parseCustomContainerConfig(service) })
+    }
+
+    /** Reconfigure an app: validate + guard, persist the new config, then recreate the container.
+     * Works for both custom apps and curated (pre-configured) apps. Editing a curated app marks it
+     * user-modified so the seeder stops overwriting the user's changes. */
+    async updateCustomApp({ request, response }: HttpContext) {
+        const payload = await request.validateUsing(updateCustomAppValidator)
+
+        const service = await Service.query().where('service_name', payload.service_name).first()
+        if (!service) {
+            return response.status(404).send({ success: false, message: `Service ${payload.service_name} not found` })
+        }
+        // Custom and curated apps are both editable; hidden dependency services (e.g. Qdrant) are not.
+        if (service.is_dependency_service) {
+            return response.status(403).send({ success: false, message: 'This service cannot be edited.' })
+        }
+
+        // Reject duplicate host ports within the request.
+        const hostPorts = (payload.ports ?? []).map((p) => p.host)
+        const duplicateHostPorts = [...new Set(hostPorts.filter((p, i) => hostPorts.indexOf(p) !== i))]
+        if (duplicateHostPorts.length) {
+            return response.status(422).send({
+                success: false,
+                message: `Duplicate host port(s): ${duplicateHostPorts.join(', ')}. Each host port can map to only one container.`,
+            })
+        }
+
+        // Security guardrails (same posture as create).
+        const guard = evaluateCustomApp({ image: payload.image, volumes: payload.volumes })
+        if (guard.blocked.length) {
+            return response.status(422).send({ success: false, message: guard.blocked.join(' '), blocked: guard.blocked })
+        }
+        if (!payload.force && guard.warnings.length) {
+            return response.status(409).send({ success: false, message: guard.warnings.join(' '), warnings: guard.warnings })
+        }
+
+        // Port conflicts — but ignore ports already held by this app's own container.
+        if (!payload.force && hostPorts.length) {
+            const { conflicts } = await this.dockerService.checkPortConflicts(hostPorts)
+            const external = conflicts.filter((c) => c.usedBy !== payload.service_name)
+            if (external.length) {
+                return response.status(409).send({
+                    success: false,
+                    message: `Port conflict: ${external
+                        .map((c) => `${c.port} (in use by ${c.usedBy})`)
+                        .join(', ')}.`,
+                    portConflicts: external,
+                })
+            }
+        }
+
+        // Merge the form fields into the app's existing config rather than rebuilding from scratch,
+        // so advanced settings a curated app ships with (GPU device requests, special env, etc.) are
+        // preserved across an edit.
+        // Preserve an explicit scheme (e.g. ui_location "https:8480") across an edit — otherwise a
+        // TLS-serving app's Open link would silently revert to http after any reconfigure.
+        const prevScheme = (service.ui_location || '').match(/^(https?):\d+$/)?.[1]
+        const { containerConfig, uiLocation } = this.mergeCustomContainerConfig(
+            service.container_config,
+            payload
+        )
+        service.friendly_name = payload.friendly_name
+        service.container_image = payload.image
+        service.container_config = JSON.stringify(containerConfig)
+        service.ui_location = prevScheme && uiLocation && /^\d+$/.test(uiLocation)
+            ? `${prevScheme}:${uiLocation}`
+            : uiLocation
+        service.category = payload.category ?? service.category ?? 'custom'
+        if (payload.icon) service.icon = payload.icon
+        // Flag as user-modified so the seeder stops overwriting this app's config on future runs.
+        service.is_user_modified = true
+        await service.save()
+
+        const result = await this.dockerService.recreateCustomAppContainer(payload.service_name)
+        if (result.success) {
+            return response.send({ success: true, message: result.message, service_name: payload.service_name })
+        }
+        return response.status(400).send({ success: false, message: result.message })
+    }
+
+    /**
+     * Build a Docker container config (HostConfig + ExposedPorts + Env) from custom-app form input,
+     * applying default resource caps. Shared by create and update so both stay in lockstep.
+     */
+    private buildCustomContainerConfig(payload: {
+        ports?: { container: number; host: number }[]
+        volumes?: { host_path: string; container_path: string }[]
+        env?: string[]
+        memory_mb?: number
+        cpus?: number
+    }): { containerConfig: Record<string, any>; uiLocation: string | null } {
+        const portBindings: Record<string, [{ HostPort: string }]> = {}
+        const exposedPorts: Record<string, {}> = {}
+        for (const { container, host } of payload.ports ?? []) {
+            portBindings[`${container}/tcp`] = [{ HostPort: String(host) }]
+            exposedPorts[`${container}/tcp`] = {}
+        }
+
+        const binds = (payload.volumes ?? []).map(
+            ({ host_path, container_path }) => `${host_path}:${container_path}`
+        )
+
+        // Resource caps so a runaway custom container can't starve the host. Memory is bytes;
+        // NanoCpus is CPUs × 1e9. Defaults are generous and user-overridable.
+        const memoryBytes = (payload.memory_mb ?? DEFAULT_MEMORY_MB) * 1024 * 1024
+        const nanoCpus = Math.round((payload.cpus ?? DEFAULT_CPUS) * 1e9)
+
+        const containerConfig: Record<string, any> = {
+            HostConfig: {
+                RestartPolicy: { Name: 'unless-stopped' },
+                PortBindings: portBindings,
+                Memory: memoryBytes,
+                NanoCpus: nanoCpus,
+                ...(binds.length ? { Binds: binds } : {}),
+            },
+            ExposedPorts: exposedPorts,
+            ...(payload.env?.length ? { Env: payload.env } : {}),
+        }
+
+        const firstHostPort = payload.ports?.[0]?.host
+        const uiLocation = firstHostPort ? String(firstHostPort) : null
+        return { containerConfig, uiLocation }
+    }
+
+    /**
+     * Merge custom-app form input into an app's *existing* container config. Used by the edit path so
+     * editing a curated app only changes the fields exposed in the form (image/ports/volumes/env and,
+     * if supplied, resource caps) while preserving everything else it ships with (GPU DeviceRequests,
+     * User, custom HostConfig keys, etc.). Unlike buildCustomContainerConfig, resource caps are NOT
+     * defaulted here — a curated app intentionally left uncapped stays uncapped unless the user sets one.
+     */
+    private mergeCustomContainerConfig(
+        existingRaw: string | null,
+        payload: {
+            ports?: { container: number; host: number }[]
+            volumes?: { host_path: string; container_path: string }[]
+            env?: string[]
+            memory_mb?: number
+            cpus?: number
+        }
+    ): { containerConfig: Record<string, any>; uiLocation: string | null } {
+        const parsed = existingRaw
+            ? typeof existingRaw === 'object'
+                ? existingRaw
+                : JSON.parse(existingRaw as string)
+            : {}
+        // Deep clone so we never mutate the parsed source.
+        const containerConfig: Record<string, any> = JSON.parse(JSON.stringify(parsed ?? {}))
+        containerConfig.HostConfig = containerConfig.HostConfig ?? {}
+        // Keep a restart policy if the existing config lacked one.
+        containerConfig.HostConfig.RestartPolicy =
+            containerConfig.HostConfig.RestartPolicy ?? { Name: 'unless-stopped' }
+
+        const portBindings: Record<string, [{ HostPort: string }]> = {}
+        const exposedPorts: Record<string, {}> = {}
+        for (const { container, host } of payload.ports ?? []) {
+            portBindings[`${container}/tcp`] = [{ HostPort: String(host) }]
+            exposedPorts[`${container}/tcp`] = {}
+        }
+        containerConfig.HostConfig.PortBindings = portBindings
+        containerConfig.ExposedPorts = exposedPorts
+
+        const binds = (payload.volumes ?? []).map(
+            ({ host_path, container_path }) => `${host_path}:${container_path}`
+        )
+        if (binds.length) containerConfig.HostConfig.Binds = binds
+        else delete containerConfig.HostConfig.Binds
+
+        if (payload.env?.length) containerConfig.Env = payload.env
+        else delete containerConfig.Env
+
+        // Only touch resource caps when the user explicitly set them — preserve existing/uncapped otherwise.
+        if (payload.memory_mb != null) {
+            containerConfig.HostConfig.Memory = payload.memory_mb * 1024 * 1024
+        }
+        if (payload.cpus != null) {
+            containerConfig.HostConfig.NanoCpus = Math.round(payload.cpus * 1e9)
+        }
+
+        const firstHostPort = payload.ports?.[0]?.host
+        const uiLocation = firstHostPort ? String(firstHostPort) : null
+        return { containerConfig, uiLocation }
+    }
+
+    /** Inverse of buildCustomContainerConfig: turn a stored Service into the editable form-shape. */
+    private parseCustomContainerConfig(service: Service) {
+        const raw = service.container_config
+        const config = raw ? (typeof raw === 'object' ? raw : JSON.parse(raw as string)) : {}
+        const hostConfig = config?.HostConfig ?? {}
+
+        const ports = Object.entries(hostConfig.PortBindings ?? {}).map(([key, val]: [string, any]) => ({
+            container: Number.parseInt(key, 10),
+            host: Number.parseInt(val?.[0]?.HostPort, 10),
+        }))
+
+        const volumes = (hostConfig.Binds ?? []).map((bind: string) => {
+            const idx = bind.indexOf(':')
+            return { host_path: bind.slice(0, idx), container_path: bind.slice(idx + 1) }
+        })
+
+        return {
+            service_name: service.service_name,
+            friendly_name: service.friendly_name,
+            image: service.container_image,
+            category: service.category ?? 'custom',
+            icon: service.icon ?? 'IconBrandDocker',
+            ports,
+            volumes,
+            env: (config?.Env ?? []) as string[],
+            memory_mb: hostConfig.Memory ? Math.round(hostConfig.Memory / (1024 * 1024)) : undefined,
+            cpus: hostConfig.NanoCpus ? hostConfig.NanoCpus / 1e9 : undefined,
+        }
+    }
 }

admin/app/controllers/zim_controller.ts

@@ -1,12 +1,19 @@
 import { ZimService } from '#services/zim_service'
 import {
-  downloadCollectionValidator,
+  assertNotPrivateUrl,
+  downloadCategoryTierValidator,
   filenameParamValidator,
-  remoteDownloadValidator,
+  remoteDownloadWithMetadataValidator,
+  selectWikipediaValidator,
 } from '#validators/common'
-import { listRemoteZimValidator } from '#validators/zim'
+import { addCustomLibraryValidator, browseLibraryValidator, idParamValidator, listRemoteZimValidator } from '#validators/zim'
 import { inject } from '@adonisjs/core'
+import logger from '@adonisjs/core/services/logger'
 import type { HttpContext } from '@adonisjs/core/http'
+import { createWriteStream } from 'fs'
+import { rename } from 'fs/promises'
+import { join, resolve, sep } from 'path'
+import { ZIM_STORAGE_PATH, ensureDirectoryExists, sanitizeFilename } from '../utils/fs.js'
 
 @inject()
 export default class ZimController {
@@ -23,8 +30,9 @@ export default class ZimController {
   }
 
   async downloadRemote({ request }: HttpContext) {
-    const payload = await request.validateUsing(remoteDownloadValidator)
-    const { filename, jobId } = await this.zimService.downloadRemote(payload.url)
+    const payload = await request.validateUsing(remoteDownloadWithMetadataValidator)
+    assertNotPrivateUrl(payload.url)
+    const { filename, jobId } = await this.zimService.downloadRemote(payload.url, payload.metadata)
 
     return {
       message: 'Download started successfully',
@@ -34,24 +42,31 @@ export default class ZimController {
     }
   }
 
-  async downloadCollection({ request }: HttpContext) {
-    const payload = await request.validateUsing(downloadCollectionValidator)
-    const resources = await this.zimService.downloadCollection(payload.slug)
+  async listCuratedCategories({}: HttpContext) {
+    return await this.zimService.listCuratedCategories()
+  }
+
+  async downloadCategoryTier({ request }: HttpContext) {
+    const payload = await request.validateUsing(downloadCategoryTierValidator)
+    const resources = await this.zimService.downloadCategoryTier(
+      payload.categorySlug,
+      payload.tierSlug
+    )
 
     return {
       message: 'Download started successfully',
-      slug: payload.slug,
+      categorySlug: payload.categorySlug,
+      tierSlug: payload.tierSlug,
       resources,
     }
   }
 
-  async listCuratedCollections({}: HttpContext) {
-    return this.zimService.listCuratedCollections()
-  }
-
-  async fetchLatestCollections({}: HttpContext) {
-    const success = await this.zimService.fetchLatestCollections()
-    return { success }
+  async rescanLibrary({}: HttpContext) {
+    const result = await this.zimService.rescanLibrary()
+    return {
+      message: 'Kiwix library rescanned',
+      ...result,
+    }
   }
 
   async delete({ request, response }: HttpContext) {
@@ -72,4 +87,143 @@ export default class ZimController {
       message: 'ZIM file deleted successfully',
     }
   }
+
+  async upload({ request, response }: HttpContext) {
+    let filename: string | null = null
+    let tmpPath: string | null = null
+    let uploadError: string | null = null
+
+    try {
+      const basePath = resolve(join(process.cwd(), ZIM_STORAGE_PATH))
+      await ensureDirectoryExists(basePath)
+
+      request.multipart.onFile('*', {}, async (part) => {
+        const clientName = part.filename || ''
+        if (!clientName.toLowerCase().endsWith('.zim')) {
+          part.resume()
+          uploadError = 'INVALID_TYPE'
+          return
+        }
+
+        const sanitized = sanitizeFilename(clientName)
+        const finalPath = resolve(join(basePath, sanitized))
+
+        if (!finalPath.startsWith(basePath + sep)) {
+          part.resume()
+          uploadError = 'INVALID_FILENAME'
+          return
+        }
+
+        const { access } = await import('fs/promises')
+        const exists = await access(finalPath).then(() => true).catch(() => false)
+        if (exists) {
+          part.resume()
+          uploadError = 'DUPLICATE_FILENAME'
+          return
+        }
+
+        filename = sanitized
+        tmpPath = finalPath + '.tmp'
+        const ws = createWriteStream(tmpPath)
+
+        await new Promise<void>((res, rej) => {
+          ws.on('error', rej)
+          ws.on('finish', res)
+          part.on('error', rej)
+          part.pipe(ws)
+        })
+
+        await rename(tmpPath, finalPath)
+        tmpPath = null
+      })
+
+      await request.multipart.process()
+
+      if (uploadError === 'INVALID_TYPE') {
+        return response.status(422).send({ message: 'Only .zim files are accepted' })
+      }
+      if (uploadError === 'INVALID_FILENAME') {
+        return response.status(422).send({ message: 'Invalid filename' })
+      }
+      if (uploadError === 'DUPLICATE_FILENAME') {
+        return response.status(409).send({ message: 'A ZIM file with that name already exists' })
+      }
+      if (!filename) {
+        return response.status(400).send({ message: 'No file received' })
+      }
+
+      const { added } = await this.zimService.registerLocalUpload(filename)
+
+      return response.status(201).send({
+        message: 'ZIM file uploaded and registered successfully',
+        filename,
+        added,
+      })
+    } catch (error) {
+      logger.error('[ZimController] Upload failed:', error)
+      if (tmpPath) {
+        const { unlink } = await import('fs/promises')
+        await unlink(tmpPath).catch(() => {})
+      }
+      return response.status(500).send({ message: 'Upload failed' })
+    }
+  }
+
+  // Wikipedia selector endpoints
+
+  async getWikipediaState({}: HttpContext) {
+    return this.zimService.getWikipediaState()
+  }
+
+  async selectWikipedia({ request }: HttpContext) {
+    const payload = await request.validateUsing(selectWikipediaValidator)
+    return this.zimService.selectWikipedia(payload.optionId)
+  }
+
+  // Custom library endpoints
+
+  async listCustomLibraries({}: HttpContext) {
+    return this.zimService.listCustomLibraries()
+  }
+
+  async addCustomLibrary({ request, response }: HttpContext) {
+    const payload = await request.validateUsing(addCustomLibraryValidator)
+    assertNotPrivateUrl(payload.base_url)
+    try {
+      const source = await this.zimService.addCustomLibrary(payload.name, payload.base_url)
+      return { message: 'Custom library added', library: source }
+    } catch (error) {
+      if (error.message === 'Maximum of 10 custom libraries allowed') {
+        return response.status(400).send({ message: error.message })
+      }
+      throw error
+    }
+  }
+
+  async removeCustomLibrary({ request, response }: HttpContext) {
+    const payload = await request.validateUsing(idParamValidator)
+    try {
+      await this.zimService.removeCustomLibrary(payload.params.id)
+      return { message: 'Custom library removed' }
+    } catch (error) {
+      if (error.message === 'Custom library not found') {
+        return response.status(404).send({ message: error.message })
+      }
+      throw error
+    }
+  }
+
+  async browseLibrary({ request, response }: HttpContext) {
+    const payload = await request.validateUsing(browseLibraryValidator)
+    try {
+      return await this.zimService.browseLibraryUrl(payload.url)
+    } catch (error) {
+      if (error.message?.includes('loopback or link-local')) {
+        return response.status(400).send({ message: error.message })
+      }
+      return response.status(502).send({
+        message: 'Could not fetch directory listing from the provided URL',
+      })
+    }
+  }
 }

admin/app/jobs/app_auto_update_job.ts

@@ -0,0 +1,78 @@
+import { Job } from 'bullmq'
+import { QueueService } from '#services/queue_service'
+import { DockerService } from '#services/docker_service'
+import { DownloadService } from '#services/download_service'
+import { SystemService } from '#services/system_service'
+import { ContainerRegistryService } from '#services/container_registry_service'
+import { AppAutoUpdateService } from '#services/app_auto_update_service'
+import logger from '@adonisjs/core/services/logger'
+
+/**
+ * Hourly job that evaluates whether any opted-in installed apps should auto-update
+ * right now and, if so, updates them. All gating (master switch, per-app opt-in,
+ * window, cool-off, pre-flight, per-app backoff) lives in {@link AppAutoUpdateService};
+ * this job is just the scheduled trigger. Runs hourly so it can act anywhere inside a
+ * user's window regardless of the window's length. Mirrors {@link AutoUpdateJob}.
+ */
+export class AppAutoUpdateJob {
+  static get queue() {
+    return 'system'
+  }
+
+  static get key() {
+    return 'app-auto-update'
+  }
+
+  async handle(_job: Job) {
+    logger.info('[AppAutoUpdateJob] Evaluating app auto-updates...')
+
+    const dockerService = new DockerService()
+    const appAutoUpdateService = new AppAutoUpdateService(
+      dockerService,
+      new DownloadService(QueueService.getInstance()),
+      new SystemService(dockerService),
+      new ContainerRegistryService()
+    )
+
+    const result = await appAutoUpdateService.attempt()
+    logger.info(`[AppAutoUpdateJob] ${result.updated} updated: ${result.reason}`)
+    return result
+  }
+
+  static async schedule() {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+
+    await queue.upsertJobScheduler(
+      'hourly-app-auto-update',
+      { pattern: '0 * * * *' }, // Top of every hour; attempt() gates on the window
+      {
+        name: this.key,
+        opts: {
+          removeOnComplete: { count: 12 },
+          removeOnFail: { count: 5 },
+        },
+      }
+    )
+
+    logger.info('[AppAutoUpdateJob] App auto-update evaluation scheduled with cron: 0 * * * *')
+  }
+
+  static async dispatch() {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+
+    const job = await queue.add(
+      this.key,
+      {},
+      {
+        attempts: 1,
+        removeOnComplete: { count: 12 },
+        removeOnFail: { count: 5 },
+      }
+    )
+
+    logger.info(`[AppAutoUpdateJob] Dispatched ad-hoc app auto-update evaluation job ${job.id}`)
+    return job
+  }
+}

admin/app/jobs/auto_update_job.ts

@@ -0,0 +1,80 @@
+import { Job } from 'bullmq'
+import { QueueService } from '#services/queue_service'
+import { DockerService } from '#services/docker_service'
+import { DownloadService } from '#services/download_service'
+import { SystemService } from '#services/system_service'
+import { SystemUpdateService } from '#services/system_update_service'
+import { ContainerRegistryService } from '#services/container_registry_service'
+import { AutoUpdateService } from '#services/auto_update_service'
+import logger from '@adonisjs/core/services/logger'
+
+/**
+ * Hourly job that evaluates whether the NOMAD application should auto-update right
+ * now and, if so, requests it. All gating (opt-in, window, eligibility, cool-off,
+ * pre-flight, backoff) lives in {@link AutoUpdateService}; this job is just the
+ * scheduled trigger. Runs hourly so it can act anywhere inside a user's window
+ * regardless of the window's length.
+ */
+export class AutoUpdateJob {
+  static get queue() {
+    return 'system'
+  }
+
+  static get key() {
+    return 'auto-update'
+  }
+
+  async handle(_job: Job) {
+    logger.info('[AutoUpdateJob] Evaluating auto-update...')
+
+    const dockerService = new DockerService()
+    const autoUpdateService = new AutoUpdateService(
+      dockerService,
+      new DownloadService(QueueService.getInstance()),
+      new SystemService(dockerService),
+      new SystemUpdateService(),
+      new ContainerRegistryService()
+    )
+
+    const result = await autoUpdateService.attempt()
+    logger.info(`[AutoUpdateJob] ${result.updated ? 'Updating' : 'No update'}: ${result.reason}`)
+    return result
+  }
+
+  static async schedule() {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+
+    await queue.upsertJobScheduler(
+      'hourly-auto-update',
+      { pattern: '0 * * * *' }, // Top of every hour; attempt() gates on the window
+      {
+        name: this.key,
+        opts: {
+          removeOnComplete: { count: 12 },
+          removeOnFail: { count: 5 },
+        },
+      }
+    )
+
+    logger.info('[AutoUpdateJob] Auto-update evaluation scheduled with cron: 0 * * * *')
+  }
+
+  static async dispatch() {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+
+    const job = await queue.add(
+      this.key,
+      {},
+      {
+        attempts: 1,
+        removeOnComplete: { count: 12 },
+        removeOnFail: { count: 5 },
+      }
+    )
+
+    logger.info(`[AutoUpdateJob] Dispatched ad-hoc auto-update evaluation job ${job.id}`)
+    return job
+  }
+}

admin/app/jobs/check_service_updates_job.ts

@@ -0,0 +1,142 @@
+import { Job } from 'bullmq'
+import { QueueService } from '#services/queue_service'
+import { DockerService } from '#services/docker_service'
+import { ContainerRegistryService } from '#services/container_registry_service'
+import Service from '#models/service'
+import logger from '@adonisjs/core/services/logger'
+import transmit from '@adonisjs/transmit/services/main'
+import { BROADCAST_CHANNELS } from '../../constants/broadcast.js'
+import { DateTime } from 'luxon'
+
+export class CheckServiceUpdatesJob {
+  static get queue() {
+    return 'service-updates'
+  }
+
+  static get key() {
+    return 'check-service-updates'
+  }
+
+  async handle(_job: Job) {
+    logger.info('[CheckServiceUpdatesJob] Checking for service updates...')
+
+    const dockerService = new DockerService()
+    const registryService = new ContainerRegistryService()
+
+    // Determine host architecture
+    const hostArch = await this.getHostArch(dockerService)
+
+    const installedServices = await Service.query().where('installed', true)
+    let updatesFound = 0
+
+    for (const service of installedServices) {
+      try {
+        const updates = await registryService.getAvailableUpdates(
+          service.container_image,
+          hostArch,
+          service.source_repo
+        )
+
+        const latestUpdate = updates.length > 0 ? updates[0].tag : null
+
+        // Stamp/clear the cool-off anchor only when the available version *changes*.
+        // Registry tags carry no publish date, so the auto-update cool-off is measured
+        // from when a version was first detected; leaving the timestamp untouched while
+        // the same version persists keeps the cool-off clock running.
+        if (latestUpdate !== service.available_update_version) {
+          service.available_update_first_seen_at = latestUpdate ? DateTime.now() : null
+        }
+
+        service.available_update_version = latestUpdate
+        service.update_checked_at = DateTime.now()
+        await service.save()
+
+        if (latestUpdate) {
+          updatesFound++
+          logger.info(
+            `[CheckServiceUpdatesJob] Update available for ${service.service_name}: ${service.container_image} → ${latestUpdate}`
+          )
+        }
+      } catch (error) {
+        logger.error(
+          `[CheckServiceUpdatesJob] Failed to check updates for ${service.service_name}: ${error.message}`
+        )
+        // Continue checking other services
+      }
+    }
+
+    logger.info(
+      `[CheckServiceUpdatesJob] Completed. ${updatesFound} update(s) found for ${installedServices.length} service(s).`
+    )
+
+    // Broadcast completion so the frontend can refresh
+    transmit.broadcast(BROADCAST_CHANNELS.SERVICE_UPDATES, {
+      status: 'completed',
+      updatesFound,
+      timestamp: new Date().toISOString(),
+    })
+
+    return { updatesFound }
+  }
+
+  private async getHostArch(dockerService: DockerService): Promise<string> {
+    try {
+      const info = await dockerService.docker.info()
+      const arch = info.Architecture || ''
+
+      // Map Docker architecture names to OCI names
+      const archMap: Record<string, string> = {
+        x86_64: 'amd64',
+        aarch64: 'arm64',
+        armv7l: 'arm',
+        amd64: 'amd64',
+        arm64: 'arm64',
+      }
+
+      return archMap[arch] || arch.toLowerCase()
+    } catch (error) {
+      logger.warn(
+        `[CheckServiceUpdatesJob] Could not detect host architecture: ${error.message}. Defaulting to amd64.`
+      )
+      return 'amd64'
+    }
+  }
+
+  static async scheduleNightly() {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+
+    await queue.upsertJobScheduler(
+      'nightly-service-update-check',
+      { pattern: '0 3 * * *' },
+      {
+        name: this.key,
+        opts: {
+          removeOnComplete: { count: 7 },
+          removeOnFail: { count: 5 },
+        },
+      }
+    )
+
+    logger.info('[CheckServiceUpdatesJob] Service update check scheduled with cron: 0 3 * * *')
+  }
+
+  static async dispatch() {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+
+    const job = await queue.add(
+      this.key,
+      {},
+      {
+        attempts: 3,
+        backoff: { type: 'exponential', delay: 60000 },
+        removeOnComplete: { count: 7 },
+        removeOnFail: { count: 5 },
+      }
+    )
+
+    logger.info(`[CheckServiceUpdatesJob] Dispatched ad-hoc service update check job ${job.id}`)
+    return job
+  }
+}

admin/app/jobs/check_update_job.ts

@@ -0,0 +1,77 @@
+import { Job } from 'bullmq'
+import { QueueService } from '#services/queue_service'
+import { DockerService } from '#services/docker_service'
+import { SystemService } from '#services/system_service'
+import logger from '@adonisjs/core/services/logger'
+import KVStore from '#models/kv_store'
+
+export class CheckUpdateJob {
+  static get queue() {
+    return 'system'
+  }
+
+  static get key() {
+    return 'check-update'
+  }
+
+  async handle(_job: Job) {
+    logger.info('[CheckUpdateJob] Running update check...')
+
+    const dockerService = new DockerService()
+    const systemService = new SystemService(dockerService)
+
+    try {
+      const result = await systemService.checkLatestVersion()
+
+      if (result.updateAvailable) {
+        logger.info(
+          `[CheckUpdateJob] Update available: ${result.currentVersion} → ${result.latestVersion}`
+        )
+      } else {
+        await KVStore.setValue('system.updateAvailable', false)
+        logger.info(
+          `[CheckUpdateJob] System is up to date (${result.currentVersion})`
+        )
+      }
+
+      return result
+    } catch (error) {
+      logger.error(`[CheckUpdateJob] Update check failed: ${error.message}`)
+      throw error
+    }
+  }
+
+  static async scheduleNightly() {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+
+    await queue.upsertJobScheduler(
+      'nightly-update-check',
+      { pattern: '0 2,14 * * *' }, // Every 12 hours at 2am and 2pm
+      {
+        name: this.key,
+        opts: {
+          removeOnComplete: { count: 7 },
+          removeOnFail: { count: 5 },
+        },
+      }
+    )
+
+    logger.info('[CheckUpdateJob] Update check scheduled with cron: 0 2,14 * * *')
+  }
+
+  static async dispatch() {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+
+    const job = await queue.add(this.key, {}, {
+      attempts: 3,
+      backoff: { type: 'exponential', delay: 60000 },
+      removeOnComplete: { count: 7 },
+      removeOnFail: { count: 5 },
+    })
+
+    logger.info(`[CheckUpdateJob] Dispatched ad-hoc update check job ${job.id}`)
+    return job
+  }
+}

admin/app/jobs/content_auto_update_job.ts

@@ -0,0 +1,72 @@
+import { Job } from 'bullmq'
+import { QueueService } from '#services/queue_service'
+import { DownloadService } from '#services/download_service'
+import { ContentAutoUpdateService } from '#services/content_auto_update_service'
+import logger from '@adonisjs/core/services/logger'
+
+/**
+ * Hourly job that evaluates whether any installed content (ZIM/map) should
+ * auto-update right now and, if so, dispatches the downloads. All gating (master
+ * switch, content window, cool-off, per-window data cap, pre-flight, backoff)
+ * lives in {@link ContentAutoUpdateService}; this job is just the scheduled
+ * trigger. Runs hourly so it can act anywhere inside a user's window regardless
+ * of the window's length. Mirrors {@link AppAutoUpdateJob}.
+ */
+export class ContentAutoUpdateJob {
+  static get queue() {
+    return 'system'
+  }
+
+  static get key() {
+    return 'content-auto-update'
+  }
+
+  async handle(_job: Job) {
+    logger.info('[ContentAutoUpdateJob] Evaluating content auto-updates...')
+
+    const contentAutoUpdateService = new ContentAutoUpdateService(
+      new DownloadService(QueueService.getInstance())
+    )
+
+    const result = await contentAutoUpdateService.attempt()
+    logger.info(`[ContentAutoUpdateJob] ${result.started} started: ${result.reason}`)
+    return result
+  }
+
+  static async schedule() {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+
+    await queue.upsertJobScheduler(
+      'hourly-content-auto-update',
+      { pattern: '0 * * * *' }, // Top of every hour; attempt() gates on the window
+      {
+        name: this.key,
+        opts: {
+          removeOnComplete: { count: 12 },
+          removeOnFail: { count: 5 },
+        },
+      }
+    )
+
+    logger.info('[ContentAutoUpdateJob] Content auto-update evaluation scheduled with cron: 0 * * * *')
+  }
+
+  static async dispatch() {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+
+    const job = await queue.add(
+      this.key,
+      {},
+      {
+        attempts: 1,
+        removeOnComplete: { count: 12 },
+        removeOnFail: { count: 5 },
+      }
+    )
+
+    logger.info(`[ContentAutoUpdateJob] Dispatched ad-hoc content auto-update evaluation job ${job.id}`)
+    return job
+  }
+}

admin/app/jobs/download_model_job.ts

@@ -1,9 +1,8 @@
-import { Job } from 'bullmq'
+import { Job, UnrecoverableError } from 'bullmq'
 import { QueueService } from '#services/queue_service'
-import { OpenWebUIService } from '#services/openwebui_service'
 import { createHash } from 'crypto'
 import logger from '@adonisjs/core/services/logger'
-import { DockerService } from '#services/docker_service'
+import { OllamaService } from '#services/ollama_service'
 
 export interface DownloadModelJobParams {
   modelName: string
@@ -22,86 +21,159 @@ export class DownloadModelJob {
     return createHash('sha256').update(modelName).digest('hex').slice(0, 16)
   }
 
+  /** In-memory registry of abort controllers for active model download jobs */
+  static abortControllers: Map<string, AbortController> = new Map()
+
+  /**
+   * Redis key used to signal cancellation across processes. Uses a `model-cancel` prefix
+   * so it cannot collide with content download cancel signals (`nomad:download:cancel:*`).
+   */
+  static cancelKey(jobId: string): string {
+    return `nomad:download:model-cancel:${jobId}`
+  }
+
+  /** Signal cancellation via Redis so the worker process can pick it up on its next poll tick */
+  static async signalCancel(jobId: string): Promise<void> {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+    const client = await queue.client
+    await client.set(this.cancelKey(jobId), '1', { EX: 300 }) // 5 min TTL
+  }
+
   async handle(job: Job) {
     const { modelName } = job.data as DownloadModelJobParams
 
     logger.info(`[DownloadModelJob] Attempting to download model: ${modelName}`)
 
-    // Check if OpenWebUI/Ollama services are ready
-    const dockerService = new DockerService()
-    const openWebUIService = new OpenWebUIService(dockerService)
+    const ollamaService = new OllamaService()
 
-    // Use getInstalledModels to check if the service is ready
     // Even if no models are installed, this should return an empty array if ready
-    const existingModels = await openWebUIService.getInstalledModels()
+    const existingModels = await ollamaService.getModels()
     if (!existingModels) {
       logger.warn(
-        `[DownloadModelJob] OpenWebUI service not ready yet for model ${modelName}. Will retry...`
+        `[DownloadModelJob] Ollama service not ready yet for model ${modelName}. Will retry...`
       )
-      throw new Error('OpenWebUI service not ready yet')
+      throw new Error('Ollama service not ready yet')
     }
 
     logger.info(
-      `[DownloadModelJob] OpenWebUI service is ready. Initiating download for ${modelName}`
+      `[DownloadModelJob] Ollama service is ready. Initiating download for ${modelName}`
     )
 
-    // Services are ready, initiate the download with progress tracking
-    const result = await openWebUIService._downloadModel(modelName, (progress) => {
-      // Update job progress in BullMQ
-      const progressData = {
-        status: progress.status,
-        percent: progress.percent,
-        completed: progress.completed,
-        total: progress.total,
+    // Register abort controller for this job — used both by in-process cancels (same process
+    // as the API server) and as the target of the Redis poll loop below.
+    const abortController = new AbortController()
+    DownloadModelJob.abortControllers.set(job.id!, abortController)
+
+    // Get Redis client for checking cancel signals from the API process
+    const queueService = QueueService.getInstance()
+    const cancelRedis = await queueService.getQueue(DownloadModelJob.queue).client
+
+    // Track whether cancellation was explicitly requested by the user. Only user-initiated
+    // cancels become UnrecoverableError — other failures (e.g., transient network errors)
+    // should still benefit from BullMQ's retry logic.
+    let userCancelled = false
+
+    // Poll Redis for cancel signal every 2s — independent of progress events so cancellation
+    // works even when the pull is mid-blob and not emitting progress updates.
+    let cancelPollInterval: ReturnType<typeof setInterval> | null = setInterval(async () => {
+      try {
+        const val = await cancelRedis.get(DownloadModelJob.cancelKey(job.id!))
+        if (val) {
+          await cancelRedis.del(DownloadModelJob.cancelKey(job.id!))
+          userCancelled = true
+          abortController.abort('user-cancel')
+        }
+      } catch {
+        // Redis errors are non-fatal; in-process AbortController covers same-process cancels
       }
+    }, 2000)
 
-      // Update the job progress (0-100 scale for BullMQ)
-      if (progress.percent !== undefined) {
-        job.updateProgress(progress.percent)
-      }
+    try {
+      // Services are ready, initiate the download with progress tracking
+      const result = await ollamaService.downloadModel(
+        modelName,
+        (progressPercent, bytes) => {
+          if (progressPercent) {
+            job.updateProgress(Math.floor(progressPercent)).catch((err) => {
+              if (err?.code !== -1) throw err
+            })
+          }
 
-      // Log progress with job context
-      if (progress.percent !== undefined) {
-        logger.info(
-          `[DownloadModelJob] Model ${modelName}: ${progress.status} - ${progress.percent}% (${progress.completed}/${progress.total} bytes)`
-        )
-      } else {
-        logger.info(`[DownloadModelJob] Model ${modelName}: ${progress.status}`)
-      }
-
-      // Store detailed progress in job data for clients to query
-      job.updateData({
-        ...job.data,
-        progress: progressData,
-      })
-    })
-
-    if (!result.success) {
-      logger.error(
-        `[DownloadModelJob] Failed to initiate download for model ${modelName}: ${result.message}`
+          // Store detailed progress in job data for clients to query
+          job.updateData({
+            ...job.data,
+            status: 'downloading',
+            progress: progressPercent,
+            downloadedBytes: bytes?.downloadedBytes,
+            totalBytes: bytes?.totalBytes,
+            progress_timestamp: new Date().toISOString(),
+          }).catch((err) => {
+            if (err?.code !== -1) throw err
+          })
+        },
+        abortController.signal,
+        job.id!
       )
-      throw new Error(`Failed to initiate download for model: ${result.message}`)
-    }
 
-    logger.info(`[DownloadModelJob] Successfully completed download for model ${modelName}`)
-    return {
-      modelName,
-      message: result.message,
+      if (!result.success) {
+        logger.error(
+          `[DownloadModelJob] Failed to initiate download for model ${modelName}: ${result.message}`
+        )
+        // User-initiated cancel — must be unrecoverable to avoid the 40-attempt retry storm.
+        // The downloadModel() catch block returns retryable: false for cancels, so this branch
+        // catches both Ollama version mismatches (existing) AND user cancels (new).
+        if (result.retryable === false) {
+          throw new UnrecoverableError(result.message)
+        }
+        throw new Error(`Failed to initiate download for model: ${result.message}`)
+      }
+
+      logger.info(`[DownloadModelJob] Successfully completed download for model ${modelName}`)
+      return {
+        modelName,
+        message: result.message,
+      }
+    } catch (error: any) {
+      // Belt-and-suspenders: if downloadModel didn't recognize the cancel (e.g., the abort
+      // fired after the response stream completed but before our code returned), the cancel
+      // flag tells us this was a user action and should be unrecoverable.
+      if (userCancelled || abortController.signal.reason === 'user-cancel') {
+        if (!(error instanceof UnrecoverableError)) {
+          throw new UnrecoverableError(`Model download cancelled: ${error.message ?? error}`)
+        }
+      }
+      throw error
+    } finally {
+      if (cancelPollInterval !== null) {
+        clearInterval(cancelPollInterval)
+        cancelPollInterval = null
+      }
+      DownloadModelJob.abortControllers.delete(job.id!)
     }
   }
 
   static async getByModelName(modelName: string): Promise<Job | undefined> {
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const queue = queueService.getQueue(this.queue)
     const jobId = this.getJobId(modelName)
     return await queue.getJob(jobId)
   }
 
   static async dispatch(params: DownloadModelJobParams) {
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const queue = queueService.getQueue(this.queue)
     const jobId = this.getJobId(params.modelName)
 
+    // Clear any previous failed job so a fresh attempt can be dispatched
+    const existing = await queue.getJob(jobId)
+    if (existing) {
+      const state = await existing.getState()
+      if (state === 'failed') {
+        await existing.remove()
+      }
+    }
+
     try {
       const job = await queue.add(this.key, params, {
         jobId,
@@ -121,9 +193,9 @@ export class DownloadModelJob {
       }
     } catch (error) {
       if (error.message.includes('job already exists')) {
-        const existing = await queue.getJob(jobId)
+        const active = await queue.getJob(jobId)
         return {
-          job: existing,
+          job: active,
           created: false,
           message: `Job already exists for model ${params.modelName}`,
         }

admin/app/jobs/embed_file_job.ts

@@ -0,0 +1,526 @@
+import { Job, UnrecoverableError } from 'bullmq'
+import { QueueService } from '#services/queue_service'
+import { EmbedJobWithProgress } from '../../types/rag.js'
+import { RagService } from '#services/rag_service'
+import { DockerService } from '#services/docker_service'
+import { OllamaService } from '#services/ollama_service'
+import KbIngestState from '#models/kb_ingest_state'
+import { createHash } from 'crypto'
+import logger from '@adonisjs/core/services/logger'
+import fs from 'node:fs/promises'
+import { ZIM_BATCH_SIZE } from '../../constants/zim_extraction.js'
+
+export interface EmbedFileJobParams {
+  filePath: string
+  fileName: string
+  fileSize?: number
+  // Batch processing for large ZIM files
+  batchOffset?: number  // Current batch offset (for ZIM files)
+  totalArticles?: number // Total articles in ZIM (for progress tracking)
+  isFinalBatch?: boolean // Whether this is the last batch (prevents premature deletion)
+  // Running total of chunks embedded across prior batches in this dispatch chain.
+  // Carried forward so the final batch can persist an accurate `chunks_embedded`
+  // count via KbIngestState.markIndexed (see #933 — without this, only the last
+  // batch's chunk count was stored while Qdrant held the full set).
+  chunksSoFar?: number
+}
+
+export class EmbedFileJob {
+  static get queue() {
+    return 'file-embeddings'
+  }
+
+  static get key() {
+    return 'embed-file'
+  }
+
+  // Delay between continuation batches when embedding runs CPU-only. Gives the OS
+  // scheduler a brief idle window so sshd / disk-collector / other services don't
+  // starve during long multi-batch ZIM ingestions. Skipped entirely when the
+  // embedding model is GPU-offloaded — see OllamaService.isEmbeddingGpuAccelerated().
+  static readonly CPU_BATCH_DELAY_MS = 1000
+
+  static getJobId(filePath: string): string {
+    return createHash('sha256').update(filePath).digest('hex').slice(0, 16)
+  }
+
+  /** Calls job.updateProgress but silently ignores "Missing key" errors (code -1),
+   *  which occur when the job has been removed from Redis (e.g. cancelled externally)
+   *  between the time the await was issued and the Redis write completed. */
+  private async safeUpdateProgress(job: Job, progress: number): Promise<void> {
+    try {
+      await job.updateProgress(progress)
+    } catch (err: any) {
+      if (err?.code !== -1) throw err
+    }
+  }
+
+  async handle(job: Job) {
+    const { filePath, fileName, batchOffset, totalArticles } = job.data as EmbedFileJobParams
+
+    const isZimBatch = batchOffset !== undefined
+    const batchInfo = isZimBatch ? ` (batch offset: ${batchOffset})` : ''
+    logger.info(`[EmbedFileJob] Starting embedding process for: ${fileName}${batchInfo}`)
+
+    const dockerService = new DockerService()
+    const ollamaService = new OllamaService()
+    const ragService = new RagService(dockerService, ollamaService)
+
+    try {
+      // Check if Ollama and Qdrant services are installed and ready
+      // Use UnrecoverableError for "not installed" so BullMQ won't retry —
+      // retrying 30x when the service doesn't exist just wastes Redis connections
+      const ollamaUrl = await dockerService.getServiceURL('nomad_ollama')
+      if (!ollamaUrl) {
+        logger.warn('[EmbedFileJob] Ollama is not installed. Skipping embedding for: %s', fileName)
+        throw new UnrecoverableError('Ollama service is not installed. Install AI Assistant to enable file embeddings.')
+      }
+
+      const existingModels = await ollamaService.getModels()
+      if (!existingModels) {
+        logger.warn('[EmbedFileJob] Ollama service not ready yet. Will retry...')
+        throw new Error('Ollama service not ready yet')
+      }
+
+      const qdrantUrl = await dockerService.getServiceURL('nomad_qdrant')
+      if (!qdrantUrl) {
+        logger.warn('[EmbedFileJob] Qdrant is not installed. Skipping embedding for: %s', fileName)
+        throw new UnrecoverableError('Qdrant service is not installed. Install AI Assistant to enable file embeddings.')
+      }
+
+      logger.info(`[EmbedFileJob] Services ready. Processing file: ${fileName}`)
+
+      // Anchor initial progress to where we are in the overall file. For a
+      // continuation batch midway through a multi-batch ZIM (e.g. offset 100k of
+      // 600k), the hardcoded 5 used to make the gauge briefly flash 0→5→real,
+      // which read as a backward jump. Fall back to 5 for single-batch files
+      // where totalArticles isn't set.
+      const initialPercent =
+        totalArticles && totalArticles > 0
+          ? Math.min(99, Math.round(((batchOffset || 0) / totalArticles) * 100))
+          : 5
+      await this.safeUpdateProgress(job, initialPercent)
+      await job.updateData({
+        ...job.data,
+        status: 'processing',
+        startedAt: job.data.startedAt || Date.now(),
+      })
+
+      logger.info(`[EmbedFileJob] Processing file: ${filePath}`)
+
+      // Progress callback. For multi-batch ZIM ingestions, scale the service-reported
+      // 0-100% (which is % through the current batch's chunks) into the overall-file
+      // frame so the UI gauge climbs monotonically across the many continuation jobs
+      // BullMQ creates per file. Without this, every new continuation jobId resets the
+      // gauge to ~5% and the user sees ingestion progress "jumping around" between
+      // each batch's local frame and the end-of-batch overall-file overwrite below.
+      //
+      // For single-batch files (uploaded PDFs, txts) totalArticles is undefined and
+      // we fall back to the original 5-95% per-job range, which is what the UI expects
+      // for a one-shot file with no continuations.
+      const onProgress = async (percent: number) => {
+        const useOverallFrame = totalArticles && totalArticles > 0
+        if (useOverallFrame) {
+          const articlesDone = (batchOffset || 0) + (percent / 100) * ZIM_BATCH_SIZE
+          const overallPercent = Math.min(99, Math.round((articlesDone / totalArticles) * 100))
+          await this.safeUpdateProgress(job, overallPercent)
+        } else {
+          await this.safeUpdateProgress(job, Math.min(95, Math.round(5 + percent * 0.9)))
+        }
+      }
+
+      // Process and embed the file
+      // Only allow deletion if explicitly marked as final batch
+      const allowDeletion = job.data.isFinalBatch === true
+      const result = await ragService.processAndEmbedFile(
+        filePath,
+        allowDeletion,
+        batchOffset,
+        onProgress
+      )
+
+      if (!result.success) {
+        logger.error(`[EmbedFileJob] Failed to process file ${fileName}: ${result.message}`)
+        throw new Error(result.message)
+      }
+
+      // For ZIM files with batching, check if more batches are needed
+      if (result.hasMoreBatches) {
+        const nextOffset = (batchOffset || 0) + (result.articlesProcessed || 0)
+        logger.info(
+          `[EmbedFileJob] Batch complete. Dispatching next batch at offset ${nextOffset}`
+        )
+
+        // Pace continuation batches when embedding is CPU-bound. Sustained 100% CPU
+        // saturation across all cores during multi-batch ZIM ingestion can starve
+        // other services (sshd has been seen to lose responsiveness hard enough to
+        // require a power-cycle). When GPU-accelerated, embeddings stream through
+        // the GPU and CPUs stay free — no pacing needed.
+        const isGpuAccelerated = await ollamaService.isEmbeddingGpuAccelerated()
+        if (!isGpuAccelerated) {
+          logger.info(
+            `[EmbedFileJob] Embedding is CPU-only — pacing ${EmbedFileJob.CPU_BATCH_DELAY_MS}ms before dispatching next batch`
+          )
+          await new Promise((resolve) => setTimeout(resolve, EmbedFileJob.CPU_BATCH_DELAY_MS))
+        }
+
+        // Bail before re-populating the queue if this job was cancelled mid-batch.
+        // cancelAllJobs() obliterates the queue (including this active job), but a
+        // worker already inside handle() would otherwise dispatch its continuation
+        // afterwards and silently revive a cancelled ZIM ingestion. If our own job
+        // key is gone, the cancel happened — skip the dispatch. Mirrors the
+        // "tolerate external removal" handling in safeUpdateProgress above.
+        const stillQueued = await QueueService.getInstance()
+          .getQueue(EmbedFileJob.queue)
+          .getJob(job.id!)
+        if (!stillQueued) {
+          logger.info(
+            `[EmbedFileJob] Job ${fileName} was cancelled; skipping continuation dispatch`
+          )
+          return { success: false, cancelled: true, fileName, filePath }
+        }
+
+        // Dispatch next batch (not final yet). Carry forward the running
+        // chunk count so the final batch can persist an accurate total (#933).
+        const chunksSoFarNext = (job.data.chunksSoFar || 0) + (result.chunks || 0)
+        await EmbedFileJob.dispatch({
+          filePath,
+          fileName,
+          batchOffset: nextOffset,
+          totalArticles: totalArticles || result.totalArticles,
+          isFinalBatch: false, // Explicitly not final
+          chunksSoFar: chunksSoFarNext,
+        })
+
+        // Calculate progress based on articles processed.
+        //
+        // nextOffset counts entries passing our isArticleEntry() filter, but the
+        // denominator (totalArticles = archive.articleCount) uses libzim's
+        // narrower article definition. On ZIMs that pack one logical article as
+        // several sub-pages (e.g. iFixit), nextOffset outruns articleCount and a
+        // raw ratio overflows past 100%, which the UI pins at 99% for the entire
+        // tail so the file looks stuck (#903). Grow the denominator once we pass
+        // the reported count so the gauge keeps creeping forward monotonically,
+        // and never report 100% before the genuinely-final batch (handled below).
+        const progress = totalArticles
+          ? Math.min(99, Math.round((nextOffset / Math.max(totalArticles, nextOffset + ZIM_BATCH_SIZE)) * 100))
+          : 50
+
+        await this.safeUpdateProgress(job, progress)
+        await job.updateData({
+          ...job.data,
+          status: 'batch_completed',
+          lastBatchAt: Date.now(),
+          chunks: chunksSoFarNext,
+        })
+
+        return {
+          success: true,
+          fileName,
+          filePath,
+          chunks: result.chunks,
+          hasMoreBatches: true,
+          nextOffset,
+          message: `Batch embedded ${result.chunks} chunks, next batch queued`,
+        }
+      }
+
+      // Final batch or non-batched file - mark as complete.
+      // chunksSoFar carries the accumulated count from prior dispatched batches
+      // (each continuation passes it forward — see EmbedFileJobParams). For a
+      // non-batched file it is undefined and we just count this single result.
+      const totalChunks = (job.data.chunksSoFar || 0) + (result.chunks || 0)
+      await this.safeUpdateProgress(job, 100)
+      await job.updateData({
+        ...job.data,
+        status: 'completed',
+        completedAt: Date.now(),
+        chunks: totalChunks,
+      })
+
+      // Persist the post-job state so scanAndSyncStorage knows this file is done.
+      // BullMQ's :completed retention (50 jobs) ages out, so the state row is
+      // the only durable record of "this file finished embedding".
+      try {
+        await KbIngestState.markIndexed(filePath, totalChunks)
+      } catch (stateErr) {
+        logger.warn(
+          `[EmbedFileJob] Failed to persist ingest state for ${fileName}: %s`,
+          stateErr instanceof Error ? stateErr.message : String(stateErr)
+        )
+      }
+
+      const batchMsg = isZimBatch ? ` (final batch, total chunks: ${totalChunks})` : ''
+      logger.info(
+        `[EmbedFileJob] Successfully embedded ${result.chunks} chunks from file: ${fileName}${batchMsg}`
+      )
+
+      return {
+        success: true,
+        fileName,
+        filePath,
+        chunks: result.chunks,
+        message: `Successfully embedded ${result.chunks} chunks`,
+      }
+    } catch (error) {
+      // A chunk that still exceeds the model's context after OllamaService's truncate-and-retry is
+      // permanently oversized for this install (e.g. a model whose context is smaller than our safe
+      // cap). Re-embedding the whole file 30x re-processes everything and can never succeed — that is
+      // the "endless queue loop" / "api/embed for weeks" (#881/#944/#959). Mark it unrecoverable so
+      // BullMQ stops after one pass instead of storming.
+      let normalizedError = error
+      if (!(error instanceof UnrecoverableError) && OllamaService.isContextLengthError(error)) {
+        logger.warn(
+          `[EmbedFileJob] Context-length overflow persisted for ${fileName} after truncation; not retrying.`
+        )
+        normalizedError = new UnrecoverableError(
+          error instanceof Error ? error.message : 'Embedding input exceeds the model context length'
+        )
+      }
+
+      logger.error(`[EmbedFileJob] Error embedding file ${fileName}:`, normalizedError)
+
+      await job.updateData({
+        ...job.data,
+        status: 'failed',
+        failedAt: Date.now(),
+        error: normalizedError instanceof Error ? normalizedError.message : 'Unknown error',
+      })
+
+      // Only persist `failed` for unrecoverable errors. Retryable errors get
+      // automatic BullMQ retries (30 attempts); marking state failed on every
+      // transient blip would suppress the retry-driven recovery path.
+      if (normalizedError instanceof UnrecoverableError) {
+        try {
+          await KbIngestState.markFailed(
+            filePath,
+            normalizedError instanceof Error ? normalizedError.message : 'Unknown error'
+          )
+        } catch (stateErr) {
+          logger.warn(
+            `[EmbedFileJob] Failed to persist failed state for ${fileName}: %s`,
+            stateErr instanceof Error ? stateErr.message : String(stateErr)
+          )
+        }
+      }
+
+      throw normalizedError
+    }
+  }
+
+  static async listActiveJobs(): Promise<EmbedJobWithProgress[]> {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+    const jobs = await queue.getJobs(['waiting', 'active', 'delayed'])
+
+    return jobs.map((job) => {
+      const data = job.data as EmbedFileJobParams & {
+        status?: string
+        lastBatchAt?: number
+        startedAt?: number
+        chunks?: number
+      }
+      return {
+        jobId: job.id!.toString(),
+        fileName: data.fileName,
+        filePath: data.filePath,
+        progress: typeof job.progress === 'number' ? job.progress : 0,
+        status: data.status ?? 'waiting',
+        lastBatchAt: data.lastBatchAt,
+        startedAt: data.startedAt,
+        chunks: data.chunks,
+      }
+    })
+  }
+
+  static async getByFilePath(filePath: string): Promise<Job | undefined> {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+    const jobId = this.getJobId(filePath)
+    return await queue.getJob(jobId)
+  }
+
+  static async dispatch(params: EmbedFileJobParams, options?: { force?: boolean }) {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+
+    // Continuation batches (batchOffset > 0) must NOT reuse the deterministic
+    // per-file jobId. Two BullMQ dedupe paths would otherwise silently swallow them:
+    //   1) The parent batch's handle() calls dispatch() before returning, so the
+    //      parent job is still `active` and locked — queue.add() with the same
+    //      jobId returns the locked parent rather than enqueueing the new batch.
+    //   2) After the parent completes, its entry stays in `completed` (held by
+    //      `removeOnComplete: { count: 50 }`), still tripping jobId dedupe.
+    // Letting BullMQ auto-generate a unique jobId for continuation batches stacks
+    // them as independent queue entries that each process via handle().
+    // Initial dispatches keep the deterministic jobId so re-triggering an install
+    // (UI re-click, sync rescan, etc.) is still idempotent.
+    // `force` skips the deterministic jobId for bulk callers (reembedAll /
+    // resetAndRebuild) where historical entries in :completed would otherwise
+    // silently swallow the new dispatch.
+    const isContinuation = !!(params.batchOffset && params.batchOffset > 0)
+    const force = !!options?.force
+    const initialJobId = this.getJobId(params.filePath)
+
+    const jobOptions: Parameters<typeof queue.add>[2] = {
+      attempts: 30,
+      backoff: {
+        type: 'fixed',
+        delay: 60000, // Check every 60 seconds for service readiness
+      },
+      removeOnComplete: { count: 50 }, // Keep last 50 completed jobs for history
+      removeOnFail: { count: 20 }, // Keep last 20 failed jobs for debugging
+    }
+    if (!isContinuation && !force) {
+      jobOptions.jobId = initialJobId
+    }
+
+    try {
+      const job = await queue.add(this.key, params, jobOptions)
+
+      const label = isContinuation
+        ? ` (continuation @ offset ${params.batchOffset})`
+        : force
+          ? ' (forced re-dispatch)'
+          : ''
+      logger.info(
+        `[EmbedFileJob] Dispatched embedding job for file: ${params.fileName}${label}`
+      )
+
+      return {
+        job,
+        created: true,
+        jobId: job.id ?? initialJobId,
+        message: `File queued for embedding: ${params.fileName}`,
+      }
+    } catch (error) {
+      if (
+        !isContinuation &&
+        !force &&
+        error.message &&
+        error.message.includes('job already exists')
+      ) {
+        const existing = await queue.getJob(initialJobId)
+        logger.info(`[EmbedFileJob] Job already exists for file: ${params.fileName}`)
+        return {
+          job: existing,
+          created: false,
+          jobId: initialJobId,
+          message: `Embedding job already exists for: ${params.fileName}`,
+        }
+      }
+      throw error
+    }
+  }
+
+  static async listFailedJobs(): Promise<EmbedJobWithProgress[]> {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+    // Jobs that have failed at least once are in 'delayed' (retrying) or terminal 'failed' state.
+    // We identify them by job.data.status === 'failed' set in the catch block of handle().
+    const jobs = await queue.getJobs(['waiting', 'delayed', 'failed'])
+
+    return jobs
+      .filter((job) => (job.data as any).status === 'failed')
+      .map((job) => ({
+        jobId: job.id!.toString(),
+        fileName: (job.data as EmbedFileJobParams).fileName,
+        filePath: (job.data as EmbedFileJobParams).filePath,
+        progress: 0,
+        status: 'failed',
+        error: (job.data as any).error,
+      }))
+  }
+
+  static async cleanupFailedJobs(): Promise<{ cleaned: number; filesDeleted: number }> {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+    const allJobs = await queue.getJobs(['waiting', 'delayed', 'failed'])
+    const failedJobs = allJobs.filter((job) => (job.data as any).status === 'failed')
+
+    let cleaned = 0
+    let filesDeleted = 0
+
+    for (const job of failedJobs) {
+      const filePath = (job.data as EmbedFileJobParams).filePath
+      if (filePath && filePath.includes(RagService.UPLOADS_STORAGE_PATH)) {
+        try {
+          await fs.unlink(filePath)
+          filesDeleted++
+        } catch {
+          // File may already be deleted — that's fine
+        }
+      }
+      await job.remove()
+      cleaned++
+    }
+
+    logger.info(`[EmbedFileJob] Cleaned up ${cleaned} failed jobs, deleted ${filesDeleted} files`)
+    return { cleaned, filesDeleted }
+  }
+
+  /** Unconditionally clear every embedding job regardless of state.
+   *
+   *  cleanupFailedJobs only removes jobs explicitly tagged status === 'failed',
+   *  which leaves stuck jobs (waiting / active / delayed / paused that never
+   *  reached 'failed') unreachable from the UI — the operator's only recourse was
+   *  flushing Redis by hand. This wipes the whole queue, including a locked active
+   *  job, via obliterate({ force: true }) (plain obliterate/job.remove throw on a
+   *  locked job). It touches only Redis, so it is safe while Qdrant/Ollama are
+   *  offline — which is exactly when jobs pile up and wedge. */
+  static async cancelAllJobs(): Promise<{ cancelled: number; filesDeleted: number }> {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+    const jobs = await queue.getJobs(['waiting', 'active', 'delayed', 'paused', 'failed'])
+
+    let filesDeleted = 0
+    for (const job of jobs) {
+      const filePath = (job.data as EmbedFileJobParams).filePath
+      // Same guard as cleanupFailedJobs: only delete user uploads, never ZIM
+      // library files or Nomad docs that live outside the uploads path.
+      if (filePath && filePath.includes(RagService.UPLOADS_STORAGE_PATH)) {
+        try {
+          await fs.unlink(filePath)
+          filesDeleted++
+        } catch {
+          // File may already be deleted — that's fine
+        }
+      }
+    }
+
+    const cancelled = jobs.length
+
+    // force: true removes the locked/active job too. An in-flight worker may keep
+    // running its current batch in memory; the self-exists guard in handle()
+    // prevents it from dispatching a continuation back into the cleared queue.
+    await queue.obliterate({ force: true })
+
+    logger.info(`[EmbedFileJob] Cancelled ${cancelled} jobs, deleted ${filesDeleted} files`)
+    return { cancelled, filesDeleted }
+  }
+
+  static async getStatus(filePath: string): Promise<{
+    exists: boolean
+    status?: string
+    progress?: number
+    chunks?: number
+    error?: string
+  }> {
+    const job = await this.getByFilePath(filePath)
+
+    if (!job) {
+      return { exists: false }
+    }
+
+    const state = await job.getState()
+    const data = job.data
+
+    return {
+      exists: true,
+      status: data.status || state,
+      progress: typeof job.progress === 'number' ? job.progress : undefined,
+      chunks: data.chunks,
+      error: data.error,
+    }
+  }
+}

admin/app/jobs/run_benchmark_job.ts

@@ -0,0 +1,101 @@
+import { Job } from 'bullmq'
+import { QueueService } from '#services/queue_service'
+import { BenchmarkService } from '#services/benchmark_service'
+import type { RunBenchmarkJobParams } from '../../types/benchmark.js'
+import logger from '@adonisjs/core/services/logger'
+import { DockerService } from '#services/docker_service'
+
+export class RunBenchmarkJob {
+  static get queue() {
+    return 'benchmarks'
+  }
+
+  static get key() {
+    return 'run-benchmark'
+  }
+
+  async handle(job: Job) {
+    const { benchmark_id, benchmark_type } = job.data as RunBenchmarkJobParams
+
+    logger.info(`[RunBenchmarkJob] Starting benchmark ${benchmark_id} of type ${benchmark_type}`)
+
+    const dockerService = new DockerService()
+    const benchmarkService = new BenchmarkService(dockerService)
+
+    try {
+      let result
+
+      switch (benchmark_type) {
+        case 'full':
+          result = await benchmarkService.runFullBenchmark()
+          break
+        case 'system':
+          result = await benchmarkService.runSystemBenchmarks()
+          break
+        case 'ai':
+          result = await benchmarkService.runAIBenchmark()
+          break
+        default:
+          throw new Error(`Unknown benchmark type: ${benchmark_type}`)
+      }
+
+      logger.info(`[RunBenchmarkJob] Benchmark ${benchmark_id} completed with NOMAD score: ${result.nomad_score}`)
+
+      return {
+        success: true,
+        benchmark_id: result.benchmark_id,
+        nomad_score: result.nomad_score,
+      }
+    } catch (error) {
+      logger.error(`[RunBenchmarkJob] Benchmark ${benchmark_id} failed: ${error.message}`)
+      throw error
+    }
+  }
+
+  static async dispatch(params: RunBenchmarkJobParams) {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+
+    try {
+      const job = await queue.add(this.key, params, {
+        jobId: params.benchmark_id,
+        attempts: 1, // Benchmarks shouldn't be retried automatically
+        removeOnComplete: {
+          count: 10, // Keep last 10 completed jobs
+        },
+        removeOnFail: {
+          count: 5, // Keep last 5 failed jobs
+        },
+      })
+
+      logger.info(`[RunBenchmarkJob] Dispatched benchmark job ${params.benchmark_id}`)
+
+      return {
+        job,
+        created: true,
+        message: `Benchmark job ${params.benchmark_id} dispatched successfully`,
+      }
+    } catch (error) {
+      if (error.message.includes('job already exists')) {
+        const existing = await queue.getJob(params.benchmark_id)
+        return {
+          job: existing,
+          created: false,
+          message: `Benchmark job ${params.benchmark_id} already exists`,
+        }
+      }
+      throw error
+    }
+  }
+
+  static async getJob(benchmarkId: string): Promise<Job | undefined> {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+    return await queue.getJob(benchmarkId)
+  }
+
+  static async getJobState(benchmarkId: string): Promise<string | undefined> {
+    const job = await this.getJob(benchmarkId)
+    return job ? await job.getState() : undefined
+  }
+}

admin/app/jobs/run_download_job.ts

@@ -1,11 +1,41 @@
-import { Job } from 'bullmq'
-import { RunDownloadJobParams } from '../../types/downloads.js'
+import { Job, UnrecoverableError } from 'bullmq'
+import { RunDownloadJobParams, DownloadProgressData } from '../../types/downloads.js'
 import { QueueService } from '#services/queue_service'
 import { doResumableDownload } from '../utils/downloads.js'
 import { createHash } from 'crypto'
 import { DockerService } from '#services/docker_service'
 import { ZimService } from '#services/zim_service'
 import { MapService } from '#services/map_service'
+import { RagService } from '#services/rag_service'
+import { OllamaService } from '#services/ollama_service'
+import { EmbedFileJob } from './embed_file_job.js'
+import { basename, join, resolve, sep } from 'node:path'
+import { ZIM_STORAGE_PATH } from '../utils/fs.js'
+
+/** Maps live under `<cwd>/storage/maps/pmtiles`; no shared constant exists. */
+const MAP_STORAGE_PATH = '/storage/maps'
+
+/**
+ * Guard for the outdated-file deletion in {@link RunDownloadJob} `onComplete`:
+ * returns true only when `oldFilePath` sits under the expected content storage
+ * root for its type AND its filename carries this resource's id prefix. This
+ * makes the delete explicit and bounded — we only ever remove the replaced
+ * resource's own previous file, never another file, even if the
+ * InstalledResource row is stale or malformed.
+ */
+function isSafeOldContentPath(
+  oldFilePath: string,
+  resourceId: string,
+  filetype: string
+): boolean {
+  const root =
+    filetype === 'zim'
+      ? join(process.cwd(), ZIM_STORAGE_PATH)
+      : join(process.cwd(), MAP_STORAGE_PATH)
+  const resolved = resolve(oldFilePath)
+  if (!resolved.startsWith(root + sep)) return false
+  return basename(resolved).startsWith(`${resourceId}_`)
+}
 
 export class RunDownloadJob {
   static get queue() {
@@ -16,78 +46,323 @@ export class RunDownloadJob {
     return 'run-download'
   }
 
+  /** In-memory registry of abort controllers for active download jobs */
+  static abortControllers: Map<string, AbortController> = new Map()
+
   static getJobId(url: string): string {
     return createHash('sha256').update(url).digest('hex').slice(0, 16)
   }
 
+  /** Redis key used to signal cancellation across processes */
+  static cancelKey(jobId: string): string {
+    return `nomad:download:cancel:${jobId}`
+  }
+
+  /** Signal cancellation via Redis so the worker process can pick it up */
+  static async signalCancel(jobId: string): Promise<void> {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+    const client = await queue.client
+    await client.set(this.cancelKey(jobId), '1', { EX: 300 }) // 5 min TTL
+  }
+
   async handle(job: Job) {
-    const { url, filepath, timeout, allowedMimeTypes, forceNew, filetype } =
+    const { url, filepath, timeout, allowedMimeTypes, forceNew, filetype, resourceMetadata } =
       job.data as RunDownloadJobParams
 
-    //    console.log("Simulating delay for job for URL:", url)
-    //  await new Promise((resolve) => setTimeout(resolve, 30000)) // Simulate initial delay
-    //  console.log("Starting download for URL:", url)
+    // Register abort controller for this job
+    const abortController = new AbortController()
+    RunDownloadJob.abortControllers.set(job.id!, abortController)
 
-    // // simulate progress updates for demonstration
-    // for (let progress = 0; progress <= 100; progress += 10) {
-    //   await new Promise((resolve) => setTimeout(resolve, 20000)) // Simulate time taken for each progress step
-    //   job.updateProgress(progress)
-    //   console.log(`Job progress for URL ${url}: ${progress}%`)
-    // }
+    // Get Redis client for checking cancel signals from the API process
+    const queueService = QueueService.getInstance()
+    const cancelRedis = await queueService.getQueue(RunDownloadJob.queue).client
 
-    await doResumableDownload({
-      url,
-      filepath,
-      timeout,
-      allowedMimeTypes,
-      forceNew,
-      onProgress(progress) {
-        const progressPercent = (progress.downloadedBytes / (progress.totalBytes || 1)) * 100
-        job.updateProgress(Math.floor(progressPercent))
-      },
-      async onComplete(url) {
-        try {
-          if (filetype === 'zim') {
-            const dockerService = new DockerService()
-            const zimService = new ZimService(dockerService)
-            await zimService.downloadRemoteSuccessCallback([url], true)
-          } else if (filetype === 'map') {
-            const mapsService = new MapService()
-            await mapsService.downloadRemoteSuccessCallback([url], false)
-          }
-        } catch (error) {
-          console.error(
-            `[RunDownloadJob] Error in ZIM download success callback for URL ${url}:`,
-            error
-          )
+    let lastKnownProgress: Pick<DownloadProgressData, 'downloadedBytes' | 'totalBytes'> = {
+      downloadedBytes: 0,
+      totalBytes: 0,
+    }
+
+    // Track whether cancellation was explicitly requested by the user (via Redis signal
+    // or in-process AbortController). BullMQ lock mismatches can also abort the download
+    // stream, but those should be retried — only user-initiated cancels are unrecoverable.
+    let userCancelled = false
+
+    // Poll Redis for cancel signal every 2s — independent of progress events so cancellation
+    // works even when the stream is stalled and no onProgress ticks are firing.
+    let cancelPollInterval: ReturnType<typeof setInterval> | null = setInterval(async () => {
+      try {
+        const val = await cancelRedis.get(RunDownloadJob.cancelKey(job.id!))
+        if (val) {
+          await cancelRedis.del(RunDownloadJob.cancelKey(job.id!))
+          userCancelled = true
+          abortController.abort('user-cancel')
         }
-        job.updateProgress(100)
-      },
-    })
+      } catch {
+        // Redis errors are non-fatal; in-process AbortController covers same-process cancels
+      }
+    }, 2000)
 
-    return {
-      url,
-      filepath,
+    try {
+      await doResumableDownload({
+        url,
+        filepath,
+        timeout,
+        allowedMimeTypes,
+        forceNew,
+        signal: abortController.signal,
+        onProgress(progress) {
+          const progressPercent = (progress.downloadedBytes / (progress.totalBytes || 1)) * 100
+          const progressData: DownloadProgressData = {
+            percent: Math.floor(progressPercent),
+            downloadedBytes: progress.downloadedBytes,
+            totalBytes: progress.totalBytes,
+            lastProgressTime: Date.now(),
+          }
+          job.updateProgress(progressData).catch((err) => {
+            // Job was removed from Redis (e.g. cancelled) between the callback firing
+            // and the Redis write completing — this is expected and safe to ignore.
+            if (err?.code !== -1) throw err
+          })
+          lastKnownProgress = { downloadedBytes: progress.downloadedBytes, totalBytes: progress.totalBytes }
+        },
+        async onComplete(url) {
+          // The previous file recorded for this resource (if any). Hoisted out of
+          // the metadata block below so the ZIM branch can decide whether this
+          // download is a content UPDATE (replacing a prior file) vs a fresh
+          // install, which changes how we reconcile the knowledge base.
+          let oldFilePath: string | null = null
+          try {
+            // Create InstalledResource entry if metadata was provided
+            if (resourceMetadata) {
+              const { default: InstalledResource } = await import('#models/installed_resource')
+              const { DateTime } = await import('luxon')
+              const { getFileStatsIfExists, deleteFileIfExists } = await import('../utils/fs.js')
+              const stats = await getFileStatsIfExists(filepath)
+
+              // Look up the old entry so we can clean up the previous file after updating
+              const oldEntry = await InstalledResource.query()
+                .where('resource_id', resourceMetadata.resource_id)
+                .where('resource_type', filetype as 'zim' | 'map')
+                .first()
+              oldFilePath = oldEntry?.file_path ?? null
+
+              const installed = await InstalledResource.updateOrCreate(
+                { resource_id: resourceMetadata.resource_id, resource_type: filetype as 'zim' | 'map' },
+                {
+                  version: resourceMetadata.version,
+                  collection_ref: resourceMetadata.collection_ref,
+                  url: url,
+                  file_path: filepath,
+                  file_size_bytes: stats ? Number(stats.size) : null,
+                  installed_at: DateTime.now(),
+                }
+              )
+
+              // A completed auto-update is the authoritative success signal for the
+              // per-resource backoff — clear it here (NOT at dispatch time, which
+              // would reset the counter every window and defeat self-disable). The
+              // matching terminal-failure increment lives in the worker `failed`
+              // handler (commands/queue/work.ts). Manual downloads (auto !== true)
+              // never touch the counter.
+              if (resourceMetadata.auto === true) {
+                try {
+                  const { recordResourceUpdateSuccess } = await import(
+                    '../utils/content_auto_update_backoff.js'
+                  )
+                  await recordResourceUpdateSuccess(installed)
+                } catch (error) {
+                  console.error(
+                    `[RunDownloadJob] Error clearing auto-update backoff for ${resourceMetadata.resource_id}:`,
+                    error
+                  )
+                }
+              }
+
+              // Step 1: delete the OUTDATED file if it differs from the new one.
+              // Guarded by isSafeOldContentPath so we can ONLY ever delete the
+              // replaced resource's own previous file — never another resource's
+              // file, even if the InstalledResource row is stale/malformed.
+              if (oldFilePath && oldFilePath !== filepath) {
+                if (isSafeOldContentPath(oldFilePath, resourceMetadata.resource_id, filetype)) {
+                  try {
+                    await deleteFileIfExists(oldFilePath)
+                    console.log(`[RunDownloadJob] Deleted old file: ${oldFilePath}`)
+                  } catch (deleteError) {
+                    console.warn(
+                      `[RunDownloadJob] Failed to delete old file ${oldFilePath}:`,
+                      deleteError
+                    )
+                  }
+                } else {
+                  console.warn(
+                    `[RunDownloadJob] Refusing to delete unexpected old file path for ` +
+                      `${resourceMetadata.resource_id} (${filetype}): ${oldFilePath}`
+                  )
+                }
+              }
+            }
+
+            if (filetype === 'zim') {
+              const dockerService = new DockerService()
+              const zimService = new ZimService(dockerService)
+              await zimService.downloadRemoteSuccessCallback([url], true)
+
+              // Only touch the knowledge base if AI Assistant (Ollama) is installed
+              const ollamaUrl = await dockerService.getServiceURL('nomad_ollama')
+              if (ollamaUrl) {
+                // A content UPDATE replaces a prior file at a DIFFERENT path
+                // (version is in the filename). A fresh install has no prior row;
+                // a same-version re-download keeps the same path. The two cases
+                // reconcile the KB differently.
+                const isReplacement = !!oldFilePath && oldFilePath !== filepath
+
+                if (isReplacement) {
+                  // CONTENT UPDATE: mirror the REPLACED file's prior indexed state
+                  // rather than the global Always/Manual policy. reconcileReplaced-
+                  // ContentFile removes the old file's points and re-queues the new
+                  // file IFF the old one was indexed and Qdrant is running; it is a
+                  // no-op otherwise (not installed / old not indexed / Qdrant down).
+                  // The user already chose whether this content is in the KB, so we
+                  // honor that choice in both directions. See the method for the
+                  // full 5-step contract.
+                  try {
+                    const ragService = new RagService(dockerService, new OllamaService())
+                    const outcome = await ragService.reconcileReplacedContentFile({
+                      oldFilePath: oldFilePath!,
+                      newFilePath: filepath,
+                      fileName: url.split('/').pop() || '',
+                    })
+                    console.log(
+                      `[RunDownloadJob] KB reconciliation for replaced ${filepath}: ${outcome}`
+                    )
+                  } catch (error) {
+                    console.error(
+                      `[RunDownloadJob] Error reconciling knowledge base for replaced file ${filepath}:`,
+                      error
+                    )
+                  }
+                } else {
+                  // FRESH INSTALL (or same-version re-download): respect the global
+                  // ingest policy. Under Manual, record the file as pending_decision
+                  // so the KB panel surfaces the per-file Index affordance (PR #909)
+                  // instead of silently auto-embedding behind the user's back. Unset
+                  // is treated as Always to preserve legacy behavior — mirrors
+                  // rag_service.ts:1587-1588.
+                  const { default: KVStore } = await import('#models/kv_store')
+                  const { default: KbIngestState } = await import('#models/kb_ingest_state')
+                  const policyRaw = await KVStore.getValue('rag.defaultIngestPolicy')
+                  const policy: 'Always' | 'Manual' = policyRaw === 'Manual' ? 'Manual' : 'Always'
+
+                  if (policy === 'Manual') {
+                    try {
+                      // firstOrCreate so a re-download doesn't demote an existing
+                      // indexed/failed row — user keeps prior state and can re-index
+                      // explicitly from the KB panel if they want fresh content.
+                      await KbIngestState.firstOrCreate(
+                        { file_path: filepath },
+                        { file_path: filepath, state: 'pending_decision', chunks_embedded: 0 }
+                      )
+                    } catch (error) {
+                      console.error(
+                        `[RunDownloadJob] Error recording pending_decision state for ${filepath}:`,
+                        error
+                      )
+                    }
+                  } else {
+                    try {
+                      await EmbedFileJob.dispatch({
+                        fileName: url.split('/').pop() || '',
+                        filePath: filepath,
+                      })
+                    } catch (error) {
+                      console.error(`[RunDownloadJob] Error dispatching EmbedFileJob for URL ${url}:`, error)
+                    }
+                  }
+                }
+              }
+            } else if (filetype === 'map') {
+              const mapsService = new MapService()
+              await mapsService.downloadRemoteSuccessCallback([url], false)
+            }
+          } catch (error) {
+            console.error(
+              `[RunDownloadJob] Error in download success callback for URL ${url}:`,
+              error
+            )
+          }
+          job.updateProgress({
+            percent: 100,
+            downloadedBytes: lastKnownProgress.downloadedBytes,
+            totalBytes: lastKnownProgress.totalBytes,
+            lastProgressTime: Date.now(),
+          } as DownloadProgressData).catch((err) => {
+            if (err?.code !== -1) throw err
+          })
+        },
+      })
+
+      return {
+        url,
+        filepath,
+      }
+    } catch (error: any) {
+      // Only prevent retries for user-initiated cancellations. BullMQ lock mismatches
+      // can also abort the stream, and those should be retried with backoff.
+      // Check both the flag (Redis poll) and abort reason (in-process cancel).
+      if (userCancelled || abortController.signal.reason === 'user-cancel') {
+        throw new UnrecoverableError(`Download cancelled: ${error.message}`)
+      }
+      throw error
+    } finally {
+      if (cancelPollInterval !== null) {
+        clearInterval(cancelPollInterval)
+        cancelPollInterval = null
+      }
+      RunDownloadJob.abortControllers.delete(job.id!)
     }
   }
 
   static async getByUrl(url: string): Promise<Job | undefined> {
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const queue = queueService.getQueue(this.queue)
     const jobId = this.getJobId(url)
     return await queue.getJob(jobId)
   }
 
+  /**
+   * Check if a download is actively in progress for the given URL.
+   * Returns the job only if it's in an active state (active, waiting, delayed).
+   * If the job exists in a terminal state (failed, completed), removes it and returns undefined.
+   */
+  static async getActiveByUrl(url: string): Promise<Job | undefined> {
+    const job = await this.getByUrl(url)
+    if (!job) return undefined
+
+    const state = await job.getState()
+    if (state === 'active' || state === 'waiting' || state === 'delayed') {
+      return job
+    }
+
+    // Terminal state -- clean up stale job so it doesn't block re-download
+    try {
+      await job.remove()
+    } catch {
+      // May already be gone
+    }
+    return undefined
+  }
+
   static async dispatch(params: RunDownloadJobParams) {
-    const queueService = new QueueService()
+    const queueService = QueueService.getInstance()
     const queue = queueService.getQueue(this.queue)
     const jobId = this.getJobId(params.url)
 
     try {
       const job = await queue.add(this.key, params, {
         jobId,
-        attempts: 3,
-        backoff: { type: 'exponential', delay: 2000 },
+        attempts: 10,
+        backoff: { type: 'exponential', delay: 30000 },
         removeOnComplete: true,
       })
 

admin/app/jobs/run_extract_pmtiles_job.ts

@@ -0,0 +1,294 @@
+import { Job, UnrecoverableError } from 'bullmq'
+import { spawn, ChildProcess } from 'child_process'
+import { createHash } from 'crypto'
+import { readdir, stat } from 'fs/promises'
+import { basename, dirname, join } from 'path'
+import { QueueService } from '#services/queue_service'
+import logger from '@adonisjs/core/services/logger'
+import { DownloadProgressData } from '../../types/downloads.js'
+import { PMTILES_BINARY_PATH, buildPmtilesExtractArgs } from '../../constants/map_regions.js'
+import { deleteFileIfExists } from '../utils/fs.js'
+
+export interface RunExtractPmtilesJobParams {
+  sourceUrl: string
+  outputFilepath: string
+  /** Path to a GeoJSON FeatureCollection file passed to `pmtiles extract --region`. */
+  regionFilepath: string
+  maxzoom?: number
+  /** Hint for progress reporting; obtained from `pmtiles extract --dry-run` preflight */
+  estimatedBytes?: number
+  filetype: 'map'
+  title?: string
+  resourceMetadata?: {
+    resource_id: string
+    version: string
+    collection_ref: string | null
+  }
+}
+
+export class RunExtractPmtilesJob {
+  static get queue() {
+    return 'pmtiles-extract'
+  }
+
+  static get key() {
+    return 'run-pmtiles-extract'
+  }
+
+  /** In-memory registry of active child processes so in-process cancels can SIGTERM them */
+  static childProcesses: Map<string, ChildProcess> = new Map()
+
+  static getJobId(sourceUrl: string, regionFilepath: string, maxzoom?: number): string {
+    const payload = JSON.stringify({ sourceUrl, regionFilepath, maxzoom: maxzoom ?? null })
+    return createHash('sha256').update(payload).digest('hex').slice(0, 16)
+  }
+
+  /** Redis key used to signal cancellation across processes */
+  static cancelKey(jobId: string): string {
+    return `nomad:download:pmtiles-cancel:${jobId}`
+  }
+
+  static async signalCancel(jobId: string): Promise<void> {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+    const client = await queue.client
+    await client.set(this.cancelKey(jobId), '1', { EX: 300 })
+  }
+
+  /** Awaits job.updateProgress and swallows BullMQ stale-job errors (code -1),
+   *  which occur when the job was removed from Redis (e.g. cancelled) between
+   *  the await being issued and the Redis write completing. Anything else
+   *  re-throws so it's caught by the surrounding try rather than becoming an
+   *  unhandled rejection. */
+  private async safeUpdateProgress(job: Job, progress: DownloadProgressData): Promise<void> {
+    try {
+      await job.updateProgress(progress)
+    } catch (err: any) {
+      if (err?.code !== -1) throw err
+    }
+  }
+
+  async handle(job: Job) {
+    const params = job.data as RunExtractPmtilesJobParams
+    const { sourceUrl, outputFilepath, regionFilepath, maxzoom, estimatedBytes } = params
+
+    logger.info(
+      `[RunExtractPmtilesJob] Starting extract: source=${sourceUrl} region=${regionFilepath} ` +
+        `maxzoom=${maxzoom ?? 'source-max'} out=${outputFilepath}`
+    )
+
+    const queueService = QueueService.getInstance()
+    const cancelRedis = await queueService.getQueue(RunExtractPmtilesJob.queue).client
+
+    let userCancelled = false
+    let proc: ChildProcess | null = null
+    let lastReportedBytes = -1
+
+    // One 2s tick polls the Redis cancel signal and reads file-size for progress. pmtiles
+    // writes incrementally but rewrites directories near the end so progress isn't strictly
+    // monotonic — we cap at 99% and skip emit when bytes are unchanged to avoid Redis chatter.
+    const tick = setInterval(async () => {
+      try {
+        const val = await cancelRedis.get(RunExtractPmtilesJob.cancelKey(job.id!))
+        if (val) {
+          await cancelRedis.del(RunExtractPmtilesJob.cancelKey(job.id!))
+          userCancelled = true
+          proc?.kill('SIGTERM')
+        }
+      } catch {
+        // Redis errors non-fatal — in-memory handle also covers same-process cancels
+      }
+
+      try {
+        const fileStat = await stat(outputFilepath)
+        const downloadedBytes = Number(fileStat.size)
+        if (downloadedBytes === lastReportedBytes) return
+        lastReportedBytes = downloadedBytes
+
+        const totalBytes = estimatedBytes ?? 0
+        const percent =
+          totalBytes > 0 ? Math.min(99, Math.floor((downloadedBytes / totalBytes) * 100)) : 0
+
+        await this.safeUpdateProgress(job, {
+          percent,
+          downloadedBytes,
+          totalBytes,
+          lastProgressTime: Date.now(),
+        } as DownloadProgressData)
+      } catch {
+        // File doesn't exist yet (subprocess still setting up)
+      }
+    }, 2000)
+
+    try {
+      const args = buildPmtilesExtractArgs({
+        sourceUrl,
+        outputFilepath,
+        regionFilepath,
+        maxzoom,
+        downloadThreads: 8,
+        overfetch: 0.2,
+      })
+      proc = spawn(PMTILES_BINARY_PATH, args, { stdio: ['ignore', 'pipe', 'pipe'] })
+      RunExtractPmtilesJob.childProcesses.set(job.id!, proc)
+
+      proc.stdout?.on('data', (chunk) => {
+        logger.debug(`[RunExtractPmtilesJob:${job.id}] ${chunk.toString().trimEnd()}`)
+      })
+      proc.stderr?.on('data', (chunk) => {
+        logger.debug(`[RunExtractPmtilesJob:${job.id}] ${chunk.toString().trimEnd()}`)
+      })
+
+      const exitCode: number = await new Promise((resolve, reject) => {
+        proc!.on('close', (code) => resolve(code ?? -1))
+        proc!.on('error', (err) => reject(err))
+      })
+
+      if (exitCode !== 0) {
+        await deleteFileIfExists(outputFilepath)
+        if (userCancelled) {
+          throw new UnrecoverableError(`Extract cancelled by user (exit ${exitCode})`)
+        }
+        throw new Error(`pmtiles extract exited with code ${exitCode}`)
+      }
+
+      // Final progress bump — tick caps at 99 so the UI doesn't flicker to 100 mid-extract
+      const finalStat = await stat(outputFilepath)
+      await this.safeUpdateProgress(job, {
+        percent: 100,
+        downloadedBytes: Number(finalStat.size),
+        totalBytes: estimatedBytes ?? Number(finalStat.size),
+        lastProgressTime: Date.now(),
+      } as DownloadProgressData)
+
+      // Reuse the HTTP download path's post-download hook so the file is registered and
+      // the previous version (if any) is deleted
+      await this.onComplete(params)
+
+      logger.info(
+        `[RunExtractPmtilesJob] Completed extract: out=${outputFilepath} size=${finalStat.size} bytes`
+      )
+
+      return { sourceUrl, outputFilepath }
+    } catch (error: any) {
+      if (userCancelled && !(error instanceof UnrecoverableError)) {
+        throw new UnrecoverableError(`Extract cancelled: ${error.message ?? error}`)
+      }
+      throw error
+    } finally {
+      clearInterval(tick)
+      RunExtractPmtilesJob.childProcesses.delete(job.id!)
+    }
+  }
+
+  private async onComplete(params: RunExtractPmtilesJobParams) {
+    if (!params.resourceMetadata) return
+
+    const [{ default: InstalledResource }, { DateTime }, fsUtils] = await Promise.all([
+      import('#models/installed_resource'),
+      import('luxon'),
+      import('../utils/fs.js'),
+    ])
+
+    const fileStat = await fsUtils.getFileStatsIfExists(params.outputFilepath)
+
+    const existing = await InstalledResource.query()
+      .where('resource_id', params.resourceMetadata.resource_id)
+      .where('resource_type', 'map')
+      .first()
+    const oldFilePath = existing?.file_path ?? null
+
+    await InstalledResource.updateOrCreate(
+      {
+        resource_id: params.resourceMetadata.resource_id,
+        resource_type: 'map',
+      },
+      {
+        version: params.resourceMetadata.version,
+        collection_ref: params.resourceMetadata.collection_ref,
+        url: params.sourceUrl,
+        file_path: params.outputFilepath,
+        file_size_bytes: fileStat ? Number(fileStat.size) : null,
+        installed_at: DateTime.now(),
+      }
+    )
+
+    if (oldFilePath && oldFilePath !== params.outputFilepath) {
+      try {
+        await fsUtils.deleteFileIfExists(oldFilePath)
+      } catch (err) {
+        logger.warn(`[RunExtractPmtilesJob] Failed to delete old file ${oldFilePath}: ${err}`)
+      }
+    }
+
+    // Fallback: scan the pmtiles dir for orphans with the same resource_id that the DB
+    // lookup above didn't catch — e.g. a prior extract crashed before writing its
+    // InstalledResource row, or an earlier bug wrote a file without registering it.
+    // Matches both curated (`<id>_YYYY-MM.pmtiles`) and regional (`<id>_YYYYMMDD_zN.pmtiles`)
+    // naming — prefix-only so new filename formats don't silently miss.
+    const dir = dirname(params.outputFilepath)
+    const keepName = basename(params.outputFilepath)
+    const prefix = `${params.resourceMetadata.resource_id}_`
+    try {
+      const entries = await readdir(dir)
+      for (const entry of entries) {
+        if (entry === keepName || !entry.endsWith('.pmtiles')) continue
+        if (!entry.startsWith(prefix)) continue
+        const orphanPath = join(dir, entry)
+        if (orphanPath === oldFilePath) continue
+        try {
+          await fsUtils.deleteFileIfExists(orphanPath)
+          logger.info(`[RunExtractPmtilesJob] Pruned orphan pmtiles ${orphanPath}`)
+        } catch (err) {
+          logger.warn(`[RunExtractPmtilesJob] Failed to prune orphan ${orphanPath}: ${err}`)
+        }
+      }
+    } catch (err) {
+      logger.warn(`[RunExtractPmtilesJob] Directory scan for orphans failed: ${err}`)
+    }
+  }
+
+  static async getById(jobId: string): Promise<Job | undefined> {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+    return await queue.getJob(jobId)
+  }
+
+  static async dispatch(params: RunExtractPmtilesJobParams) {
+    const queueService = QueueService.getInstance()
+    const queue = queueService.getQueue(this.queue)
+    const jobId = this.getJobId(params.sourceUrl, params.regionFilepath, params.maxzoom)
+
+    const existing = await queue.getJob(jobId)
+    if (existing) {
+      const state = await existing.getState()
+      if (state === 'active' || state === 'waiting' || state === 'delayed') {
+        return {
+          job: existing,
+          created: false,
+          message: `Extract job already exists for these params`,
+        }
+      }
+      // Stale (completed/failed) — remove so we can re-dispatch under the same deterministic id
+      try {
+        await existing.remove()
+      } catch {
+        // Already gone or locked — add() below will still report a meaningful error
+      }
+    }
+
+    // Fewer attempts than HTTP downloads — a failed extract usually means the source URL
+    // rotated or the CDN is throttling, and resuming mid-extract isn't supported by the CLI
+    const job = await queue.add(this.key, params, {
+      jobId,
+      attempts: 3,
+      backoff: { type: 'exponential', delay: 60000 },
+      removeOnComplete: true,
+    })
+    return {
+      job,
+      created: true,
+      message: `Dispatched pmtiles extract job`,
+    }
+  }
+}

admin/app/middleware/compression_middleware.ts

@@ -0,0 +1,35 @@
+import env from '#start/env'
+import type { HttpContext } from '@adonisjs/core/http'
+import type { NextFn } from '@adonisjs/core/types/http'
+import compression from 'compression'
+
+// Skip compression for Server-Sent Events. The compression library buffers
+// response writes to determine encoding, which collapses per-token streaming
+// into a single block delivered after generation completes (regression in
+// v1.31.0-rc.2, reported in #781 by @toasterking).
+const compress = env.get('DISABLE_COMPRESSION')
+  ? null
+  : compression({
+      filter: (req: any, res: any) => {
+        const contentType = res.getHeader('Content-Type')
+        if (typeof contentType === 'string' && contentType.includes('text/event-stream')) {
+          return false
+        }
+        return compression.filter(req, res)
+      },
+    })
+
+export default class CompressionMiddleware {
+  async handle({ request, response }: HttpContext, next: NextFn) {
+    if (!compress) return await next()
+
+    await new Promise<void>((resolve, reject) => {
+      compress(request.request as any, response.response as any, (err?: any) => {
+        if (err) reject(err)
+        else resolve()
+      })
+    })
+
+    await next()
+  }
+}

admin/app/models/benchmark_result.ts

@@ -0,0 +1,85 @@
+import { DateTime } from 'luxon'
+import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
+import type { BenchmarkType, DiskType } from '../../types/benchmark.js'
+
+export default class BenchmarkResult extends BaseModel {
+  static namingStrategy = new SnakeCaseNamingStrategy()
+
+  @column({ isPrimary: true })
+  declare id: number
+
+  @column()
+  declare benchmark_id: string
+
+  @column()
+  declare benchmark_type: BenchmarkType
+
+  // Hardware information
+  @column()
+  declare cpu_model: string
+
+  @column()
+  declare cpu_cores: number
+
+  @column()
+  declare cpu_threads: number
+
+  @column()
+  declare ram_bytes: number
+
+  @column()
+  declare disk_type: DiskType
+
+  @column()
+  declare gpu_model: string | null
+
+  // System benchmark scores
+  @column()
+  declare cpu_score: number
+
+  @column()
+  declare memory_score: number
+
+  @column()
+  declare disk_read_score: number
+
+  @column()
+  declare disk_write_score: number
+
+  // AI benchmark scores (nullable for system-only benchmarks)
+  @column()
+  declare ai_tokens_per_second: number | null
+
+  @column()
+  declare ai_model_used: string | null
+
+  @column()
+  declare ai_time_to_first_token: number | null
+
+  // Composite NOMAD score (0-100)
+  @column()
+  declare nomad_score: number
+
+  // Repository submission tracking
+  @column({
+    serialize(value) {
+      return Boolean(value)
+    },
+  })
+  declare submitted_to_repository: boolean
+
+  @column.dateTime()
+  declare submitted_at: DateTime | null
+
+  @column()
+  declare repository_id: string | null
+
+  @column()
+  declare builder_tag: string | null
+
+  @column.dateTime({ autoCreate: true })
+  declare created_at: DateTime
+
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updated_at: DateTime
+}

admin/app/models/benchmark_setting.ts

@@ -0,0 +1,60 @@
+import { DateTime } from 'luxon'
+import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
+import type { BenchmarkSettingKey } from '../../types/benchmark.js'
+
+export default class BenchmarkSetting extends BaseModel {
+  static namingStrategy = new SnakeCaseNamingStrategy()
+
+  @column({ isPrimary: true })
+  declare id: number
+
+  @column()
+  declare key: BenchmarkSettingKey
+
+  @column()
+  declare value: string | null
+
+  @column.dateTime({ autoCreate: true })
+  declare created_at: DateTime
+
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updated_at: DateTime
+
+  /**
+   * Get a setting value by key
+   */
+  static async getValue(key: BenchmarkSettingKey): Promise<string | null> {
+    const setting = await this.findBy('key', key)
+    return setting?.value ?? null
+  }
+
+  /**
+   * Set a setting value by key (creates if not exists)
+   */
+  static async setValue(key: BenchmarkSettingKey, value: string | null): Promise<BenchmarkSetting> {
+    const setting = await this.firstOrCreate({ key }, { key, value })
+    if (setting.value !== value) {
+      setting.value = value
+      await setting.save()
+    }
+    return setting
+  }
+
+  /**
+   * Get all benchmark settings as a typed object
+   */
+  static async getAllSettings(): Promise<{
+    allow_anonymous_submission: boolean
+    installation_id: string | null
+    last_benchmark_run: string | null
+  }> {
+    const settings = await this.all()
+    const map = new Map(settings.map((s) => [s.key, s.value]))
+
+    return {
+      allow_anonymous_submission: map.get('allow_anonymous_submission') === 'true',
+      installation_id: map.get('installation_id') ?? null,
+      last_benchmark_run: map.get('last_benchmark_run') ?? null,
+    }
+  }
+}

admin/app/models/chat_message.ts

@@ -0,0 +1,29 @@
+import { DateTime } from 'luxon'
+import { BaseModel, column, belongsTo, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
+import type { BelongsTo } from '@adonisjs/lucid/types/relations'
+import ChatSession from './chat_session.js'
+
+export default class ChatMessage extends BaseModel {
+  static namingStrategy = new SnakeCaseNamingStrategy()
+
+  @column({ isPrimary: true })
+  declare id: number
+
+  @column()
+  declare session_id: number
+
+  @column()
+  declare role: 'system' | 'user' | 'assistant'
+
+  @column()
+  declare content: string
+
+  @belongsTo(() => ChatSession, { foreignKey: 'session_id', localKey: 'id' })
+  declare session: BelongsTo<typeof ChatSession>
+
+  @column.dateTime({ autoCreate: true })
+  declare created_at: DateTime
+
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updated_at: DateTime
+}

admin/app/models/chat_session.ts

@@ -0,0 +1,29 @@
+import { DateTime } from 'luxon'
+import { BaseModel, column, hasMany, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
+import type { HasMany } from '@adonisjs/lucid/types/relations'
+import ChatMessage from './chat_message.js'
+
+export default class ChatSession extends BaseModel {
+  static namingStrategy = new SnakeCaseNamingStrategy()
+
+  @column({ isPrimary: true })
+  declare id: number
+
+  @column()
+  declare title: string
+
+  @column()
+  declare model: string | null
+
+  @hasMany(() => ChatMessage, {
+    foreignKey: 'session_id',
+    localKey: 'id',
+  })
+  declare messages: HasMany<typeof ChatMessage>
+
+  @column.dateTime({ autoCreate: true })
+  declare created_at: DateTime
+
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updated_at: DateTime
+}

admin/app/models/collection_manifest.ts

@@ -0,0 +1,22 @@
+import { DateTime } from 'luxon'
+import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
+import type { ManifestType } from '../../types/collections.js'
+
+export default class CollectionManifest extends BaseModel {
+  static namingStrategy = new SnakeCaseNamingStrategy()
+
+  @column({ isPrimary: true })
+  declare type: ManifestType
+
+  @column()
+  declare spec_version: string
+
+  @column({
+    consume: (value: string) => (typeof value === 'string' ? JSON.parse(value) : value),
+    prepare: (value: any) => JSON.stringify(value),
+  })
+  declare spec_data: any
+
+  @column.dateTime()
+  declare fetched_at: DateTime
+}

admin/app/models/curated_collection.ts

@@ -1,39 +0,0 @@
-import { DateTime } from 'luxon'
-import { BaseModel, column, hasMany, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
-import CuratedCollectionResource from './curated_collection_resource.js'
-import type { HasMany } from '@adonisjs/lucid/types/relations'
-import type { CuratedCollectionType } from '../../types/curated_collections.js'
-
-export default class CuratedCollection extends BaseModel {
-  static namingStrategy = new SnakeCaseNamingStrategy()
-
-  @column({ isPrimary: true })
-  declare slug: string
-
-  @column()
-  declare type: CuratedCollectionType
-
-  @column()
-  declare name: string
-
-  @column()
-  declare description: string
-
-  @column()
-  declare icon: string
-
-  @column()
-  declare language: string
-
-  @hasMany(() => CuratedCollectionResource, {
-    foreignKey: 'curated_collection_slug',
-    localKey: 'slug',
-  })
-  declare resources: HasMany<typeof CuratedCollectionResource>
-
-  @column.dateTime({ autoCreate: true })
-  declare created_at: DateTime
-
-  @column.dateTime({ autoCreate: true, autoUpdate: true })
-  declare updated_at: DateTime
-}

admin/app/models/curated_collection_resource.ts

@@ -1,41 +0,0 @@
-import { DateTime } from 'luxon'
-import { BaseModel, belongsTo, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
-import CuratedCollection from './curated_collection.js'
-import type { BelongsTo } from '@adonisjs/lucid/types/relations'
-
-export default class CuratedCollectionResource extends BaseModel {
-  static namingStrategy = new SnakeCaseNamingStrategy()
-
-  @column({ isPrimary: true })
-  declare id: number
-
-  @column()
-  declare curated_collection_slug: string
-
-  @belongsTo(() => CuratedCollection, {
-    foreignKey: 'slug',
-    localKey: 'curated_collection_slug',
-  })
-  declare curated_collection: BelongsTo<typeof CuratedCollection>
-
-  @column()
-  declare title: string
-
-  @column()
-  declare url: string
-
-  @column()
-  declare description: string
-
-  @column()
-  declare size_mb: number
-
-  @column()
-  declare downloaded: boolean
-
-  @column.dateTime({ autoCreate: true })
-  declare created_at: DateTime
-
-  @column.dateTime({ autoCreate: true, autoUpdate: true })
-  declare updated_at: DateTime
-}

admin/app/models/custom_library_source.ts

@@ -0,0 +1,24 @@
+import { DateTime } from 'luxon'
+import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
+
+export default class CustomLibrarySource extends BaseModel {
+  static namingStrategy = new SnakeCaseNamingStrategy()
+
+  @column({ isPrimary: true })
+  declare id: number
+
+  @column()
+  declare name: string
+
+  @column()
+  declare base_url: string
+
+  @column()
+  declare is_default: boolean
+
+  @column.dateTime({ autoCreate: true })
+  declare created_at: DateTime
+
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updated_at: DateTime
+}

admin/app/models/installed_resource.ts

@@ -0,0 +1,54 @@
+import { DateTime } from 'luxon'
+import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
+
+export default class InstalledResource extends BaseModel {
+  static namingStrategy = new SnakeCaseNamingStrategy()
+
+  @column({ isPrimary: true })
+  declare id: number
+
+  @column()
+  declare resource_id: string
+
+  @column()
+  declare resource_type: 'zim' | 'map'
+
+  @column()
+  declare collection_ref: string | null
+
+  @column()
+  declare version: string
+
+  @column()
+  declare url: string
+
+  @column()
+  declare file_path: string
+
+  @column()
+  declare file_size_bytes: number | null
+
+  @column.dateTime()
+  declare installed_at: DateTime
+
+  // ── Content auto-update state (global opt-in; gated by `contentAutoUpdate.enabled`) ──
+
+  /** Newest catalog version (YYYY-MM) detected, or null when already current. */
+  @column()
+  declare available_update_version: string | null
+
+  /** Size (bytes) of the available update, captured from the catalog. */
+  @column()
+  declare available_update_size_bytes: number | null
+
+  /** Cool-off anchor: when the current available update was first detected. */
+  @column.dateTime()
+  declare available_update_first_seen_at: DateTime | null
+
+  /** Per-resource failure backoff so one flapping download self-disables. */
+  @column()
+  declare auto_update_consecutive_failures: number
+
+  @column()
+  declare auto_update_disabled_reason: string | null
+}

admin/app/models/kb_ingest_state.ts

@@ -0,0 +1,77 @@
+import { DateTime } from 'luxon'
+import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
+import type { KbIngestStateValue } from '../../types/kb_ingest_state.js'
+
+const LAST_ERROR_MAX_LEN = 1024
+
+/**
+ * Tracks the per-file decision and outcome of AI knowledge-base ingestion.
+ *
+ * The row exists for any embeddable file the scanner has seen and is independent
+ * of `installed_resources` (which only covers curated downloads). Replaces the
+ * earlier "any chunks in qdrant ⇒ embedded" binary check, which conflated
+ * partially-stalled ingestions with fully-indexed files. See RFC #883.
+ */
+export default class KbIngestState extends BaseModel {
+  static table = 'kb_ingest_state'
+  static namingStrategy = new SnakeCaseNamingStrategy()
+
+  @column({ isPrimary: true })
+  declare id: number
+
+  @column()
+  declare file_path: string
+
+  @column()
+  declare state: KbIngestStateValue
+
+  @column()
+  declare chunks_embedded: number
+
+  @column()
+  declare last_error: string | null
+
+  @column.dateTime({ autoCreate: true })
+  declare created_at: DateTime
+
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updated_at: DateTime
+
+  static async getOrCreate(filePath: string): Promise<KbIngestState> {
+    return this.firstOrCreate(
+      { file_path: filePath },
+      { file_path: filePath, state: 'pending_decision', chunks_embedded: 0 }
+    )
+  }
+
+  static async markIndexed(filePath: string, chunksEmbedded: number): Promise<void> {
+    const row = await this.getOrCreate(filePath)
+    row.state = 'indexed'
+    row.chunks_embedded = chunksEmbedded
+    row.last_error = null
+    await row.save()
+  }
+
+  static async markFailed(filePath: string, errorMessage: string): Promise<void> {
+    const row = await this.getOrCreate(filePath)
+    row.state = 'failed'
+    row.last_error = errorMessage.slice(0, LAST_ERROR_MAX_LEN)
+    await row.save()
+  }
+
+  static async markBrowseOnly(filePath: string): Promise<void> {
+    const row = await this.getOrCreate(filePath)
+    row.state = 'browse_only'
+    await row.save()
+  }
+
+  static async markStalled(filePath: string): Promise<void> {
+    const row = await this.getOrCreate(filePath)
+    row.state = 'stalled'
+    await row.save()
+  }
+
+  static async remove(filePath: string): Promise<void> {
+    await this.query().where('file_path', filePath).delete()
+  }
+}

admin/app/models/kb_ratio_registry.ts

@@ -0,0 +1,79 @@
+import { DateTime } from 'luxon'
+import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
+import {
+  findChunksPerMb,
+  estimateChunkCount,
+  estimateBatch,
+  type BatchEstimate,
+  type BatchEstimateInput,
+} from '../utils/kb_ratio_lookup.js'
+
+/**
+ * Self-calibrating registry of `{filename-prefix → chunks_per_mb}` ratios used
+ * for disk-footprint and time-to-embed estimates surfaced in the KB panel.
+ *
+ * Migration seeds the registry with heuristic defaults from the RFC #883
+ * appendix; Phase 4 self-calibration will update rows in place as ZIMs finish
+ * ingesting and the real ratio becomes known. Lookup is longest-prefix-match
+ * (see `kb_ratio_lookup.ts`) so a specific entry (`wikipedia_en_simple_`)
+ * overrides a broader one (`wikipedia_en_`).
+ */
+export default class KbRatioRegistry extends BaseModel {
+  static table = 'kb_ratio_registry'
+  static namingStrategy = new SnakeCaseNamingStrategy()
+
+  @column({ isPrimary: true })
+  declare id: number
+
+  @column()
+  declare pattern: string
+
+  @column()
+  declare chunks_per_mb: number
+
+  @column()
+  declare sample_count: number
+
+  @column()
+  declare notes: string | null
+
+  @column.dateTime({ autoCreate: true })
+  declare created_at: DateTime
+
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updated_at: DateTime
+
+  /** Look up chunks_per_mb for a filename by longest-prefix match. */
+  static async lookup(filename: string): Promise<number | null> {
+    const rows = await this.all()
+    return findChunksPerMb(filename, rows)
+  }
+
+  /**
+   * Estimate total chunks for a file of the given size on disk.
+   *
+   * `ignoreCatchAll` excludes the empty-pattern fallback, returning `null` for
+   * filenames that only the catch-all would match. The partial_stall warning
+   * uses this so it never flags ZIMs the registry can't specifically
+   * characterize (e.g. PDF/link-out-heavy archives whose byte size wildly
+   * over-predicts embeddable chunks). See #913.
+   */
+  static async estimateChunks(
+    filename: string,
+    fileSizeBytes: number,
+    opts: { ignoreCatchAll?: boolean } = {}
+  ): Promise<number | null> {
+    const rows = await this.all()
+    return estimateChunkCount(filename, fileSizeBytes, rows, opts)
+  }
+
+  /**
+   * Aggregate an embedding-disk-cost estimate across a batch of files. Used by
+   * the curated-tier-change UI to show "you're about to add ~X GB of
+   * embeddings on top of the ZIM downloads" before the user commits.
+   */
+  static async estimateBatch(files: BatchEstimateInput[]): Promise<BatchEstimate> {
+    const rows = await this.all()
+    return estimateBatch(files, rows)
+  }
+}

admin/app/models/kv_store.ts

@@ -0,0 +1,64 @@
+import { DateTime } from 'luxon'
+import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
+import { KV_STORE_SCHEMA, type KVStoreKey, type KVStoreValue } from '../../types/kv_store.js'
+import { parseBoolean } from '../utils/misc.js'
+
+/**
+ * Generic key-value store model for storing various settings
+ * that don't necessitate their own dedicated models.
+ */
+export default class KVStore extends BaseModel {
+  static table = 'kv_store'
+  static namingStrategy = new SnakeCaseNamingStrategy()
+
+  @column({ isPrimary: true })
+  declare id: number
+
+  @column()
+  declare key: KVStoreKey
+
+  @column()
+  declare value: string | null
+
+  @column.dateTime({ autoCreate: true })
+  declare created_at: DateTime
+
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updated_at: DateTime
+
+  /**
+   * Get a setting value by key, automatically deserializing to the correct type.
+   */
+  static async getValue<K extends KVStoreKey>(key: K): Promise<KVStoreValue<K> | null> {
+    const setting = await this.findBy('key', key)
+    if (!setting || setting.value === undefined || setting.value === null) {
+      return null
+    }
+    const raw = String(setting.value)
+    return (KV_STORE_SCHEMA[key] === 'boolean' ? parseBoolean(raw) : raw) as KVStoreValue<K>
+  }
+
+  /**
+   * Set a setting value by key (creates if not exists), automatically serializing to string.
+   */
+  static async setValue<K extends KVStoreKey>(key: K, value: KVStoreValue<K>): Promise<KVStore> {
+    const serialized = String(value)
+    const setting = await this.firstOrCreate({ key }, { key, value: serialized })
+    if (setting.value !== serialized) {
+      setting.value = serialized
+      await setting.save()
+    }
+    return setting
+  }
+
+  /**
+   * Clear a setting value by key, storing null so getValue returns null.
+   */
+  static async clearValue<K extends KVStoreKey>(key: K): Promise<void> {
+    const setting = await this.findBy('key', key)
+    if (setting && setting.value !== null) {
+      setting.value = null
+      await setting.save()
+    }
+  }
+}

admin/app/models/map_marker.ts

@@ -0,0 +1,43 @@
+import { DateTime } from 'luxon'
+import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
+
+export default class MapMarker extends BaseModel {
+  static namingStrategy = new SnakeCaseNamingStrategy()
+
+  @column({ isPrimary: true })
+  declare id: number
+
+  @column()
+  declare name: string
+
+  @column()
+  declare longitude: number
+
+  @column()
+  declare latitude: number
+
+  @column()
+  declare color: string
+
+  // 'pin' for user-placed markers, 'waypoint' for route points (future)
+  @column()
+  declare marker_type: string
+
+  // Groups markers into a route (future)
+  @column()
+  declare route_id: string | null
+
+  // Order within a route (future)
+  @column()
+  declare route_order: number | null
+
+  // Optional user notes for a location
+  @column()
+  declare notes: string | null
+
+  @column.dateTime({ autoCreate: true })
+  declare created_at: DateTime
+
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updated_at: DateTime
+}

admin/app/models/service.ts

@@ -26,6 +26,12 @@ export default class Service extends BaseModel {
   @column()
   declare description: string | null
 
+  @column()
+  declare powered_by: string | null
+
+  @column()
+  declare display_order: number | null
+
   @column()
   declare icon: string | null // must be a TablerIcons name to be properly rendered in the UI (e.g. "IconBrandDocker")
 
@@ -53,9 +59,73 @@ export default class Service extends BaseModel {
   @column()
   declare ui_location: string | null
 
+  // User-set override for the launch ("Open") link (e.g. a reverse-proxy/local-DNS host like
+  // https://jellyfin.myhomelab.net). When null, the default host + port link derived from
+  // ui_location is used. Only affects user-facing links — never internal service-to-service URLs.
+  @column()
+  declare custom_url: string | null
+
   @column()
   declare metadata: string | null
 
+  @column({
+    serialize(value) {
+      return Boolean(value)
+    },
+  })
+  declare is_custom: boolean
+
+  @column({
+    serialize(value) {
+      return Boolean(value)
+    },
+  })
+  declare is_user_modified: boolean
+
+  @column()
+  declare category: string | null
+
+  // When true the service is sunset: hidden from the install catalog unless it is already
+  // installed (see SystemService.getServices). Lets a deprecated app stay manageable for users who
+  // still run it while keeping new users from installing it.
+  @column({
+    serialize(value) {
+      return Boolean(value)
+    },
+  })
+  declare is_deprecated: boolean
+
+  @column()
+  declare source_repo: string | null
+
+  @column()
+  declare available_update_version: string | null
+
+  @column.dateTime()
+  declare update_checked_at: DateTime | null
+
+  // Per-app opt-in for automatic updates. An app auto-updates only when both this
+  // and the global `appAutoUpdate.enabled` master switch are on.
+  @column({
+    serialize(value) {
+      return Boolean(value)
+    },
+  })
+  declare auto_update_enabled: boolean
+
+  // When the current `available_update_version` was first detected — the anchor for
+  // the auto-update cool-off (registry tags carry no publish timestamp).
+  @column.dateTime()
+  declare available_update_first_seen_at: DateTime | null
+
+  // Per-app auto-update failure backoff; at the threshold the app self-disables via
+  // `auto_update_disabled_reason` without affecting other apps.
+  @column()
+  declare auto_update_consecutive_failures: number
+
+  @column()
+  declare auto_update_disabled_reason: string | null
+
   @column.dateTime({ autoCreate: true })
   declare created_at: DateTime
 

admin/app/models/wikipedia_selection.ts

@@ -0,0 +1,27 @@
+import { DateTime } from 'luxon'
+import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
+
+export default class WikipediaSelection extends BaseModel {
+  static namingStrategy = new SnakeCaseNamingStrategy()
+
+  @column({ isPrimary: true })
+  declare id: number
+
+  @column()
+  declare option_id: string
+
+  @column()
+  declare url: string | null
+
+  @column()
+  declare filename: string | null
+
+  @column()
+  declare status: 'none' | 'downloading' | 'installed' | 'failed'
+
+  @column.dateTime({ autoCreate: true })
+  declare created_at: DateTime
+
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updated_at: DateTime
+}

admin/app/services/app_auto_update_service.ts

@@ -0,0 +1,398 @@
+import { inject } from '@adonisjs/core'
+import logger from '@adonisjs/core/services/logger'
+import { DateTime } from 'luxon'
+import KVStore from '#models/kv_store'
+import Service from '#models/service'
+import { DockerService } from '#services/docker_service'
+import { DownloadService } from '#services/download_service'
+import { SystemService } from '#services/system_service'
+import { ContainerRegistryService } from '#services/container_registry_service'
+import { isNewerVersion, parseMajorVersion } from '../utils/version.js'
+import { isWithinWindow } from '../utils/update_window.js'
+import {
+  checkImageDiskSpace,
+  type Blocker,
+  type PreflightResult,
+} from '../utils/image_disk_preflight.js'
+
+/**
+ * Defaults shared with the core auto-update. App auto-updates intentionally reuse
+ * the SAME window/cool-off settings (`autoUpdate.windowStart/windowEnd/cooloffHours`);
+ * only the enable flag (`appAutoUpdate.enabled`) is separate.
+ */
+const DEFAULT_WINDOW_START = '02:00'
+const DEFAULT_WINDOW_END = '05:00'
+const DEFAULT_COOLOFF_HOURS = 72
+
+/** Per-app genuine failures before that app self-disables (others keep running). */
+const MAX_CONSECUTIVE_FAILURES = 3
+
+export interface AppAutoUpdateConfig {
+  /** Global master switch (`appAutoUpdate.enabled`). */
+  enabled: boolean
+  windowStart: string
+  windowEnd: string
+  cooloffHours: number
+}
+
+/** An installed app that should be auto-updated this run. */
+export interface AppUpdateTarget {
+  service: Service
+  /** Exact registry tag to update to (the value in `available_update_version`). */
+  targetVersion: string
+}
+
+/** Per-app eligibility verdict (drives both selection and the status UI). */
+export interface AppEligibility {
+  eligible: boolean
+  reason: string
+  cooloffRemainingHours: number | null
+}
+
+export interface AppAutoUpdateAppStatus {
+  service_name: string
+  friendly_name: string | null
+  auto_update_enabled: boolean
+  current_version: string
+  available_update_version: string | null
+  first_seen_at: string | null
+  eligible: boolean
+  reason: string
+  cooloff_remaining_hours: number | null
+  consecutive_failures: number
+  auto_disabled_reason: string | null
+}
+
+export interface AppAutoUpdateStatus extends AppAutoUpdateConfig {
+  withinWindow: boolean
+  lastAttemptAt: string | null
+  lastResult: string | null
+  apps: AppAutoUpdateAppStatus[]
+}
+
+/**
+ * Decision + safety layer for automatic updates of installed sibling apps (the
+ * containers NOMAD deploys via the Docker socket and manages in Supply Depot).
+ *
+ * This is the app-side counterpart to {@link AutoUpdateService} and intentionally
+ * reuses its generic window/disk pre-flight helpers. Unlike the core update, an
+ * app update needs no sidecar — the admin container recreates its siblings directly
+ * via {@link DockerService.updateContainer} (in-process pull → rename → health-check
+ * → rollback). Auto-update only decides *whether* each opted-in app should update now
+ * (master switch on + per-app toggle on + in window + an eligible minor/patch past
+ * its cool-off + pre-flight passes) and then drives the existing update path.
+ *
+ * Minor/patch-only is already guaranteed upstream by
+ * {@link ContainerRegistryService.getAvailableUpdates} (same-major filter); the
+ * major-version check here is defense-in-depth.
+ */
+@inject()
+export class AppAutoUpdateService {
+  constructor(
+    private dockerService: DockerService,
+    private downloadService: DownloadService,
+    private systemService: SystemService,
+    private containerRegistryService: ContainerRegistryService
+  ) {}
+
+  /** Read the global master switch plus the shared window/cool-off settings. */
+  async getConfig(): Promise<AppAutoUpdateConfig> {
+    const [enabled, windowStart, windowEnd, cooloffHours] = await Promise.all([
+      KVStore.getValue('appAutoUpdate.enabled'),
+      KVStore.getValue('autoUpdate.windowStart'),
+      KVStore.getValue('autoUpdate.windowEnd'),
+      KVStore.getValue('autoUpdate.cooloffHours'),
+    ])
+
+    const parsedCooloff = Number(cooloffHours)
+    return {
+      enabled: enabled ?? false,
+      windowStart: windowStart || DEFAULT_WINDOW_START,
+      windowEnd: windowEnd || DEFAULT_WINDOW_END,
+      // `Number(null) === 0`, so an unset value must fall through to the default
+      // rather than silently resolving to a zero cool-off. An explicit 0 is honored.
+      cooloffHours:
+        cooloffHours !== null && Number.isFinite(parsedCooloff) && parsedCooloff >= 0
+          ? parsedCooloff
+          : DEFAULT_COOLOFF_HOURS,
+    }
+  }
+
+  /**
+   * Pure per-app eligibility verdict. An app is eligible when it has a detected
+   * update that is the same major (defense-in-depth), strictly newer, not self-
+   * disabled, and past its cool-off (measured from first-detected).
+   */
+  appEligibility(service: Service, cooloffHours: number, now: DateTime): AppEligibility {
+    if (!service.available_update_version) {
+      return { eligible: false, reason: 'Up to date', cooloffRemainingHours: null }
+    }
+    if (service.auto_update_disabled_reason) {
+      return {
+        eligible: false,
+        reason: 'Auto-update disabled after repeated failures',
+        cooloffRemainingHours: null,
+      }
+    }
+
+    const currentTag = this.containerRegistryService.parseImageReference(
+      service.container_image
+    ).tag
+    if (currentTag === 'latest') {
+      return {
+        eligible: false,
+        reason: 'Pinned to :latest — cannot version-check',
+        cooloffRemainingHours: null,
+      }
+    }
+    if (parseMajorVersion(service.available_update_version) !== parseMajorVersion(currentTag)) {
+      return {
+        eligible: false,
+        reason: 'Major version — manual update required',
+        cooloffRemainingHours: null,
+      }
+    }
+    if (!isNewerVersion(service.available_update_version, currentTag)) {
+      return { eligible: false, reason: 'Up to date', cooloffRemainingHours: null }
+    }
+    if (!service.available_update_first_seen_at) {
+      return { eligible: false, reason: 'Cool-off pending', cooloffRemainingHours: cooloffHours }
+    }
+
+    const ageHours = now.diff(service.available_update_first_seen_at, 'hours').hours
+    const remaining = cooloffHours - ageHours
+    if (remaining > 0) {
+      const rounded = Math.ceil(remaining)
+      return {
+        eligible: false,
+        reason: `In cool-off (${rounded}h remaining)`,
+        cooloffRemainingHours: rounded,
+      }
+    }
+
+    return {
+      eligible: true,
+      reason: `Eligible → ${service.available_update_version}`,
+      cooloffRemainingHours: 0,
+    }
+  }
+
+  /** Installed, opted-in apps that are eligible to update right now. */
+  async getEligibleApps(config: AppAutoUpdateConfig, now: DateTime): Promise<AppUpdateTarget[]> {
+    const apps = await Service.query().where('installed', true).where('auto_update_enabled', true)
+    const targets: AppUpdateTarget[] = []
+    for (const service of apps) {
+      const verdict = this.appEligibility(service, config.cooloffHours, now)
+      if (verdict.eligible) {
+        targets.push({ service, targetVersion: service.available_update_version! })
+      }
+    }
+    return targets
+  }
+
+  /**
+   * Run-wide pre-flight checked once per attempt (independent of any single app):
+   * never auto-update while content/model downloads are running. Transient → `skip`.
+   */
+  async runGlobalPreflight(): Promise<PreflightResult> {
+    const blockers: Blocker[] = []
+    try {
+      const downloads = await this.downloadService.listDownloadJobs()
+      const active = downloads.filter(
+        (d) => !!d.status && ['waiting', 'active', 'delayed'].includes(d.status)
+      )
+      if (active.length > 0) {
+        blockers.push({ reason: `${active.length} download(s) in progress`, severity: 'skip' })
+      }
+    } catch (error) {
+      logger.warn(`[AppAutoUpdateService] Could not check active downloads: ${error.message}`)
+    }
+    return { ok: blockers.length === 0, blockers }
+  }
+
+  /** Per-app pre-flight: not already mid-operation (`skip`) and enough disk (`failure`). */
+  async runAppPreflight(target: AppUpdateTarget): Promise<PreflightResult> {
+    const blockers: Blocker[] = []
+    const service = target.service
+
+    if (service.installation_status !== 'idle') {
+      blockers.push({
+        reason: `App has an operation in progress (status: ${service.installation_status})`,
+        severity: 'skip',
+      })
+    }
+
+    const hostArch = await this.getHostArch()
+    const targetImage = `${this.imageBase(service.container_image)}:${target.targetVersion}`
+    const diskBlocker = await checkImageDiskSpace({
+      image: targetImage,
+      hostArch,
+      containerRegistryService: this.containerRegistryService,
+      systemService: this.systemService,
+    })
+    if (diskBlocker) blockers.push(diskBlocker)
+
+    return { ok: blockers.length === 0, blockers }
+  }
+
+  /**
+   * Entry point invoked by AppAutoUpdateJob. Gates on the master switch + window,
+   * then runs each eligible app through pre-flight and {@link DockerService.updateContainer}.
+   * A failing app self-disables after repeated failures without affecting the others.
+   */
+  async attempt(): Promise<{ updated: number; reason: string }> {
+    const config = await this.getConfig()
+    const now = DateTime.now()
+
+    if (!config.enabled) {
+      return { updated: 0, reason: 'App auto-update is disabled' }
+    }
+    if (!isWithinWindow(config.windowStart, config.windowEnd, now)) {
+      const reason = `Outside update window (${config.windowStart}-${config.windowEnd})`
+      await this.recordRun(reason)
+      return { updated: 0, reason }
+    }
+
+    const eligible = await this.getEligibleApps(config, now)
+    if (eligible.length === 0) {
+      const reason = 'No eligible app updates (all current, in cool-off, or major-only)'
+      await this.recordRun(reason)
+      return { updated: 0, reason }
+    }
+
+    const global = await this.runGlobalPreflight()
+    if (!global.ok) {
+      const reason = `Pre-flight blocked: ${global.blockers.map((b) => b.reason).join('; ')}`
+      await this.recordRun(reason)
+      return { updated: 0, reason }
+    }
+
+    let updated = 0
+    let failed = 0
+    let skipped = 0
+
+    for (const target of eligible) {
+      const name = target.service.service_name
+      const preflight = await this.runAppPreflight(target)
+      if (!preflight.ok) {
+        const summary = preflight.blockers.map((b) => b.reason).join('; ')
+        if (preflight.blockers.some((b) => b.severity === 'failure')) {
+          await this.recordAppFailure(target.service, summary)
+          failed++
+        } else {
+          logger.info(`[AppAutoUpdateService] Skipped ${name}: ${summary}`)
+          skipped++
+        }
+        continue
+      }
+
+      logger.info(`[AppAutoUpdateService] Updating ${name} → ${target.targetVersion}`)
+      const result = await this.dockerService.updateContainer(name, target.targetVersion)
+      if (result.success) {
+        await this.recordAppSuccess(target.service)
+        updated++
+      } else {
+        await this.recordAppFailure(target.service, result.message)
+        failed++
+      }
+    }
+
+    const reason = `${updated} updated, ${failed} failed, ${skipped} skipped`
+    await this.recordRun(reason)
+    logger.info(`[AppAutoUpdateService] Run complete: ${reason}`)
+    return { updated, reason }
+  }
+
+  /** Clear an app's failure backoff after a successful auto-update. */
+  private async recordAppSuccess(service: Service): Promise<void> {
+    // updateContainer already advanced container_image and cleared
+    // available_update_version on its own (fresh) row; here we only touch the
+    // backoff fields, so Lucid persists just those dirty columns.
+    service.auto_update_consecutive_failures = 0
+    service.auto_update_disabled_reason = null
+    await service.save()
+  }
+
+  /** Record an app failure and self-disable it once the threshold is reached. */
+  private async recordAppFailure(service: Service, reason: string): Promise<void> {
+    const failures = (service.auto_update_consecutive_failures || 0) + 1
+    service.auto_update_consecutive_failures = failures
+    if (failures >= MAX_CONSECUTIVE_FAILURES) {
+      service.auto_update_disabled_reason = `Auto-update disabled after ${failures} consecutive failures. Last error: ${reason}`
+      logger.error(
+        `[AppAutoUpdateService] ${service.service_name} auto-disabled after ${failures} failures`
+      )
+    }
+    await service.save()
+    logger.error(
+      `[AppAutoUpdateService] ${service.service_name} failure ${failures}/${MAX_CONSECUTIVE_FAILURES}: ${reason}`
+    )
+  }
+
+  /** Record the global last-attempt summary for the settings UI. */
+  private async recordRun(reason: string): Promise<void> {
+    await KVStore.setValue('appAutoUpdate.lastAttemptAt', DateTime.now().toISO()!)
+    await KVStore.setValue('appAutoUpdate.lastResult', reason)
+  }
+
+  /** Full state snapshot for the settings UI (opted-in apps + their eligibility). */
+  async getStatus(): Promise<AppAutoUpdateStatus> {
+    const config = await this.getConfig()
+    const now = DateTime.now()
+
+    const apps = await Service.query().where('installed', true).where('auto_update_enabled', true)
+    const appStatuses: AppAutoUpdateAppStatus[] = apps.map((service) => {
+      const verdict = this.appEligibility(service, config.cooloffHours, now)
+      return {
+        service_name: service.service_name,
+        friendly_name: service.friendly_name,
+        auto_update_enabled: service.auto_update_enabled,
+        current_version: this.containerRegistryService.parseImageReference(service.container_image)
+          .tag,
+        available_update_version: service.available_update_version,
+        first_seen_at: service.available_update_first_seen_at?.toISO() ?? null,
+        eligible: verdict.eligible,
+        reason: verdict.reason,
+        cooloff_remaining_hours: verdict.cooloffRemainingHours,
+        consecutive_failures: service.auto_update_consecutive_failures || 0,
+        auto_disabled_reason: service.auto_update_disabled_reason,
+      }
+    })
+
+    const [lastAttemptAt, lastResult] = await Promise.all([
+      KVStore.getValue('appAutoUpdate.lastAttemptAt'),
+      KVStore.getValue('appAutoUpdate.lastResult'),
+    ])
+
+    return {
+      ...config,
+      withinWindow: isWithinWindow(config.windowStart, config.windowEnd, now),
+      lastAttemptAt: lastAttemptAt || null,
+      lastResult: lastResult || null,
+      apps: appStatuses,
+    }
+  }
+
+  /** Strip the tag from an image reference, leaving "registry/namespace/repo". */
+  private imageBase(image: string): string {
+    return image.includes(':') ? image.substring(0, image.lastIndexOf(':')) : image
+  }
+
+  /** Map the Docker daemon's architecture string to OCI naming (amd64/arm64/...). */
+  private async getHostArch(): Promise<string> {
+    try {
+      const info = await this.dockerService.docker.info()
+      const arch = info.Architecture || ''
+      const archMap: Record<string, string> = {
+        x86_64: 'amd64',
+        aarch64: 'arm64',
+        armv7l: 'arm',
+        amd64: 'amd64',
+        arm64: 'arm64',
+      }
+      return archMap[arch] || arch.toLowerCase()
+    } catch {
+      return 'amd64'
+    }
+  }
+}

admin/app/services/auto_update_service.ts

@@ -0,0 +1,580 @@
+import { inject } from '@adonisjs/core'
+import logger from '@adonisjs/core/services/logger'
+import axios from 'axios'
+import { DateTime } from 'luxon'
+import KVStore from '#models/kv_store'
+import Service from '#models/service'
+import { DockerService } from '#services/docker_service'
+import { DownloadService } from '#services/download_service'
+import { SystemService } from '#services/system_service'
+import { SystemUpdateService } from '#services/system_update_service'
+import { ContainerRegistryService } from '#services/container_registry_service'
+import { isNewerVersion, parseMajorVersion } from '../utils/version.js'
+import { isWithinWindow as isWithinWindowUtil } from '../utils/update_window.js'
+import {
+  checkImageDiskSpace,
+  type Blocker,
+  type PreflightResult,
+} from '../utils/image_disk_preflight.js'
+
+/** Docker image repository for the NOMAD admin/core image (tag applied per-release). */
+const NOMAD_IMAGE_REPO = 'ghcr.io/crosstalk-solutions/project-nomad'
+const RELEASES_URL = 'https://api.github.com/repos/Crosstalk-Solutions/project-nomad/releases'
+
+/** Defaults for user-configurable settings (server-local time window + cool-off). */
+const DEFAULT_WINDOW_START = '02:00'
+const DEFAULT_WINDOW_END = '05:00'
+const DEFAULT_COOLOFF_HOURS = 72
+
+/** Genuine failures before auto-update disables itself to avoid an update loop. */
+const MAX_CONSECUTIVE_FAILURES = 3
+
+/**
+ * Only tags matching strict semver are eligible. Defense-in-depth: the selected
+ * tag becomes `target_tag`, which the sidecar interpolates into a host-side `sed`
+ * (install/sidecar-updater/update-watcher.sh) — so a malformed tag must never be
+ * able to reach it, even though releases come from a trusted repo.
+ */
+const SEMVER_TAG = /^\d+\.\d+\.\d+$/
+
+/** Cache the GitHub releases feed in-process to avoid hammering the API. */
+const RELEASES_CACHE_TTL_MS = 15 * 60 * 1000
+/** Briefly remember a failed fetch so repeated calls don't each block on the timeout. */
+const RELEASES_FAILURE_TTL_MS = 60 * 1000
+
+export interface AutoUpdateConfig {
+  enabled: boolean
+  windowStart: string
+  windowEnd: string
+  cooloffHours: number
+}
+
+export interface EligibleTarget {
+  version: string
+  tag: string
+  publishedAt: string
+}
+
+// Pre-flight types/primitives are shared with AppAutoUpdateService; re-exported
+// here for back-compat with existing imports of this module.
+export type { Blocker, BlockerSeverity, PreflightResult } from '../utils/image_disk_preflight.js'
+
+/** Minimal shape of a GitHub release entry we depend on. */
+export interface GithubRelease {
+  tag_name?: string
+  published_at?: string
+  draft?: boolean
+  prerelease?: boolean
+}
+
+/**
+ * Inputs that can be injected to exercise the decision pipeline deterministically
+ * (used by the dry-run command/tests). All are optional; when omitted the real
+ * settings/clock/GitHub feed/pre-flight are used, exactly as production runs.
+ */
+export interface EvaluateOverrides {
+  currentVersion?: string
+  releases?: GithubRelease[]
+  now?: DateTime
+  forceEnabled?: boolean
+  windowStart?: string
+  windowEnd?: string
+  cooloffHours?: number
+  /** Treat pre-flight as passing without touching Docker/disk/queues. */
+  skipPreflight?: boolean
+  /** Substitute a canned pre-flight result. */
+  fakePreflight?: PreflightResult
+}
+
+export type DecisionOutcome =
+  | 'disabled'
+  | 'outside-window'
+  | 'eligibility-error'
+  | 'no-eligible'
+  | 'blocked'
+  | 'ready'
+
+/** Side-effect-free verdict of the decision pipeline. */
+export interface AutoUpdateDecision {
+  enabled: boolean
+  currentVersion: string
+  config: AutoUpdateConfig
+  withinWindow: boolean
+  eligibleTarget: EligibleTarget | null
+  preflight: PreflightResult | null
+  outcome: DecisionOutcome
+  reason: string
+}
+
+export interface AutoUpdateStatus extends AutoUpdateConfig {
+  currentVersion: string
+  withinWindow: boolean
+  eligibleTarget: EligibleTarget | null
+  lastAttemptAt: string | null
+  lastResult: string | null
+  lastError: string | null
+  consecutiveFailures: number
+  autoDisabledReason: string | null
+}
+
+/**
+ * Decision + safety layer for automatic updates of the NOMAD application itself.
+ *
+ * It does NOT recreate containers — that remains the sidecar's job. This service
+ * decides *whether* an update should run right now (opt-in, in-window, an eligible
+ * minor/patch release exists past its cool-off, pre-flight checks pass) and, if so,
+ * drives the existing {@link SystemUpdateService.requestUpdate} with an explicit,
+ * eligibility-vetted image tag.
+ *
+ * The window/pre-flight helpers are intentionally generic so a future PR can reuse
+ * them to auto-update installed apps (driving DockerService.updateContainer instead).
+ */
+@inject()
+export class AutoUpdateService {
+  constructor(
+    private dockerService: DockerService,
+    private downloadService: DownloadService,
+    private systemService: SystemService,
+    private systemUpdateService: SystemUpdateService,
+    private containerRegistryService: ContainerRegistryService
+  ) {}
+
+  /** In-process cache of the last successful releases fetch (per-process). */
+  private static releasesCache: { releases: GithubRelease[]; at: number } | null = null
+  /** Timestamp of the last failed fetch, for short-lived negative caching. */
+  private static releasesFailureAt = 0
+
+  /** Read user-configurable settings, applying defaults. */
+  async getConfig(): Promise<AutoUpdateConfig> {
+    const [enabled, windowStart, windowEnd, cooloffHours] = await Promise.all([
+      KVStore.getValue('autoUpdate.enabled'),
+      KVStore.getValue('autoUpdate.windowStart'),
+      KVStore.getValue('autoUpdate.windowEnd'),
+      KVStore.getValue('autoUpdate.cooloffHours'),
+    ])
+
+    const parsedCooloff = Number(cooloffHours)
+    return {
+      enabled: enabled ?? false,
+      windowStart: windowStart || DEFAULT_WINDOW_START,
+      windowEnd: windowEnd || DEFAULT_WINDOW_END,
+      // `Number(null) === 0`, so an *unset* value must fall through to the default
+      // rather than silently resolving to a zero cool-off. An explicit 0 is honored.
+      cooloffHours:
+        cooloffHours != null && Number.isFinite(parsedCooloff) && parsedCooloff >= 0
+          ? parsedCooloff
+          : DEFAULT_COOLOFF_HOURS,
+    }
+  }
+
+  /**
+   * Determine whether `now` falls inside the configured update window. The window
+   * is interpreted in the container's local time (set via the TZ env var). Windows
+   * that wrap past midnight (start > end, e.g. 22:00-02:00) are handled.
+   */
+  isWithinWindow(config: AutoUpdateConfig, now: DateTime = DateTime.now()): boolean {
+    return isWithinWindowUtil(config.windowStart, config.windowEnd, now)
+  }
+
+  /**
+   * Find the newest release that is safe to auto-apply: same major version as the
+   * running build (major bumps are deliberately left for manual update), strictly
+   * newer than current, and published at least `cooloffHours` ago. Prereleases and
+   * drafts are ignored — auto-update never rides early access.
+   *
+   * Returns null when nothing qualifies (e.g. only a major bump is newer, or the
+   * newest eligible release is still inside its cool-off window).
+   */
+  async getEligibleTarget(config: AutoUpdateConfig): Promise<EligibleTarget | null> {
+    const releases = await this.fetchReleases()
+    return this.selectEligibleTarget(
+      releases,
+      SystemService.getAppVersion(),
+      config.cooloffHours,
+      DateTime.now()
+    )
+  }
+
+  /**
+   * Fetch the published GitHub releases for the NOMAD repo, cached in-process.
+   * A successful result is reused for {@link RELEASES_CACHE_TTL_MS} so repeated
+   * status-page loads don't each hit (and risk rate-limiting) the unauthenticated
+   * GitHub API. A recent failure is negatively cached for {@link RELEASES_FAILURE_TTL_MS}
+   * so back-to-back calls while offline don't each block on the request timeout.
+   */
+  async fetchReleases(): Promise<GithubRelease[]> {
+    const now = Date.now()
+    const cached = AutoUpdateService.releasesCache
+    if (cached && now - cached.at < RELEASES_CACHE_TTL_MS) {
+      return cached.releases
+    }
+    if (now - AutoUpdateService.releasesFailureAt < RELEASES_FAILURE_TTL_MS) {
+      throw new Error('GitHub releases fetch recently failed; backing off')
+    }
+
+    try {
+      const response = await axios.get(RELEASES_URL, {
+        headers: { Accept: 'application/vnd.github+json' },
+        timeout: 5000,
+      })
+      if (!Array.isArray(response.data)) {
+        throw new Error('Unexpected response from GitHub releases API')
+      }
+      AutoUpdateService.releasesCache = { releases: response.data, at: now }
+      return response.data
+    } catch (error) {
+      AutoUpdateService.releasesFailureAt = now
+      throw error
+    }
+  }
+
+  /**
+   * Pure selection of the newest auto-applicable release from a release list.
+   * Extracted so the dry-run command and tests can drive it with fixtures.
+   * Same major as `currentVersion`, strictly newer, published on/before
+   * `now - cooloffHours`, prereleases/drafts excluded. Returns null for dev
+   * builds or when nothing qualifies.
+   */
+  selectEligibleTarget(
+    releases: GithubRelease[],
+    currentVersion: string,
+    cooloffHours: number,
+    now: DateTime
+  ): EligibleTarget | null {
+    if (currentVersion === 'dev' || currentVersion === '0.0.0') {
+      return null
+    }
+    const currentMajor = parseMajorVersion(currentVersion)
+    const cutoff = now.minus({ hours: cooloffHours })
+
+    const candidates = releases
+      .filter((r) => r && !r.draft && !r.prerelease && r.tag_name && r.published_at)
+      .map((r) => ({
+        version: String(r.tag_name).replace(/^v/, '').trim(),
+        publishedAt: String(r.published_at),
+      }))
+      .filter((r) => SEMVER_TAG.test(r.version))
+      .filter((r) => parseMajorVersion(r.version) === currentMajor)
+      .filter((r) => isNewerVersion(r.version, currentVersion))
+      .filter((r) => DateTime.fromISO(r.publishedAt) <= cutoff)
+      .sort((a, b) => (isNewerVersion(a.version, b.version) ? -1 : 1))
+
+    const best = candidates[0]
+    if (!best) return null
+
+    return {
+      version: best.version,
+      tag: `v${best.version}`,
+      publishedAt: best.publishedAt,
+    }
+  }
+
+  /**
+   * Pre-flight checks that gate an auto-update. `skip` blockers are transient
+   * (retry next window, no penalty); `failure` blockers count toward the backoff
+   * that eventually auto-disables auto-update.
+   */
+  async runPreflight(targetTag: string): Promise<PreflightResult> {
+    const blockers: Blocker[] = []
+
+    // 1. Sidecar must be present to perform the update.
+    if (!this.systemUpdateService.isSidecarAvailable()) {
+      blockers.push({ reason: 'Update sidecar is not available', severity: 'failure' })
+    }
+
+    // 2. No system update already running.
+    const updateStatus = this.systemUpdateService.getUpdateStatus()
+    if (updateStatus && !['idle', 'complete', 'error'].includes(updateStatus.stage)) {
+      blockers.push({
+        reason: `A system update is already in progress (stage: ${updateStatus.stage})`,
+        severity: 'skip',
+      })
+    }
+
+    // 3. No content/model downloads in progress.
+    try {
+      const downloads = await this.downloadService.listDownloadJobs()
+      const active = downloads.filter(
+        (d) => !!d.status && ['waiting', 'active', 'delayed'].includes(d.status)
+      )
+      if (active.length > 0) {
+        blockers.push({
+          reason: `${active.length} download(s) in progress`,
+          severity: 'skip',
+        })
+      }
+    } catch (error) {
+      logger.warn(`[AutoUpdateService] Could not check active downloads: ${error.message}`)
+    }
+
+    // 4. No app (container) install/update in progress.
+    try {
+      const installing = await Service.query().whereNot('installation_status', 'idle')
+      if (installing.length > 0) {
+        blockers.push({
+          reason: `${installing.length} app install/update(s) in progress`,
+          severity: 'skip',
+        })
+      }
+    } catch (error) {
+      logger.warn(`[AutoUpdateService] Could not check app installations: ${error.message}`)
+    }
+
+    // 5. Sufficient host storage for the new image.
+    const diskBlocker = await this.checkDiskSpace(targetTag)
+    if (diskBlocker) blockers.push(diskBlocker)
+
+    return { ok: blockers.length === 0, blockers }
+  }
+
+  /** Returns a disk blocker if free space is insufficient, otherwise null. */
+  private async checkDiskSpace(targetTag: string): Promise<Blocker | null> {
+    const hostArch = await this.getHostArch()
+    return checkImageDiskSpace({
+      image: `${NOMAD_IMAGE_REPO}:${targetTag}`,
+      hostArch,
+      containerRegistryService: this.containerRegistryService,
+      systemService: this.systemService,
+    })
+  }
+
+  /** Map the Docker daemon's architecture string to OCI naming (amd64/arm64/...). */
+  private async getHostArch(): Promise<string> {
+    try {
+      const info = await this.dockerService.docker.info()
+      const arch = info.Architecture || ''
+      const archMap: Record<string, string> = {
+        x86_64: 'amd64',
+        aarch64: 'arm64',
+        armv7l: 'arm',
+        amd64: 'amd64',
+        arm64: 'arm64',
+      }
+      return archMap[arch] || arch.toLowerCase()
+    } catch {
+      return 'amd64'
+    }
+  }
+
+  /**
+   * Side-effect-free core of the decision pipeline. Resolves the effective config
+   * (settings, overridable), checks the window, finds an eligible target, and runs
+   * pre-flight — returning a verdict WITHOUT requesting an update or mutating any
+   * persisted state. Both {@link attempt} (production) and {@link dryRun} (testing)
+   * are built on this so a dry run faithfully reflects what a real run would do.
+   */
+  async evaluate(overrides: EvaluateOverrides = {}): Promise<AutoUpdateDecision> {
+    const baseConfig = await this.getConfig()
+    const config: AutoUpdateConfig = {
+      enabled: overrides.forceEnabled ?? baseConfig.enabled,
+      windowStart: overrides.windowStart ?? baseConfig.windowStart,
+      windowEnd: overrides.windowEnd ?? baseConfig.windowEnd,
+      cooloffHours: overrides.cooloffHours ?? baseConfig.cooloffHours,
+    }
+    const now = overrides.now ?? DateTime.now()
+    const currentVersion = overrides.currentVersion ?? SystemService.getAppVersion()
+
+    const base = {
+      enabled: config.enabled,
+      currentVersion,
+      config,
+      withinWindow: false,
+      eligibleTarget: null as EligibleTarget | null,
+      preflight: null as PreflightResult | null,
+    }
+
+    if (!config.enabled) {
+      return { ...base, outcome: 'disabled', reason: 'Auto-update is disabled' }
+    }
+
+    const withinWindow = this.isWithinWindow(config, now)
+    if (!withinWindow) {
+      return {
+        ...base,
+        outcome: 'outside-window',
+        reason: `Outside update window (${config.windowStart}-${config.windowEnd})`,
+      }
+    }
+
+    let eligibleTarget: EligibleTarget | null
+    try {
+      const releases = overrides.releases ?? (await this.fetchReleases())
+      eligibleTarget = this.selectEligibleTarget(releases, currentVersion, config.cooloffHours, now)
+    } catch (error) {
+      return {
+        ...base,
+        withinWindow,
+        outcome: 'eligibility-error',
+        reason: `Failed to determine eligible version: ${error.message}`,
+      }
+    }
+
+    if (!eligibleTarget) {
+      return {
+        ...base,
+        withinWindow,
+        outcome: 'no-eligible',
+        reason: 'No eligible minor/patch update available (or still in cool-off)',
+      }
+    }
+
+    const preflight = overrides.fakePreflight
+      ? overrides.fakePreflight
+      : overrides.skipPreflight
+        ? { ok: true, blockers: [] }
+        : await this.runPreflight(eligibleTarget.tag)
+
+    if (!preflight.ok) {
+      const summary = preflight.blockers.map((b) => b.reason).join('; ')
+      return {
+        ...base,
+        withinWindow,
+        eligibleTarget,
+        preflight,
+        outcome: 'blocked',
+        reason: `Pre-flight blocked: ${summary}`,
+      }
+    }
+
+    return {
+      ...base,
+      withinWindow,
+      eligibleTarget,
+      preflight,
+      outcome: 'ready',
+      reason: `Ready to update to ${eligibleTarget.tag}`,
+    }
+  }
+
+  /**
+   * Run the full decision pipeline WITHOUT requesting an update or recording any
+   * state. Accepts the same injectable overrides as {@link evaluate}, so callers can
+   * simulate any scenario (a given current version, a canned release list, a fixed
+   * clock, a forced window) and see exactly what a real run would decide.
+   */
+  async dryRun(overrides: EvaluateOverrides = {}): Promise<AutoUpdateDecision> {
+    return this.evaluate(overrides)
+  }
+
+  /**
+   * The entry point invoked by AutoUpdateJob. Evaluates the decision pipeline and,
+   * when everything passes, requests the update with the vetted tag — recording the
+   * outcome to the KVStore (for the UI) and applying failure backoff.
+   */
+  async attempt(): Promise<{ updated: boolean; reason: string }> {
+    const decision = await this.evaluate()
+
+    switch (decision.outcome) {
+      case 'disabled':
+        return { updated: false, reason: decision.reason }
+
+      case 'outside-window':
+      case 'no-eligible':
+      // A failed release lookup is transient (offline-first appliances are
+      // routinely without connectivity) — treat as a skip so it never trips the
+      // backoff that auto-disables the feature. Only real update-request failures
+      // (the `ready` case below) count toward MAX_CONSECUTIVE_FAILURES.
+      case 'eligibility-error':
+        await this.recordSkip(decision.reason)
+        return { updated: false, reason: decision.reason }
+
+      case 'blocked': {
+        const hasFailure = decision.preflight!.blockers.some((b) => b.severity === 'failure')
+        if (hasFailure) {
+          await this.recordFailure(decision.reason)
+        } else {
+          await this.recordSkip(decision.reason)
+        }
+        return { updated: false, reason: decision.reason }
+      }
+
+      case 'ready': {
+        const target = decision.eligibleTarget!
+        const result = await this.systemUpdateService.requestUpdate({
+          targetTag: target.tag,
+          requester: 'auto-update',
+        })
+
+        if (result.success) {
+          await this.recordSuccess(target)
+          logger.info(`[AutoUpdateService] Auto-update requested: ${target.tag}`)
+          return { updated: true, reason: `Update requested: ${target.tag}` }
+        }
+
+        await this.recordFailure(`Update request failed: ${result.message}`)
+        return { updated: false, reason: result.message }
+      }
+    }
+  }
+
+  // --- Outcome recording -----------------------------------------------------
+
+  private async recordSuccess(target: EligibleTarget): Promise<void> {
+    await KVStore.setValue('autoUpdate.lastAttemptAt', DateTime.now().toISO()!)
+    await KVStore.setValue('autoUpdate.lastResult', `Update requested: ${target.tag}`)
+    await KVStore.clearValue('autoUpdate.lastError')
+    await KVStore.setValue('autoUpdate.consecutiveFailures', '0')
+  }
+
+  private async recordSkip(reason: string): Promise<void> {
+    await KVStore.setValue('autoUpdate.lastAttemptAt', DateTime.now().toISO()!)
+    await KVStore.setValue('autoUpdate.lastResult', reason)
+    logger.info(`[AutoUpdateService] Skipped: ${reason}`)
+  }
+
+  private async recordFailure(reason: string): Promise<void> {
+    await KVStore.setValue('autoUpdate.lastAttemptAt', DateTime.now().toISO()!)
+    await KVStore.setValue('autoUpdate.lastResult', reason)
+    await KVStore.setValue('autoUpdate.lastError', reason)
+
+    const prior = Number(await KVStore.getValue('autoUpdate.consecutiveFailures')) || 0
+    const failures = prior + 1
+    await KVStore.setValue('autoUpdate.consecutiveFailures', String(failures))
+    logger.error(`[AutoUpdateService] Failure ${failures}/${MAX_CONSECUTIVE_FAILURES}: ${reason}`)
+
+    if (failures >= MAX_CONSECUTIVE_FAILURES) {
+      await KVStore.setValue('autoUpdate.enabled', false)
+      await KVStore.setValue(
+        'autoUpdate.autoDisabledReason',
+        `Auto-update disabled after ${failures} consecutive failures. Last error: ${reason}`
+      )
+      logger.error(
+        `[AutoUpdateService] Auto-update auto-disabled after ${failures} consecutive failures`
+      )
+    }
+  }
+
+  /** Full state snapshot for the settings UI. */
+  async getStatus(): Promise<AutoUpdateStatus> {
+    const config = await this.getConfig()
+    const currentVersion = SystemService.getAppVersion()
+
+    let eligibleTarget: EligibleTarget | null = null
+    try {
+      eligibleTarget = await this.getEligibleTarget(config)
+    } catch (error) {
+      logger.warn(`[AutoUpdateService] getStatus eligibility lookup failed: ${error.message}`)
+    }
+
+    const [lastAttemptAt, lastResult, lastError, consecutiveFailures, autoDisabledReason] =
+      await Promise.all([
+        KVStore.getValue('autoUpdate.lastAttemptAt'),
+        KVStore.getValue('autoUpdate.lastResult'),
+        KVStore.getValue('autoUpdate.lastError'),
+        KVStore.getValue('autoUpdate.consecutiveFailures'),
+        KVStore.getValue('autoUpdate.autoDisabledReason'),
+      ])
+
+    return {
+      ...config,
+      currentVersion,
+      withinWindow: this.isWithinWindow(config),
+      eligibleTarget,
+      lastAttemptAt: lastAttemptAt || null,
+      lastResult: lastResult || null,
+      lastError: lastError || null,
+      consecutiveFailures: Number(consecutiveFailures) || 0,
+      autoDisabledReason: autoDisabledReason || null,
+    }
+  }
+}

admin/app/services/benchmark_service.ts

@@ -0,0 +1,850 @@
+import { inject } from '@adonisjs/core'
+import logger from '@adonisjs/core/services/logger'
+import transmit from '@adonisjs/transmit/services/main'
+import si from 'systeminformation'
+import axios from 'axios'
+import { DateTime } from 'luxon'
+import BenchmarkResult from '#models/benchmark_result'
+import BenchmarkSetting from '#models/benchmark_setting'
+import { SystemService } from '#services/system_service'
+import type {
+  BenchmarkType,
+  BenchmarkStatus,
+  BenchmarkProgress,
+  HardwareInfo,
+  DiskType,
+  SystemScores,
+  AIScores,
+  SysbenchCpuResult,
+  SysbenchMemoryResult,
+  SysbenchDiskResult,
+  RepositorySubmission,
+  RepositorySubmitResponse,
+  RepositoryStats,
+} from '../../types/benchmark.js'
+import { randomUUID, createHmac } from 'node:crypto'
+import { DockerService } from './docker_service.js'
+import { SERVICE_NAMES } from '../../constants/service_names.js'
+import { BROADCAST_CHANNELS } from '../../constants/broadcast.js'
+import Dockerode from 'dockerode'
+
+// HMAC secret for signing submissions to the benchmark repository
+// This provides basic protection against casual API abuse.
+// Note: Since NOMAD is open source, a determined attacker could extract this.
+// For stronger protection, see challenge-response authentication.
+const BENCHMARK_HMAC_SECRET = '778ba65d0bc0e23119e5ffce4b3716648a7d071f0a47ec3f'
+
+// Re-export default weights for use in service
+const SCORE_WEIGHTS = {
+  ai_tokens_per_second: 0.30,
+  cpu: 0.25,
+  memory: 0.15,
+  ai_ttft: 0.10,
+  disk_read: 0.10,
+  disk_write: 0.10,
+}
+
+// Benchmark configuration constants
+const SYSBENCH_IMAGE = 'severalnines/sysbench:latest'
+const SYSBENCH_CONTAINER_NAME = 'nomad_benchmark_sysbench'
+
+// Reference model for AI benchmark - small but meaningful
+const AI_BENCHMARK_MODEL = 'llama3.2:1b'
+const AI_BENCHMARK_PROMPT = 'Explain recursion in programming in exactly 100 words.'
+
+// Reference scores for normalization (calibrated to 0-100 scale)
+// These represent "expected" scores for a mid-range system (score ~50)
+const REFERENCE_SCORES = {
+  cpu_events_per_second: 5000, // sysbench cpu events/sec for ~50 score
+  memory_ops_per_second: 5000000, // sysbench memory ops/sec for ~50 score
+  disk_read_mb_per_sec: 500, // 500 MB/s read for ~50 score
+  disk_write_mb_per_sec: 400, // 400 MB/s write for ~50 score
+  ai_tokens_per_second: 30, // 30 tok/s for ~50 score
+  ai_ttft_ms: 500, // 500ms time to first token for ~50 score (lower is better)
+}
+
+@inject()
+export class BenchmarkService {
+  private currentBenchmarkId: string | null = null
+  private currentStatus: BenchmarkStatus = 'idle'
+
+  constructor(private dockerService: DockerService) {}
+
+  /**
+   * Run a full benchmark suite
+   */
+  async runFullBenchmark(): Promise<BenchmarkResult> {
+    return this._runBenchmark('full', true)
+  }
+
+  /**
+   * Run system benchmarks only (CPU, memory, disk)
+   */
+  async runSystemBenchmarks(): Promise<BenchmarkResult> {
+    return this._runBenchmark('system', false)
+  }
+
+  /**
+   * Run AI benchmark only
+   */
+  async runAIBenchmark(): Promise<BenchmarkResult> {
+    return this._runBenchmark('ai', true)
+  }
+
+  /**
+   * Get the latest benchmark result
+   */
+  async getLatestResult(): Promise<BenchmarkResult | null> {
+    return await BenchmarkResult.query().orderBy('created_at', 'desc').first()
+  }
+
+  /**
+   * Get all benchmark results
+   */
+  async getAllResults(): Promise<BenchmarkResult[]> {
+    return await BenchmarkResult.query().orderBy('created_at', 'desc')
+  }
+
+  /**
+   * Get a specific benchmark result by ID
+   */
+  async getResultById(benchmarkId: string): Promise<BenchmarkResult | null> {
+    return await BenchmarkResult.findBy('benchmark_id', benchmarkId)
+  }
+
+  /**
+   * Submit benchmark results to central repository
+   */
+  async submitToRepository(benchmarkId?: string, anonymous?: boolean): Promise<RepositorySubmitResponse> {
+    const result = benchmarkId
+      ? await this.getResultById(benchmarkId)
+      : await this.getLatestResult()
+
+    if (!result) {
+      throw new Error('No benchmark result found to submit')
+    }
+
+    // Only allow full benchmarks with AI data to be submitted to repository
+    if (result.benchmark_type !== 'full') {
+      throw new Error('Only full benchmarks can be shared with the community. Run a Full Benchmark to share your results.')
+    }
+
+    if (!result.ai_tokens_per_second || result.ai_tokens_per_second <= 0) {
+      throw new Error('Benchmark must include AI performance data. Ensure AI Assistant is installed and run a Full Benchmark.')
+    }
+
+    if (result.submitted_to_repository) {
+      throw new Error('Benchmark result has already been submitted')
+    }
+
+    const submission: RepositorySubmission = {
+      cpu_model: result.cpu_model,
+      cpu_cores: result.cpu_cores,
+      cpu_threads: result.cpu_threads,
+      ram_gb: Math.round(result.ram_bytes / (1024 * 1024 * 1024)),
+      disk_type: result.disk_type,
+      gpu_model: result.gpu_model,
+      cpu_score: result.cpu_score,
+      memory_score: result.memory_score,
+      disk_read_score: result.disk_read_score,
+      disk_write_score: result.disk_write_score,
+      ai_tokens_per_second: result.ai_tokens_per_second,
+      ai_time_to_first_token: result.ai_time_to_first_token,
+      nomad_score: result.nomad_score,
+      nomad_version: SystemService.getAppVersion(),
+      benchmark_version: '1.0.0',
+      builder_tag: anonymous ? null : result.builder_tag,
+    }
+
+    try {
+      // Generate HMAC signature for submission verification
+      const timestamp = Date.now().toString()
+      const payload = timestamp + JSON.stringify(submission)
+      const signature = createHmac('sha256', BENCHMARK_HMAC_SECRET)
+        .update(payload)
+        .digest('hex')
+
+      const response = await axios.post(
+        'https://benchmark.projectnomad.us/api/v1/submit',
+        submission,
+        {
+          timeout: 30000,
+          headers: {
+            'X-NOMAD-Timestamp': timestamp,
+            'X-NOMAD-Signature': signature,
+          },
+        }
+      )
+
+      if (response.data.success) {
+        result.submitted_to_repository = true
+        result.submitted_at = DateTime.now()
+        result.repository_id = response.data.repository_id
+        await result.save()
+
+        await BenchmarkSetting.setValue('last_benchmark_run', new Date().toISOString())
+      }
+
+      return response.data as RepositorySubmitResponse
+    } catch (error) {
+      const detail = error.response?.data?.error || error.message || 'Unknown error'
+      const statusCode = error.response?.status
+      logger.error(`Failed to submit benchmark to repository: ${detail} (Status: ${statusCode})`)
+      
+      // Create an error with the status code attached for proper handling upstream
+      const err: any = new Error(`Failed to submit benchmark: ${detail}`)
+      err.statusCode = statusCode
+      throw err
+    }
+  }
+
+  /**
+   * Get comparison stats from central repository
+   */
+  async getComparisonStats(): Promise<RepositoryStats | null> {
+    try {
+      const response = await axios.get('https://benchmark.projectnomad.us/api/v1/stats', {
+        timeout: 10000,
+      })
+      return response.data as RepositoryStats
+    } catch (error) {
+      logger.warn(`Failed to fetch comparison stats: ${error.message}`)
+      return null
+    }
+  }
+
+  /**
+   * Get current benchmark status
+   */
+  getStatus(): { status: BenchmarkStatus; benchmarkId: string | null } {
+    return {
+      status: this.currentStatus,
+      benchmarkId: this.currentBenchmarkId,
+    }
+  }
+
+  /**
+   * Detect system hardware information
+   */
+  async getHardwareInfo(): Promise<HardwareInfo> {
+    this._updateStatus('detecting_hardware', 'Detecting system hardware...')
+
+    try {
+      const [cpu, mem, diskLayout, graphics] = await Promise.all([
+        si.cpu(),
+        si.mem(),
+        si.diskLayout(),
+        si.graphics(),
+      ])
+
+      // Determine disk type from primary disk
+      let diskType: DiskType = 'unknown'
+      if (diskLayout.length > 0) {
+        const primaryDisk = diskLayout[0]
+        if (primaryDisk.type?.toLowerCase().includes('nvme')) {
+          diskType = 'nvme'
+        } else if (primaryDisk.type?.toLowerCase().includes('ssd')) {
+          diskType = 'ssd'
+        } else if (primaryDisk.type?.toLowerCase().includes('hdd') || primaryDisk.interfaceType === 'SATA') {
+          // SATA could be SSD or HDD, check if it's rotational
+          diskType = 'hdd'
+        }
+      }
+
+      // Get GPU model (prefer discrete GPU with dedicated VRAM)
+      let gpuModel: string | null = null
+      if (graphics.controllers && graphics.controllers.length > 0) {
+        // First, look for discrete GPUs (NVIDIA, AMD discrete, or any with significant VRAM)
+        const discreteGpu = graphics.controllers.find((g) => {
+          const vendor = g.vendor?.toLowerCase() || ''
+          const model = g.model?.toLowerCase() || ''
+          // NVIDIA GPUs are always discrete
+          if (vendor.includes('nvidia') || model.includes('geforce') || model.includes('rtx') || model.includes('quadro')) {
+            return true
+          }
+          // AMD discrete GPUs (Radeon, not integrated APU graphics)
+          if ((vendor.includes('amd') || vendor.includes('ati')) &&
+              (model.includes('radeon') || model.includes('rx ') || model.includes('vega')) &&
+              !model.includes('graphics')) {
+            return true
+          }
+          // Any GPU with dedicated VRAM > 512MB is likely discrete
+          if (g.vram && g.vram > 512) {
+            return true
+          }
+          return false
+        })
+        gpuModel = discreteGpu?.model || graphics.controllers[0]?.model || null
+      }
+
+      // Fallback: Check Docker for nvidia runtime and query GPU model via nvidia-smi
+      if (!gpuModel) {
+        try {
+          const dockerInfo = await this.dockerService.docker.info()
+          const runtimes = dockerInfo.Runtimes || {}
+          if ('nvidia' in runtimes) {
+            logger.info('[BenchmarkService] NVIDIA container runtime detected, querying GPU model via nvidia-smi')
+
+            const systemService = new (await import('./system_service.js')).SystemService(this.dockerService)
+            const nvidiaInfo = await systemService.getNvidiaSmiInfo()
+            if (Array.isArray(nvidiaInfo) && nvidiaInfo.length > 0) {
+              gpuModel = nvidiaInfo[0].model
+            } else {
+              logger.warn(`[BenchmarkService] NVIDIA runtime detected but failed to get GPU info: ${typeof nvidiaInfo === 'string' ? nvidiaInfo : JSON.stringify(nvidiaInfo)}`)
+            }
+          }
+        } catch (dockerError) {
+          logger.warn(`[BenchmarkService] Could not query Docker info for GPU detection: ${dockerError.message}`)
+        }
+      }
+
+      // Fallback: Extract integrated GPU from CPU model name
+      if (!gpuModel) {
+        const cpuFullName = `${cpu.manufacturer} ${cpu.brand}`
+
+        // AMD APUs: e.g., "AMD Ryzen AI 9 HX 370 w/ Radeon 890M" -> "Radeon 890M"
+        const radeonMatch = cpuFullName.match(/w\/\s*(Radeon\s+\d+\w*)/i)
+        if (radeonMatch) {
+          gpuModel = radeonMatch[1]
+        }
+
+        // Intel Core Ultra: These have Intel Arc Graphics integrated
+        // e.g., "Intel Core Ultra 9 285HX" -> "Intel Arc Graphics (Integrated)"
+        if (!gpuModel && cpu.manufacturer?.toLowerCase().includes('intel')) {
+          if (cpu.brand?.toLowerCase().includes('core ultra')) {
+            gpuModel = 'Intel Arc Graphics (Integrated)'
+          }
+        }
+      }
+
+      // Fallback: AMD discrete cards. si.graphics() returns empty inside Docker for AMD,
+      // the nvidia-smi path doesn't apply, and the APU regex only catches integrated parts.
+      // SystemService.getSystemInfo() already handles AMD via the marker file + Ollama log
+      // probe added in PR #804, so reuse that plumbing rather than duplicating it here.
+      if (!gpuModel) {
+        try {
+          const systemService = new (await import('./system_service.js')).SystemService(this.dockerService)
+          const sysInfo = await systemService.getSystemInfo()
+          const sysGpuModel = sysInfo?.graphics?.controllers?.[0]?.model
+          if (sysGpuModel) {
+            gpuModel = sysGpuModel
+          }
+        } catch (sysError: any) {
+          logger.warn(`[BenchmarkService] system_service AMD fallback failed: ${sysError.message}`)
+        }
+      }
+
+      return {
+        cpu_model: `${cpu.manufacturer} ${cpu.brand}`,
+        cpu_cores: cpu.physicalCores,
+        cpu_threads: cpu.cores,
+        ram_bytes: mem.total,
+        disk_type: diskType,
+        gpu_model: gpuModel,
+      }
+    } catch (error) {
+      logger.error(`Error detecting hardware: ${error.message}`)
+      throw new Error(`Failed to detect hardware: ${error.message}`)
+    }
+  }
+
+  /**
+   * Main benchmark execution method
+   */
+  private async _runBenchmark(type: BenchmarkType, includeAI: boolean): Promise<BenchmarkResult> {
+    if (this.currentStatus !== 'idle') {
+      throw new Error('A benchmark is already running')
+    }
+
+    this.currentBenchmarkId = randomUUID()
+    this._updateStatus('starting', 'Starting benchmark...')
+
+    try {
+      // Detect hardware
+      const hardware = await this.getHardwareInfo()
+
+      // Run system benchmarks
+      let systemScores: SystemScores = {
+        cpu_score: 0,
+        memory_score: 0,
+        disk_read_score: 0,
+        disk_write_score: 0,
+      }
+
+      if (type === 'full' || type === 'system') {
+        systemScores = await this._runSystemBenchmarks()
+      }
+
+      // Run AI benchmark if requested and Ollama is available
+      let aiScores: Partial<AIScores> = {}
+      if (includeAI && (type === 'full' || type === 'ai')) {
+        try {
+          aiScores = await this._runAIBenchmark()
+        } catch (error) {
+          // For AI-only benchmarks, failing is fatal - don't save useless results with all zeros
+          if (type === 'ai') {
+            throw new Error(`AI benchmark failed: ${error.message}. Make sure AI Assistant is installed and running.`)
+          }
+          // For full benchmarks, AI is optional - continue without it
+          logger.warn(`AI benchmark skipped: ${error.message}`)
+        }
+      }
+
+      // Calculate NOMAD score
+      this._updateStatus('calculating_score', 'Calculating NOMAD score...')
+      const nomadScore = this._calculateNomadScore(systemScores, aiScores)
+
+      // Save result
+      const result = await BenchmarkResult.create({
+        benchmark_id: this.currentBenchmarkId,
+        benchmark_type: type,
+        cpu_model: hardware.cpu_model,
+        cpu_cores: hardware.cpu_cores,
+        cpu_threads: hardware.cpu_threads,
+        ram_bytes: hardware.ram_bytes,
+        disk_type: hardware.disk_type,
+        gpu_model: hardware.gpu_model,
+        cpu_score: systemScores.cpu_score,
+        memory_score: systemScores.memory_score,
+        disk_read_score: systemScores.disk_read_score,
+        disk_write_score: systemScores.disk_write_score,
+        ai_tokens_per_second: aiScores.ai_tokens_per_second || null,
+        ai_model_used: aiScores.ai_model_used || null,
+        ai_time_to_first_token: aiScores.ai_time_to_first_token || null,
+        nomad_score: nomadScore,
+        submitted_to_repository: false,
+      })
+
+      this._updateStatus('completed', 'Benchmark completed successfully')
+      this.currentStatus = 'idle'
+      this.currentBenchmarkId = null
+
+      return result
+    } catch (error) {
+      this._updateStatus('error', `Benchmark failed: ${error.message}`)
+      this.currentStatus = 'idle'
+      this.currentBenchmarkId = null
+      throw error
+    }
+  }
+
+  /**
+   * Run system benchmarks using sysbench in Docker
+   */
+  private async _runSystemBenchmarks(): Promise<SystemScores> {
+    // Ensure sysbench image is available
+    await this._ensureSysbenchImage()
+
+    // Run CPU benchmark
+    this._updateStatus('running_cpu', 'Running CPU benchmark...')
+    const cpuResult = await this._runSysbenchCpu()
+
+    // Run memory benchmark
+    this._updateStatus('running_memory', 'Running memory benchmark...')
+    const memoryResult = await this._runSysbenchMemory()
+
+    // Run disk benchmarks
+    this._updateStatus('running_disk_read', 'Running disk read benchmark...')
+    const diskReadResult = await this._runSysbenchDiskRead()
+
+    this._updateStatus('running_disk_write', 'Running disk write benchmark...')
+    const diskWriteResult = await this._runSysbenchDiskWrite()
+
+    // Normalize scores to 0-100 scale
+    return {
+      cpu_score: this._normalizeScore(cpuResult.events_per_second, REFERENCE_SCORES.cpu_events_per_second),
+      memory_score: this._normalizeScore(memoryResult.operations_per_second, REFERENCE_SCORES.memory_ops_per_second),
+      disk_read_score: this._normalizeScore(diskReadResult.read_mb_per_sec, REFERENCE_SCORES.disk_read_mb_per_sec),
+      disk_write_score: this._normalizeScore(diskWriteResult.write_mb_per_sec, REFERENCE_SCORES.disk_write_mb_per_sec),
+    }
+  }
+
+  /**
+   * Run AI benchmark using Ollama
+   */
+  private async _runAIBenchmark(): Promise<AIScores> {
+    try {
+
+    this._updateStatus('running_ai', 'Running AI benchmark...')
+
+    const ollamaAPIURL = await this.dockerService.getServiceURL(SERVICE_NAMES.OLLAMA)
+    if (!ollamaAPIURL) {
+      throw new Error('AI Assistant service location could not be determined. Ensure AI Assistant is installed and running.')
+    }
+
+    // Check if Ollama is available
+    try {
+      await axios.get(`${ollamaAPIURL}/api/tags`, { timeout: 5000 })
+    } catch (error) {
+      const errorCode = error.code || error.response?.status || 'unknown'
+      throw new Error(`Ollama is not running or not accessible (${errorCode}). Ensure AI Assistant is installed and running.`)
+    }
+
+    // Check if the benchmark model is available, pull if not
+    const ollamaService = new (await import('./ollama_service.js')).OllamaService()
+    const modelResponse = await ollamaService.downloadModel(AI_BENCHMARK_MODEL)
+    if (!modelResponse.success) {
+      throw new Error(`Model does not exist and failed to download: ${modelResponse.message}`)
+    }
+
+    // Run inference benchmark
+    const startTime = Date.now()
+
+      const response = await axios.post(
+        `${ollamaAPIURL}/api/generate`,
+        {
+          model: AI_BENCHMARK_MODEL,
+          prompt: AI_BENCHMARK_PROMPT,
+          stream: false,
+        },
+        { timeout: 120000 }
+      )
+
+      const endTime = Date.now()
+      const totalTime = (endTime - startTime) / 1000 // seconds
+
+      // Ollama returns eval_count (tokens generated) and eval_duration (nanoseconds)
+      if (response.data.eval_count && response.data.eval_duration) {
+        const tokenCount = response.data.eval_count
+        const evalDurationSeconds = response.data.eval_duration / 1e9
+        const tokensPerSecond = tokenCount / evalDurationSeconds
+
+        // Time to first token from prompt_eval_duration
+        const ttft = response.data.prompt_eval_duration
+          ? response.data.prompt_eval_duration / 1e6 // Convert to ms
+          : (totalTime * 1000) / 2 // Estimate if not available
+
+        return {
+          ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100,
+          ai_model_used: AI_BENCHMARK_MODEL,
+          ai_time_to_first_token: Math.round(ttft * 100) / 100,
+        }
+      }
+
+      // Fallback calculation
+      const estimatedTokens = response.data.response?.split(' ').length * 1.3 || 100
+      const tokensPerSecond = estimatedTokens / totalTime
+
+      return {
+        ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100,
+        ai_model_used: AI_BENCHMARK_MODEL,
+        ai_time_to_first_token: Math.round((totalTime * 1000) / 2),
+      }
+    } catch (error) {
+      throw new Error(`AI benchmark failed: ${error.message}`)
+    }
+  }
+
+  /**
+   * Calculate weighted NOMAD score
+   */
+  private _calculateNomadScore(systemScores: SystemScores, aiScores: Partial<AIScores>): number {
+    let totalWeight = 0
+    let weightedSum = 0
+
+    // CPU score
+    weightedSum += systemScores.cpu_score * SCORE_WEIGHTS.cpu
+    totalWeight += SCORE_WEIGHTS.cpu
+
+    // Memory score
+    weightedSum += systemScores.memory_score * SCORE_WEIGHTS.memory
+    totalWeight += SCORE_WEIGHTS.memory
+
+    // Disk scores
+    weightedSum += systemScores.disk_read_score * SCORE_WEIGHTS.disk_read
+    totalWeight += SCORE_WEIGHTS.disk_read
+    weightedSum += systemScores.disk_write_score * SCORE_WEIGHTS.disk_write
+    totalWeight += SCORE_WEIGHTS.disk_write
+
+    // AI scores (if available)
+    if (aiScores.ai_tokens_per_second !== undefined && aiScores.ai_tokens_per_second !== null) {
+      const aiScore = this._normalizeScore(
+        aiScores.ai_tokens_per_second,
+        REFERENCE_SCORES.ai_tokens_per_second
+      )
+      weightedSum += aiScore * SCORE_WEIGHTS.ai_tokens_per_second
+      totalWeight += SCORE_WEIGHTS.ai_tokens_per_second
+    }
+
+    if (aiScores.ai_time_to_first_token !== undefined && aiScores.ai_time_to_first_token !== null) {
+      // For TTFT, lower is better, so we invert the score
+      const ttftScore = this._normalizeScoreInverse(
+        aiScores.ai_time_to_first_token,
+        REFERENCE_SCORES.ai_ttft_ms
+      )
+      weightedSum += ttftScore * SCORE_WEIGHTS.ai_ttft
+      totalWeight += SCORE_WEIGHTS.ai_ttft
+    }
+
+    // Normalize by actual weight used (in case AI benchmarks were skipped)
+    const nomadScore = totalWeight > 0 ? (weightedSum / totalWeight) * 100 : 0
+
+    return Math.round(Math.min(100, Math.max(0, nomadScore)) * 100) / 100
+  }
+
+  /**
+   * Normalize a raw score to 0-100 scale using log scaling
+   * This provides diminishing returns for very high scores
+   */
+  private _normalizeScore(value: number, reference: number): number {
+    if (value <= 0) return 0
+    // Log scale with widened range: dividing log2 by 3 prevents scores from
+    // clamping to 0% for below-average hardware. Gives 50% at reference value.
+    const ratio = value / reference
+    const score = 50 * (1 + Math.log2(Math.max(0.01, ratio)) / 3)
+    return Math.min(100, Math.max(0, score)) / 100
+  }
+
+  /**
+   * Normalize a score where lower is better (like latency)
+   */
+  private _normalizeScoreInverse(value: number, reference: number): number {
+    if (value <= 0) return 1
+    // Inverse: lower values = higher scores, with widened log range
+    const ratio = reference / value
+    const score = 50 * (1 + Math.log2(Math.max(0.01, ratio)) / 3)
+    return Math.min(100, Math.max(0, score)) / 100
+  }
+
+  /**
+   * Ensure sysbench Docker image is available
+   */
+  private async _ensureSysbenchImage(): Promise<void> {
+    try {
+      await this.dockerService.docker.getImage(SYSBENCH_IMAGE).inspect()
+    } catch {
+      this._updateStatus('starting', `Pulling sysbench image...`)
+      await this.dockerService.pullImage(SYSBENCH_IMAGE)
+    }
+  }
+
+  /**
+   * Run sysbench CPU benchmark
+   */
+  private async _runSysbenchCpu(): Promise<SysbenchCpuResult> {
+    const output = await this._runSysbenchCommand([
+      'sysbench',
+      'cpu',
+      '--cpu-max-prime=20000',
+      '--threads=4',
+      '--time=30',
+      'run',
+    ])
+
+    // Parse output for events per second
+    const eventsMatch = output.match(/events per second:\s*([\d.]+)/i)
+    const totalTimeMatch = output.match(/total time:\s*([\d.]+)s/i)
+    const totalEventsMatch = output.match(/total number of events:\s*(\d+)/i)
+    logger.debug(`[BenchmarkService] CPU output parsing - events/s: ${eventsMatch?.[1]}, total_time: ${totalTimeMatch?.[1]}, total_events: ${totalEventsMatch?.[1]}`)
+
+    return {
+      events_per_second: eventsMatch ? parseFloat(eventsMatch[1]) : 0,
+      total_time: totalTimeMatch ? parseFloat(totalTimeMatch[1]) : 30,
+      total_events: totalEventsMatch ? parseInt(totalEventsMatch[1]) : 0,
+    }
+  }
+
+  /**
+   * Run sysbench memory benchmark
+   */
+  private async _runSysbenchMemory(): Promise<SysbenchMemoryResult> {
+    const output = await this._runSysbenchCommand([
+      'sysbench',
+      'memory',
+      '--memory-block-size=1K',
+      '--memory-total-size=10G',
+      '--threads=4',
+      'run',
+    ])
+
+    // Parse output
+    const opsMatch = output.match(/Total operations:\s*\d+\s*\(([\d.]+)\s*per second\)/i)
+    const transferMatch = output.match(/([\d.]+)\s*MiB\/sec/i)
+    const timeMatch = output.match(/total time:\s*([\d.]+)s/i)
+
+    return {
+      operations_per_second: opsMatch ? parseFloat(opsMatch[1]) : 0,
+      transfer_rate_mb_per_sec: transferMatch ? parseFloat(transferMatch[1]) : 0,
+      total_time: timeMatch ? parseFloat(timeMatch[1]) : 0,
+    }
+  }
+
+  /**
+   * Run sysbench disk read benchmark
+   */
+  private async _runSysbenchDiskRead(): Promise<SysbenchDiskResult> {
+    // Run prepare, test, and cleanup in a single container
+    // This is necessary because each container has its own filesystem
+    const output = await this._runSysbenchCommand([
+      'sh',
+      '-c',
+      'sysbench fileio --file-total-size=1G --file-num=4 prepare && ' +
+        'sysbench fileio --file-total-size=1G --file-num=4 --file-test-mode=seqrd --time=30 run && ' +
+        'sysbench fileio --file-total-size=1G --file-num=4 cleanup',
+    ])
+
+    // Parse output - look for the Throughput section
+    const readMatch = output.match(/read,\s*MiB\/s:\s*([\d.]+)/i)
+    const readsPerSecMatch = output.match(/reads\/s:\s*([\d.]+)/i)
+
+    logger.debug(`[BenchmarkService] Disk read output parsing - read: ${readMatch?.[1]}, reads/s: ${readsPerSecMatch?.[1]}`)
+
+    return {
+      reads_per_second: readsPerSecMatch ? parseFloat(readsPerSecMatch[1]) : 0,
+      writes_per_second: 0,
+      read_mb_per_sec: readMatch ? parseFloat(readMatch[1]) : 0,
+      write_mb_per_sec: 0,
+      total_time: 30,
+    }
+  }
+
+  /**
+   * Run sysbench disk write benchmark
+   */
+  private async _runSysbenchDiskWrite(): Promise<SysbenchDiskResult> {
+    // Run prepare, test, and cleanup in a single container
+    // This is necessary because each container has its own filesystem
+    const output = await this._runSysbenchCommand([
+      'sh',
+      '-c',
+      'sysbench fileio --file-total-size=1G --file-num=4 prepare && ' +
+        'sysbench fileio --file-total-size=1G --file-num=4 --file-test-mode=seqwr --time=30 run && ' +
+        'sysbench fileio --file-total-size=1G --file-num=4 cleanup',
+    ])
+
+    // Parse output - look for the Throughput section
+    const writeMatch = output.match(/written,\s*MiB\/s:\s*([\d.]+)/i)
+    const writesPerSecMatch = output.match(/writes\/s:\s*([\d.]+)/i)
+
+    logger.debug(`[BenchmarkService] Disk write output parsing - written: ${writeMatch?.[1]}, writes/s: ${writesPerSecMatch?.[1]}`)
+
+    return {
+      reads_per_second: 0,
+      writes_per_second: writesPerSecMatch ? parseFloat(writesPerSecMatch[1]) : 0,
+      read_mb_per_sec: 0,
+      write_mb_per_sec: writeMatch ? parseFloat(writeMatch[1]) : 0,
+      total_time: 30,
+    }
+  }
+
+  /**
+   * Run a sysbench command in a Docker container
+   */
+  private async _runSysbenchCommand(cmd: string[]): Promise<string> {
+    let container: Dockerode.Container | null = null
+    try {
+      // Create container with TTY to avoid multiplexed output
+      container = await this.dockerService.docker.createContainer({
+        Image: SYSBENCH_IMAGE,
+        Cmd: cmd,
+        name: `${SYSBENCH_CONTAINER_NAME}_${Date.now()}`,
+        Tty: true, // Important: prevents multiplexed stdout/stderr headers
+        HostConfig: {
+          AutoRemove: false, // Don't auto-remove to avoid race condition with fetching logs
+        },
+      })
+
+      // Start container
+      await container.start()
+
+      // Wait for completion
+      await container.wait()
+      
+      // Get logs after container has finished
+      const logs = await container.logs({
+        stdout: true,
+        stderr: true,
+      })
+
+      // Parse logs (Docker logs include header bytes)
+      const output = logs.toString('utf8')
+        .replace(/[\x00-\x08]/g, '') // Remove control characters
+        .trim()
+
+      // Manually remove the container after getting logs
+      try {
+        await container.remove()
+      } catch (removeError) {
+        // Log but don't fail if removal fails (container might already be gone)
+        logger.warn(`Failed to remove sysbench container: ${removeError.message}`)
+      }
+
+      return output
+    } catch (error) {
+      // Clean up container on error if it exists
+      if (container) {
+        try {
+          await container.remove({ force: true })
+        } catch (removeError) {
+          // Ignore removal errors
+        }
+      }
+      logger.error(`Sysbench command failed: ${error.message}`)
+      throw new Error(`Sysbench command failed: ${error.message}`)
+    }
+  }
+
+  /**
+   * Broadcast benchmark progress update
+   */
+  private _updateStatus(status: BenchmarkStatus, message: string) {
+    this.currentStatus = status
+
+    const progress: BenchmarkProgress = {
+      status,
+      progress: this._getProgressPercent(status),
+      message,
+      current_stage: this._getStageLabel(status),
+      timestamp: new Date().toISOString(),
+    }
+
+    transmit.broadcast(BROADCAST_CHANNELS.BENCHMARK_PROGRESS, {
+      benchmark_id: this.currentBenchmarkId,
+      ...progress,
+    })
+
+    logger.info(`[BenchmarkService] ${status}: ${message}`)
+  }
+
+  /**
+   * Get progress percentage for a given status
+   */
+  private _getProgressPercent(status: BenchmarkStatus): number {
+    const progressMap: Record<BenchmarkStatus, number> = {
+      idle: 0,
+      starting: 5,
+      detecting_hardware: 10,
+      running_cpu: 25,
+      running_memory: 40,
+      running_disk_read: 55,
+      running_disk_write: 70,
+      downloading_ai_model: 80,
+      running_ai: 85,
+      calculating_score: 95,
+      completed: 100,
+      error: 0,
+    }
+    return progressMap[status] || 0
+  }
+
+  /**
+   * Get human-readable stage label
+   */
+  private _getStageLabel(status: BenchmarkStatus): string {
+    const labelMap: Record<BenchmarkStatus, string> = {
+      idle: 'Idle',
+      starting: 'Starting',
+      detecting_hardware: 'Detecting Hardware',
+      running_cpu: 'CPU Benchmark',
+      running_memory: 'Memory Benchmark',
+      running_disk_read: 'Disk Read Test',
+      running_disk_write: 'Disk Write Test',
+      downloading_ai_model: 'Downloading AI Model',
+      running_ai: 'AI Inference Test',
+      calculating_score: 'Calculating Score',
+      completed: 'Complete',
+      error: 'Error',
+    }
+    return labelMap[status] || status
+  }
+}

admin/app/services/chat_service.ts

@@ -0,0 +1,289 @@
+import ChatSession from '#models/chat_session'
+import ChatMessage from '#models/chat_message'
+import KVStore from '#models/kv_store'
+import logger from '@adonisjs/core/services/logger'
+import { DateTime } from 'luxon'
+import { inject } from '@adonisjs/core'
+import { OllamaService } from './ollama_service.js'
+import { SYSTEM_PROMPTS } from '../../constants/ollama.js'
+import { toTitleCase } from '../utils/misc.js'
+
+@inject()
+export class ChatService {
+  constructor(private ollamaService: OllamaService) {}
+
+  async getAllSessions() {
+    try {
+      const sessions = await ChatSession.query().orderBy('updated_at', 'desc')
+      return sessions.map((session) => ({
+        id: session.id.toString(),
+        title: session.title,
+        model: session.model,
+        timestamp: session.updated_at.toJSDate(),
+        lastMessage: null, // Will be populated from messages if needed
+      }))
+    } catch (error) {
+      logger.error(
+        `[ChatService] Failed to get sessions: ${error instanceof Error ? error.message : error}`
+      )
+      return []
+    }
+  }
+
+  async getChatSuggestions() {
+    try {
+      const models = await this.ollamaService.getModels()
+      if (!models || models.length === 0) {
+        return [] // If no models are available, return empty suggestions
+      }
+
+      // Prefer the user's selected chat model. Fall back to the smallest
+      // installed model — picking the largest by file size is unsafe: if any
+      // installed model exceeds available VRAM (e.g. llama3.1:405b on a 96 GB
+      // GPU), Ollama spends minutes trying to load it and the request 500s.
+      // Suggestions are short prompts that don't benefit from a flagship model.
+      const lastModel = await KVStore.getValue('chat.lastModel')
+      const preferred = lastModel ? models.find((m) => m.name === lastModel) : undefined
+      const chosen =
+        preferred ??
+        models.reduce((prev, current) => (prev.size < current.size ? prev : current))
+
+      if (!chosen) {
+        return []
+      }
+
+      const response = await this.ollamaService.chat({
+        model: chosen.name,
+        messages: [
+          {
+            role: 'user',
+            content: SYSTEM_PROMPTS.chat_suggestions,
+          }
+        ],
+        stream: false,
+      })
+
+      if (response && response.message && response.message.content) {
+        const content = response.message.content.trim()
+        
+        // Handle both comma-separated and newline-separated formats
+        let suggestions: string[] = []
+        
+        // Try splitting by commas first
+        if (content.includes(',')) {
+          suggestions = content.split(',').map((s) => s.trim())
+        } 
+        // Fall back to newline separation
+        else {
+          suggestions = content
+            .split(/\r?\n/)
+            .map((s) => s.trim())
+            // Remove numbered list markers (1., 2., 3., etc.) and bullet points
+            .map((s) => s.replace(/^\d+\.\s*/, '').replace(/^[-*•]\s*/, ''))
+            // Remove surrounding quotes if present
+            .map((s) => s.replace(/^["']|["']$/g, ''))
+        }
+        
+        // Filter out empty strings and limit to 3 suggestions
+        const filtered =  suggestions
+          .filter((s) => s.length > 0)
+          .slice(0, 3)
+
+        return filtered.map((s) => toTitleCase(s))
+      } else {
+        return []
+      }
+    } catch (error) {
+      logger.error(
+        `[ChatService] Failed to get chat suggestions: ${
+          error instanceof Error ? error.message : error
+        }`
+      )
+      return []
+    }
+  }
+
+  async getSession(sessionId: number) {
+    try {
+      const session = await ChatSession.query().where('id', sessionId).preload('messages').first()
+
+      if (!session) {
+        return null
+      }
+
+      return {
+        id: session.id.toString(),
+        title: session.title,
+        model: session.model,
+        timestamp: session.updated_at.toJSDate(),
+        messages: session.messages.map((msg) => ({
+          id: msg.id.toString(),
+          role: msg.role,
+          content: msg.content,
+          timestamp: msg.created_at.toJSDate(),
+        })),
+      }
+    } catch (error) {
+      logger.error(
+        `[ChatService] Failed to get session ${sessionId}: ${
+          error instanceof Error ? error.message : error
+        }`
+      )
+      return null
+    }
+  }
+
+  async createSession(title: string, model?: string) {
+    try {
+      const session = await ChatSession.create({
+        title,
+        model: model || null,
+      })
+
+      return {
+        id: session.id.toString(),
+        title: session.title,
+        model: session.model,
+        timestamp: session.created_at.toJSDate(),
+      }
+    } catch (error) {
+      logger.error(
+        `[ChatService] Failed to create session: ${error instanceof Error ? error.message : error}`
+      )
+      throw new Error('Failed to create chat session')
+    }
+  }
+
+  async updateSession(sessionId: number, data: { title?: string; model?: string }) {
+    try {
+      const session = await ChatSession.findOrFail(sessionId)
+
+      if (data.title) {
+        session.title = data.title
+      }
+      if (data.model !== undefined) {
+        session.model = data.model
+      }
+
+      await session.save()
+
+      return {
+        id: session.id.toString(),
+        title: session.title,
+        model: session.model,
+        timestamp: session.updated_at.toJSDate(),
+      }
+    } catch (error) {
+      logger.error(
+        `[ChatService] Failed to update session ${sessionId}: ${
+          error instanceof Error ? error.message : error
+        }`
+      )
+      throw new Error('Failed to update chat session')
+    }
+  }
+
+  async addMessage(sessionId: number, role: 'system' | 'user' | 'assistant', content: string) {
+    try {
+      const message = await ChatMessage.create({
+        session_id: sessionId,
+        role,
+        content,
+      })
+
+      // Update session's updated_at timestamp
+      const session = await ChatSession.findOrFail(sessionId)
+      session.updated_at = DateTime.now()
+      await session.save()
+
+      return {
+        id: message.id.toString(),
+        role: message.role,
+        content: message.content,
+        timestamp: message.created_at.toJSDate(),
+      }
+    } catch (error) {
+      logger.error(
+        `[ChatService] Failed to add message to session ${sessionId}: ${
+          error instanceof Error ? error.message : error
+        }`
+      )
+      throw new Error('Failed to add message')
+    }
+  }
+
+  async deleteSession(sessionId: number) {
+    try {
+      const session = await ChatSession.findOrFail(sessionId)
+      await session.delete()
+      return { success: true }
+    } catch (error) {
+      logger.error(
+        `[ChatService] Failed to delete session ${sessionId}: ${
+          error instanceof Error ? error.message : error
+        }`
+      )
+      throw new Error('Failed to delete chat session')
+    }
+  }
+
+  async getMessageCount(sessionId: number): Promise<number> {
+    try {
+      const count = await ChatMessage.query().where('session_id', sessionId).count('* as total')
+      return Number(count[0].$extras.total)
+    } catch (error) {
+      logger.error(
+        `[ChatService] Failed to get message count for session ${sessionId}: ${error instanceof Error ? error.message : error}`
+      )
+      return 0
+    }
+  }
+
+  async generateTitle(sessionId: number, userMessage: string, assistantMessage: string, model: string) {
+    try {
+      let title: string
+
+      const response = await this.ollamaService.chat({
+        model,
+        messages: [
+          { role: 'system', content: SYSTEM_PROMPTS.title_generation },
+          { role: 'user', content: userMessage },
+          { role: 'assistant', content: assistantMessage },
+        ],
+      })
+
+      title = response?.message?.content?.trim()
+      if (!title) {
+        title = userMessage.slice(0, 57) + (userMessage.length > 57 ? '...' : '')
+      }
+
+      await this.updateSession(sessionId, { title })
+      logger.info(`[ChatService] Generated title for session ${sessionId}: "${title}"`)
+    } catch (error) {
+      logger.error(
+        `[ChatService] Failed to generate title for session ${sessionId}: ${error instanceof Error ? error.message : error}`
+      )
+      // Fall back to truncated user message
+      try {
+        const fallbackTitle = userMessage.slice(0, 57) + (userMessage.length > 57 ? '...' : '')
+        await this.updateSession(sessionId, { title: fallbackTitle })
+      } catch {
+        // Silently fail - session keeps "New Chat" title
+      }
+    }
+  }
+
+  async deleteAllSessions() {
+    try {
+      await ChatSession.query().delete()
+      return { success: true, message: 'All chat sessions deleted' }
+    } catch (error) {
+      logger.error(
+        `[ChatService] Failed to delete all sessions: ${
+          error instanceof Error ? error.message : error
+        }`
+      )
+      throw new Error('Failed to delete all chat sessions')
+    }
+  }
+}

admin/app/services/collection_manifest_service.ts

@@ -0,0 +1,391 @@
+import axios from 'axios'
+import vine from '@vinejs/vine'
+import logger from '@adonisjs/core/services/logger'
+import { DateTime } from 'luxon'
+import { join } from 'path'
+import CollectionManifest from '#models/collection_manifest'
+import InstalledResource from '#models/installed_resource'
+import WikipediaSelection from '#models/wikipedia_selection'
+import { QueueService } from './queue_service.js'
+import { RunDownloadJob } from '#jobs/run_download_job'
+import { zimCategoriesSpecSchema, mapsSpecSchema, wikipediaSpecSchema } from '#validators/curated_collections'
+import {
+  ensureDirectoryExists,
+  listDirectoryContents,
+  getFileStatsIfExists,
+  ZIM_STORAGE_PATH,
+} from '../utils/fs.js'
+import type {
+  ManifestType,
+  ZimCategoriesSpec,
+  MapsSpec,
+  CategoryWithStatus,
+  CollectionWithStatus,
+  SpecResource,
+  SpecTier,
+} from '../../types/collections.js'
+
+const SPEC_URLS: Record<ManifestType, string> = {
+  zim_categories: 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/kiwix-categories.json',
+  maps: 'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/main/collections/maps.json',
+  wikipedia: 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/wikipedia.json',
+}
+
+const VALIDATORS: Record<ManifestType, any> = {
+  zim_categories: zimCategoriesSpecSchema,
+  maps: mapsSpecSchema,
+  wikipedia: wikipediaSpecSchema,
+}
+
+export class CollectionManifestService {
+  private readonly mapStoragePath = '/storage/maps'
+
+  // ---- Spec management ----
+
+  async fetchAndCacheSpec(type: ManifestType): Promise<boolean> {
+    try {
+      const response = await axios.get(SPEC_URLS[type], { timeout: 15000 })
+
+      const validated = await vine.validate({
+        schema: VALIDATORS[type],
+        data: response.data,
+      })
+
+      const existing = await CollectionManifest.find(type)
+      const specVersion = validated.spec_version
+
+      if (existing) {
+        const changed = existing.spec_version !== specVersion
+        existing.spec_version = specVersion
+        existing.spec_data = validated
+        existing.fetched_at = DateTime.now()
+        await existing.save()
+        return changed
+      }
+
+      await CollectionManifest.create({
+        type,
+        spec_version: specVersion,
+        spec_data: validated,
+        fetched_at: DateTime.now(),
+      })
+
+      return true
+    } catch (error) {
+      logger.error(`[CollectionManifestService] Failed to fetch spec for ${type}:`, error?.message || error)
+      return false
+    }
+  }
+
+  async getCachedSpec<T>(type: ManifestType): Promise<T | null> {
+    const manifest = await CollectionManifest.find(type)
+    if (!manifest) return null
+    return manifest.spec_data as T
+  }
+
+  async getSpecWithFallback<T>(type: ManifestType): Promise<T | null> {
+    try {
+      await this.fetchAndCacheSpec(type)
+    } catch {
+      // Fetch failed, will fall back to cache
+    }
+    return this.getCachedSpec<T>(type)
+  }
+
+  // ---- Status computation ----
+
+  async getCategoriesWithStatus(): Promise<CategoryWithStatus[]> {
+    const spec = await this.getSpecWithFallback<ZimCategoriesSpec>('zim_categories')
+    if (!spec) return []
+
+    const installedResources = await InstalledResource.query().where('resource_type', 'zim')
+    const installedMap = new Map(installedResources.map((r) => [r.resource_id, r]))
+
+    // In-flight ZIM download resource IDs from the BullMQ queue. Used to
+    // surface the user's tier intent immediately on submit, before any single
+    // file has finished downloading. Failed jobs are excluded so a stuck
+    // queue entry doesn't keep claiming the user's pick forever.
+    const inFlightIds = await this.getInFlightZimResourceIds()
+
+    return spec.categories.map((category) => {
+      const installedTierSlug = this.getInstalledTierForCategory(category.tiers, installedMap)
+      const downloadingTierSlug = this.getDownloadingTierForCategory(
+        category.tiers,
+        installedMap,
+        inFlightIds,
+        installedTierSlug
+      )
+      return { ...category, installedTierSlug, downloadingTierSlug }
+    })
+  }
+
+  private async getInFlightZimResourceIds(): Promise<Set<string>> {
+    const ids = new Set<string>()
+    try {
+      const queue = QueueService.getInstance().getQueue(RunDownloadJob.queue)
+      const jobs = await queue.getJobs(['waiting', 'active', 'delayed'])
+      for (const job of jobs) {
+        if (job.data?.filetype !== 'zim') continue
+        const resourceId = job.data?.resourceMetadata?.resource_id
+        if (typeof resourceId === 'string') ids.add(resourceId)
+      }
+    } catch (error) {
+      // Don't fail the whole categories endpoint if the queue is briefly
+      // unreachable — just report no in-flight downloads.
+      logger.warn('[CollectionManifestService] Could not read download queue:', error?.message || error)
+    }
+    return ids
+  }
+
+  /**
+   * Highest tier whose every resource is installed OR has an in-flight
+   * download. Returns undefined when there are no in-flight downloads for this
+   * category, or when the result would just duplicate installedTierSlug (i.e.
+   * everything that's downloading is already installed — nothing new to show).
+   */
+  getDownloadingTierForCategory(
+    tiers: SpecTier[],
+    installedMap: Map<string, InstalledResource>,
+    inFlightIds: Set<string>,
+    installedTierSlug: string | undefined
+  ): string | undefined {
+    if (inFlightIds.size === 0) return undefined
+
+    // Cheap pre-check: any of this category's resources actually in flight?
+    const anyInFlight = tiers.some((tier) =>
+      CollectionManifestService.resolveTierResources(tier, tiers).some((r) => inFlightIds.has(r.id))
+    )
+    if (!anyInFlight) return undefined
+
+    const reversedTiers = [...tiers].reverse()
+    for (const tier of reversedTiers) {
+      const resolved = CollectionManifestService.resolveTierResources(tier, tiers)
+      if (resolved.length === 0) continue
+      const allAccountedFor = resolved.every(
+        (r) => installedMap.has(r.id) || inFlightIds.has(r.id)
+      )
+      if (allAccountedFor) {
+        return tier.slug === installedTierSlug ? undefined : tier.slug
+      }
+    }
+    return undefined
+  }
+
+  async getMapCollectionsWithStatus(): Promise<CollectionWithStatus[]> {
+    const spec = await this.getSpecWithFallback<MapsSpec>('maps')
+    if (!spec) return []
+
+    const installedResources = await InstalledResource.query().where('resource_type', 'map')
+    const installedIds = new Set(installedResources.map((r) => r.resource_id))
+
+    return spec.collections.map((collection) => {
+      const installedCount = collection.resources.filter((r) => installedIds.has(r.id)).length
+      return {
+        ...collection,
+        all_installed: installedCount === collection.resources.length,
+        installed_count: installedCount,
+        total_count: collection.resources.length,
+      }
+    })
+  }
+
+  // ---- Tier resolution ----
+
+  static resolveTierResources(tier: SpecTier, allTiers: SpecTier[]): SpecResource[] {
+    const visited = new Set<string>()
+    return CollectionManifestService._resolveTierResourcesInner(tier, allTiers, visited)
+  }
+
+  private static _resolveTierResourcesInner(
+    tier: SpecTier,
+    allTiers: SpecTier[],
+    visited: Set<string>
+  ): SpecResource[] {
+    if (visited.has(tier.slug)) return [] // cycle detection
+    visited.add(tier.slug)
+
+    const resources: SpecResource[] = []
+
+    if (tier.includesTier) {
+      const included = allTiers.find((t) => t.slug === tier.includesTier)
+      if (included) {
+        resources.push(...CollectionManifestService._resolveTierResourcesInner(included, allTiers, visited))
+      }
+    }
+
+    resources.push(...tier.resources)
+    return resources
+  }
+
+  getInstalledTierForCategory(
+    tiers: SpecTier[],
+    installedMap: Map<string, InstalledResource>
+  ): string | undefined {
+    // Check from highest tier to lowest (tiers are ordered low to high in spec)
+    const reversedTiers = [...tiers].reverse()
+
+    for (const tier of reversedTiers) {
+      const resolved = CollectionManifestService.resolveTierResources(tier, tiers)
+      if (resolved.length === 0) continue
+
+      const allInstalled = resolved.every((r) => installedMap.has(r.id))
+      if (allInstalled) {
+        return tier.slug
+      }
+    }
+
+    return undefined
+  }
+
+  // ---- Filename parsing ----
+
+  static parseZimFilename(filename: string): { resource_id: string; version: string } | null {
+    const name = filename.replace(/\.zim$/, '')
+    const match = name.match(/^(.+)_(\d{4}-\d{2})$/)
+    if (!match) return null
+    return { resource_id: match[1], version: match[2] }
+  }
+
+  static parseMapFilename(filename: string): { resource_id: string; version: string } | null {
+    const name = filename.replace(/\.pmtiles$/, '')
+    const match = name.match(/^(.+)_(\d{4}-\d{2})$/)
+    if (!match) return null
+    return { resource_id: match[1], version: match[2] }
+  }
+
+  // ---- Filesystem reconciliation ----
+
+  async reconcileFromFilesystem(): Promise<{ zim: number; map: number }> {
+    let zimCount = 0
+    let mapCount = 0
+
+    console.log("RECONCILING FILESYSTEM MANIFESTS...")
+
+    // Reconcile ZIM files
+    try {
+      const zimDir = join(process.cwd(), ZIM_STORAGE_PATH)
+      await ensureDirectoryExists(zimDir)
+      const zimItems = await listDirectoryContents(zimDir)
+      const zimFiles = zimItems.filter((f) => f.name.endsWith('.zim'))
+
+      console.log(`Found ${zimFiles.length} ZIM files on disk. Reconciling with database...`)
+
+      // Get spec for URL lookup
+      const zimSpec = await this.getCachedSpec<ZimCategoriesSpec>('zim_categories')
+      const specResourceMap = new Map<string, SpecResource>()
+      if (zimSpec) {
+        for (const cat of zimSpec.categories) {
+          for (const tier of cat.tiers) {
+            for (const res of tier.resources) {
+              specResourceMap.set(res.id, res)
+            }
+          }
+        }
+      }
+
+      const seenZimIds = new Set<string>()
+
+      // Only skip the single Wikipedia file tracked by WikipediaSelection — not every file
+      // starting with `wikipedia_en_`. Curated category tiers (e.g. Medicine → Comprehensive)
+      // ship Wikipedia-themed ZIMs like `wikipedia_en_medicine_maxi` that must reconcile
+      // normally; otherwise their InstalledResource row gets wiped on every restart and the
+      // tier detection silently downgrades.
+      const wikipediaSelection = await WikipediaSelection.query().first()
+      const managedWikipediaFilename = wikipediaSelection?.filename ?? null
+
+      for (const file of zimFiles) {
+        console.log(`Processing ZIM file: ${file.name}`)
+        if (managedWikipediaFilename && file.name === managedWikipediaFilename) continue
+
+        const parsed = CollectionManifestService.parseZimFilename(file.name)
+        console.log(`Parsed ZIM filename:`, parsed)
+        if (!parsed) continue
+
+        seenZimIds.add(parsed.resource_id)
+
+        const specRes = specResourceMap.get(parsed.resource_id)
+        const filePath = join(zimDir, file.name)
+        const stats = await getFileStatsIfExists(filePath)
+
+        await InstalledResource.updateOrCreate(
+          { resource_id: parsed.resource_id, resource_type: 'zim' },
+          {
+            version: parsed.version,
+            url: specRes?.url || '',
+            file_path: filePath,
+            file_size_bytes: stats ? Number(stats.size) : null,
+            installed_at: DateTime.now(),
+          }
+        )
+        zimCount++
+      }
+
+      // Remove entries for ZIM files no longer on disk
+      const existingZim = await InstalledResource.query().where('resource_type', 'zim')
+      for (const entry of existingZim) {
+        if (!seenZimIds.has(entry.resource_id)) {
+          await entry.delete()
+        }
+      }
+    } catch (error) {
+      logger.error('[CollectionManifestService] Error reconciling ZIM files:', error)
+    }
+
+    // Reconcile map files
+    try {
+      const mapDir = join(process.cwd(), this.mapStoragePath, 'pmtiles')
+      await ensureDirectoryExists(mapDir)
+      const mapItems = await listDirectoryContents(mapDir)
+      const mapFiles = mapItems.filter((f) => f.name.endsWith('.pmtiles'))
+
+      // Get spec for URL/version lookup
+      const mapSpec = await this.getCachedSpec<MapsSpec>('maps')
+      const mapResourceMap = new Map<string, SpecResource>()
+      if (mapSpec) {
+        for (const col of mapSpec.collections) {
+          for (const res of col.resources) {
+            mapResourceMap.set(res.id, res)
+          }
+        }
+      }
+
+      const seenMapIds = new Set<string>()
+
+      for (const file of mapFiles) {
+        const parsed = CollectionManifestService.parseMapFilename(file.name)
+        if (!parsed) continue
+
+        seenMapIds.add(parsed.resource_id)
+
+        const specRes = mapResourceMap.get(parsed.resource_id)
+        const filePath = join(mapDir, file.name)
+        const stats = await getFileStatsIfExists(filePath)
+
+        await InstalledResource.updateOrCreate(
+          { resource_id: parsed.resource_id, resource_type: 'map' },
+          {
+            version: parsed.version,
+            url: specRes?.url || '',
+            file_path: filePath,
+            file_size_bytes: stats ? Number(stats.size) : null,
+            installed_at: DateTime.now(),
+          }
+        )
+        mapCount++
+      }
+
+      // Remove entries for map files no longer on disk
+      const existingMaps = await InstalledResource.query().where('resource_type', 'map')
+      for (const entry of existingMaps) {
+        if (!seenMapIds.has(entry.resource_id)) {
+          await entry.delete()
+        }
+      }
+    } catch (error) {
+      logger.error('[CollectionManifestService] Error reconciling map files:', error)
+    }
+
+    logger.info(`[CollectionManifestService] Reconciled ${zimCount} ZIM files, ${mapCount} map files`)
+    return { zim: zimCount, map: mapCount }
+  }
+}

admin/app/services/collection_update_service.ts

@@ -0,0 +1,179 @@
+import logger from '@adonisjs/core/services/logger'
+import axios from 'axios'
+import { DateTime } from 'luxon'
+import InstalledResource from '#models/installed_resource'
+import { RunDownloadJob } from '../jobs/run_download_job.js'
+import { ZIM_STORAGE_PATH } from '../utils/fs.js'
+import { join } from 'path'
+import type {
+  ResourceUpdateInfo,
+  ContentUpdateCheckResult,
+} from '../../types/collections.js'
+import { KiwixCatalogService, reconcileResourceUpdateState } from './kiwix_catalog_service.js'
+
+const MAP_STORAGE_PATH = '/storage/maps'
+
+const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
+const PMTILES_MIME_TYPES = ['application/vnd.pmtiles', 'application/octet-stream']
+
+export class CollectionUpdateService {
+  /**
+   * Check every installed resource against the upstream catalogs locally (Kiwix
+   * OPDS for ZIMs, GitHub for maps) — no longer routed through the external
+   * project-nomad-api. Side-effect: persists each resource's available-update
+   * state (version + cool-off anchor) so the auto-updater can act on it later.
+   */
+  async checkForUpdates(): Promise<ContentUpdateCheckResult> {
+    const installed = await InstalledResource.all()
+    if (installed.length === 0) {
+      return {
+        updates: [],
+        checked_at: new Date().toISOString(),
+      }
+    }
+
+    try {
+      const catalog = new KiwixCatalogService()
+      const latestByKey = await catalog.getLatestForResources(
+        installed.map((r) => ({ resource_id: r.resource_id, resource_type: r.resource_type }))
+      )
+
+      const now = DateTime.now()
+      const updates: ResourceUpdateInfo[] = []
+      for (const resource of installed) {
+        const latest = latestByKey.get(`${resource.resource_type}:${resource.resource_id}`) ?? null
+        await reconcileResourceUpdateState(resource, latest, now)
+
+        if (latest && latest.version > resource.version) {
+          updates.push({
+            resource_id: resource.resource_id,
+            resource_type: resource.resource_type,
+            installed_version: resource.version,
+            latest_version: latest.version,
+            download_url: latest.download_url,
+            size_bytes: latest.size_bytes || undefined,
+          })
+        }
+      }
+
+      logger.info(
+        `[CollectionUpdateService] Local update check complete: ${updates.length} update(s) available`
+      )
+
+      const enriched = await this.enrichWithSizes(updates)
+      return {
+        updates: enriched,
+        checked_at: new Date().toISOString(),
+      }
+    } catch (error) {
+      const message = error instanceof Error ? error.message : 'Unknown error during update check'
+      logger.error(`[CollectionUpdateService] Failed to check for updates: ${message}`)
+      return {
+        updates: [],
+        checked_at: new Date().toISOString(),
+        error: 'Failed to check for content updates. Please try again later.',
+      }
+    }
+  }
+
+  async applyUpdate(
+    update: ResourceUpdateInfo,
+    options?: { auto?: boolean }
+  ): Promise<{ success: boolean; jobId?: string; error?: string }> {
+    // Check if a download is already in progress for this URL
+    const existingJob = await RunDownloadJob.getByUrl(update.download_url)
+    if (existingJob) {
+      const state = await existingJob.getState()
+      if (state === 'active' || state === 'waiting' || state === 'delayed') {
+        return {
+          success: false,
+          error: `A download is already in progress for ${update.resource_id}`,
+        }
+      }
+    }
+
+    const filename = this.buildFilename(update)
+    const filepath = this.buildFilepath(update, filename)
+
+    const result = await RunDownloadJob.dispatch({
+      url: update.download_url,
+      filepath,
+      timeout: 30000,
+      allowedMimeTypes:
+        update.resource_type === 'zim' ? ZIM_MIME_TYPES : PMTILES_MIME_TYPES,
+      forceNew: true,
+      filetype: update.resource_type,
+      title: update.resource_id,
+      totalBytes: update.size_bytes,
+      resourceMetadata: {
+        resource_id: update.resource_id,
+        version: update.latest_version,
+        collection_ref: null,
+        auto: options?.auto ?? false,
+      },
+    })
+
+    if (!result || !result.job) {
+      return { success: false, error: 'Failed to dispatch download job' }
+    }
+
+    logger.info(
+      `[CollectionUpdateService] Dispatched update download for ${update.resource_id}: ${update.installed_version} → ${update.latest_version}`
+    )
+
+    return { success: true, jobId: result.job.id }
+  }
+
+  async applyAllUpdates(
+    updates: ResourceUpdateInfo[]
+  ): Promise<{ results: Array<{ resource_id: string; success: boolean; jobId?: string; error?: string }> }> {
+    const results = await Promise.all(
+      updates.map(async (update) => {
+        const result = await this.applyUpdate(update)
+        return { resource_id: update.resource_id, ...result }
+      })
+    )
+
+    return { results }
+  }
+
+  /**
+   * Fetch Content-Length for each update URL in parallel. HEAD failures are non-fatal —
+   * the update row just renders without a size. Bounded to HEAD_TIMEOUT_MS so a slow
+   * mirror doesn't block the whole check.
+   */
+  private async enrichWithSizes(updates: ResourceUpdateInfo[]): Promise<ResourceUpdateInfo[]> {
+    const HEAD_TIMEOUT_MS = 5000
+
+    return await Promise.all(
+      updates.map(async (update) => {
+        if (update.size_bytes) return update // Trust upstream if it already gave us one
+        try {
+          const head = await axios.head(update.download_url, {
+            timeout: HEAD_TIMEOUT_MS,
+            maxRedirects: 5,
+            validateStatus: (s) => s >= 200 && s < 400,
+          })
+          const len = Number(head.headers['content-length'])
+          return Number.isFinite(len) && len > 0 ? { ...update, size_bytes: len } : update
+        } catch {
+          return update
+        }
+      })
+    )
+  }
+
+  private buildFilename(update: ResourceUpdateInfo): string {
+    if (update.resource_type === 'zim') {
+      return `${update.resource_id}_${update.latest_version}.zim`
+    }
+    return `${update.resource_id}_${update.latest_version}.pmtiles`
+  }
+
+  private buildFilepath(update: ResourceUpdateInfo, filename: string): string {
+    if (update.resource_type === 'zim') {
+      return join(process.cwd(), ZIM_STORAGE_PATH, filename)
+    }
+    return join(process.cwd(), MAP_STORAGE_PATH, 'pmtiles', filename)
+  }
+}

admin/app/services/container_registry_service.ts

@@ -0,0 +1,550 @@
+import logger from '@adonisjs/core/services/logger'
+import { isNewerVersion, parseMajorVersion } from '../utils/version.js'
+
+export interface ParsedImageReference {
+  registry: string
+  namespace: string
+  repo: string
+  tag: string
+  /** Full name for registry API calls: namespace/repo */
+  fullName: string
+}
+
+export interface AvailableUpdate {
+  tag: string
+  isLatest: boolean
+  releaseUrl?: string
+}
+
+interface TokenCacheEntry {
+  token: string
+  expiresAt: number
+}
+
+const SEMVER_TAG_PATTERN = /^v?(\d+\.\d+(?:\.\d+)?)$/
+const PLATFORM_SUFFIXES = ['-arm64', '-amd64', '-alpine', '-slim', '-cuda', '-rocm']
+const REJECTED_TAGS = new Set(['latest', 'nightly', 'edge', 'dev', 'beta', 'alpha', 'canary', 'rc', 'test', 'debug'])
+
+export class ContainerRegistryService {
+  private tokenCache = new Map<string, TokenCacheEntry>()
+  private sourceUrlCache = new Map<string, string | null>()
+  private releaseTagPrefixCache = new Map<string, string>()
+
+  /**
+   * Parse a Docker image reference string into its components.
+   */
+  parseImageReference(image: string): ParsedImageReference {
+    let registry: string
+    let remainder: string
+    let tag = 'latest'
+
+    // Split off the tag
+    const lastColon = image.lastIndexOf(':')
+    if (lastColon > -1 && !image.substring(lastColon).includes('/')) {
+      tag = image.substring(lastColon + 1)
+      image = image.substring(0, lastColon)
+    }
+
+    // Determine registry vs image path
+    const parts = image.split('/')
+
+    if (parts.length === 1) {
+      // e.g. "nginx" → Docker Hub library image
+      registry = 'registry-1.docker.io'
+      remainder = `library/${parts[0]}`
+    } else if (parts.length === 2 && !parts[0].includes('.') && !parts[0].includes(':')) {
+      // e.g. "ollama/ollama" → Docker Hub user image
+      registry = 'registry-1.docker.io'
+      remainder = image
+    } else {
+      // e.g. "ghcr.io/kiwix/kiwix-serve" → custom registry
+      registry = parts[0]
+      remainder = parts.slice(1).join('/')
+    }
+
+    const namespaceParts = remainder.split('/')
+    const repo = namespaceParts.pop()!
+    const namespace = namespaceParts.join('/')
+
+    return {
+      registry,
+      namespace,
+      repo,
+      tag,
+      fullName: remainder,
+    }
+  }
+
+  /**
+   * Get an anonymous auth token for the given registry and repository.
+   * NOTE: This could be expanded in the future to support private repo authentication
+   */
+  private async getToken(registry: string, fullName: string): Promise<string> {
+    const cacheKey = `${registry}/${fullName}`
+    const cached = this.tokenCache.get(cacheKey)
+    if (cached && cached.expiresAt > Date.now()) {
+      return cached.token
+    }
+
+    let tokenUrl: string
+    if (registry === 'registry-1.docker.io') {
+      tokenUrl = `https://auth.docker.io/token?service=registry.docker.io&scope=repository:${fullName}:pull`
+    } else if (registry === 'ghcr.io') {
+      tokenUrl = `https://ghcr.io/token?service=ghcr.io&scope=repository:${fullName}:pull`
+    } else {
+      // For other registries, try the standard v2 token endpoint
+      tokenUrl = `https://${registry}/token?service=${registry}&scope=repository:${fullName}:pull`
+    }
+
+    const response = await this.fetchWithRetry(tokenUrl)
+    if (!response.ok) {
+      throw new Error(`Failed to get auth token from ${registry}: ${response.status}`)
+    }
+
+    const data = (await response.json()) as { token?: string; access_token?: string }
+    const token = data.token || data.access_token || ''
+
+    if (!token) {
+      throw new Error(`No token returned from ${registry}`)
+    }
+
+    // Cache for 5 minutes (tokens usually last longer, but be conservative)
+    this.tokenCache.set(cacheKey, {
+      token,
+      expiresAt: Date.now() + 5 * 60 * 1000,
+    })
+
+    return token
+  }
+
+  /**
+   * List all tags for a given image from the registry.
+   */
+  async listTags(parsed: ParsedImageReference): Promise<string[]> {
+    const token = await this.getToken(parsed.registry, parsed.fullName)
+    const allTags: string[] = []
+    let url = `https://${parsed.registry}/v2/${parsed.fullName}/tags/list?n=1000`
+
+    while (url) {
+      const response = await this.fetchWithRetry(url, {
+        headers: { Authorization: `Bearer ${token}` },
+      })
+
+      if (!response.ok) {
+        throw new Error(`Failed to list tags for ${parsed.fullName}: ${response.status}`)
+      }
+
+      const data = (await response.json()) as { tags?: string[] }
+      if (data.tags) {
+        allTags.push(...data.tags)
+      }
+
+      // Handle pagination via Link header. Per the OCI/Docker registry spec the next-page
+      // URL is relative (e.g. "/v2/<repo>/tags/list?last=<tag>&n=1000"), so it must be
+      // resolved against the registry origin before re-fetching — assigning the raw relative
+      // path to `url` makes fetch() throw "Failed to parse URL". This silently broke update
+      // checks for any repo with >1000 tags (e.g. ollama/ollama, filebrowser/filebrowser),
+      // which is the root cause of #945. new URL(relative, base) also passes absolute
+      // next-URLs through unchanged, so it's safe for registries that return those.
+      const linkHeader = response.headers.get('link')
+      if (linkHeader) {
+        const match = linkHeader.match(/<([^>]+)>;\s*rel="next"/)
+        url = match ? new URL(match[1], `https://${parsed.registry}`).toString() : ''
+      } else {
+        url = ''
+      }
+    }
+
+    return allTags
+  }
+
+  /**
+   * Check if a specific tag supports the given architecture by fetching its manifest.
+   */
+  async checkArchSupport(parsed: ParsedImageReference, tag: string, hostArch: string): Promise<boolean> {
+    try {
+      const token = await this.getToken(parsed.registry, parsed.fullName)
+      const url = `https://${parsed.registry}/v2/${parsed.fullName}/manifests/${tag}`
+
+      const response = await this.fetchWithRetry(url, {
+        headers: {
+          Authorization: `Bearer ${token}`,
+          Accept: [
+            'application/vnd.oci.image.index.v1+json',
+            'application/vnd.docker.distribution.manifest.list.v2+json',
+            'application/vnd.oci.image.manifest.v1+json',
+            'application/vnd.docker.distribution.manifest.v2+json',
+          ].join(', '),
+        },
+      })
+
+      if (!response.ok) return true // If we can't check, assume it's compatible
+
+      const manifest = (await response.json()) as {
+        mediaType?: string
+        manifests?: Array<{ platform?: { architecture?: string } }>
+      }
+      const mediaType = manifest.mediaType || response.headers.get('content-type') || ''
+
+      // Manifest list — check if any platform matches
+      if (
+        mediaType.includes('manifest.list') ||
+        mediaType.includes('image.index') ||
+        manifest.manifests
+      ) {
+        const manifests = manifest.manifests || []
+        return manifests.some(
+          (m: any) => m.platform && m.platform.architecture === hostArch
+        )
+      }
+
+      // Single manifest — assume compatible (can't easily determine arch without fetching config blob)
+      return true
+    } catch (error) {
+      logger.warn(`[ContainerRegistryService] Error checking arch for ${tag}: ${error.message}`)
+      return true // Assume compatible on error
+    }
+  }
+
+  /**
+   * Estimate the compressed download size (in bytes) of an image tag for the
+   * given host architecture by summing its layer sizes from the manifest.
+   *
+   * Resolves a multi-arch manifest list/index down to the platform-specific
+   * manifest before summing `layers[].size`. Returns null on any failure so
+   * callers can fall back to a conservative fixed threshold rather than
+   * silently skipping a disk pre-flight check.
+   */
+  async getImageDownloadSize(
+    parsed: ParsedImageReference,
+    tag: string,
+    hostArch: string
+  ): Promise<number | null> {
+    try {
+      const token = await this.getToken(parsed.registry, parsed.fullName)
+      const manifestAccept = [
+        'application/vnd.oci.image.index.v1+json',
+        'application/vnd.docker.distribution.manifest.list.v2+json',
+        'application/vnd.oci.image.manifest.v1+json',
+        'application/vnd.docker.distribution.manifest.v2+json',
+      ].join(', ')
+
+      const fetchManifest = async (ref: string) =>
+        this.fetchWithRetry(`https://${parsed.registry}/v2/${parsed.fullName}/manifests/${ref}`, {
+          headers: { Authorization: `Bearer ${token}`, Accept: manifestAccept },
+        })
+
+      const topRes = await fetchManifest(tag)
+      if (!topRes.ok) return null
+
+      let manifest = (await topRes.json()) as {
+        mediaType?: string
+        layers?: Array<{ size?: number }>
+        manifests?: Array<{ digest?: string; platform?: { architecture?: string } }>
+      }
+
+      // Multi-arch manifest list/index — resolve to the host-arch child manifest.
+      if (manifest.manifests?.length) {
+        const match =
+          manifest.manifests.find((m) => m.platform?.architecture === hostArch) ||
+          manifest.manifests[0]
+        if (!match?.digest) return null
+
+        const childRes = await fetchManifest(match.digest)
+        if (!childRes.ok) return null
+        manifest = (await childRes.json()) as { layers?: Array<{ size?: number }> }
+      }
+
+      if (!manifest.layers?.length) return null
+
+      return manifest.layers.reduce((total, layer) => total + (layer.size || 0), 0)
+    } catch (error) {
+      logger.warn(
+        `[ContainerRegistryService] Failed to get image size for ${parsed.fullName}:${tag}: ${error.message}`
+      )
+      return null
+    }
+  }
+
+  /**
+   * Extract the source repository URL from an image's OCI labels.
+   * Uses the standardized `org.opencontainers.image.source` label.
+   * Result is cached per image (not per tag).
+   */
+  async getSourceUrl(parsed: ParsedImageReference): Promise<string | null> {
+    const cacheKey = `${parsed.registry}/${parsed.fullName}`
+    if (this.sourceUrlCache.has(cacheKey)) {
+      return this.sourceUrlCache.get(cacheKey)!
+    }
+
+    try {
+      const token = await this.getToken(parsed.registry, parsed.fullName)
+
+      // First get the manifest to find the config blob digest
+      const manifestUrl = `https://${parsed.registry}/v2/${parsed.fullName}/manifests/${parsed.tag}`
+      const manifestRes = await this.fetchWithRetry(manifestUrl, {
+        headers: {
+          Authorization: `Bearer ${token}`,
+          Accept: [
+            'application/vnd.oci.image.manifest.v1+json',
+            'application/vnd.docker.distribution.manifest.v2+json',
+            'application/vnd.oci.image.index.v1+json',
+            'application/vnd.docker.distribution.manifest.list.v2+json',
+          ].join(', '),
+        },
+      })
+
+      if (!manifestRes.ok) {
+        this.sourceUrlCache.set(cacheKey, null)
+        return null
+      }
+
+      const manifest = (await manifestRes.json()) as {
+        config?: { digest?: string }
+        manifests?: Array<{ digest?: string; mediaType?: string; platform?: { architecture?: string } }>
+      }
+
+      // If this is a manifest list, pick the first manifest to get the config
+      let configDigest = manifest.config?.digest
+      if (!configDigest && manifest.manifests?.length) {
+        const firstManifest = manifest.manifests[0]
+        if (firstManifest.digest) {
+          const childRes = await this.fetchWithRetry(
+            `https://${parsed.registry}/v2/${parsed.fullName}/manifests/${firstManifest.digest}`,
+            {
+              headers: {
+                Authorization: `Bearer ${token}`,
+                Accept: 'application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json',
+              },
+            }
+          )
+          if (childRes.ok) {
+            const childManifest = (await childRes.json()) as { config?: { digest?: string } }
+            configDigest = childManifest.config?.digest
+          }
+        }
+      }
+
+      if (!configDigest) {
+        this.sourceUrlCache.set(cacheKey, null)
+        return null
+      }
+
+      // Fetch the config blob to read labels
+      const blobUrl = `https://${parsed.registry}/v2/${parsed.fullName}/blobs/${configDigest}`
+      const blobRes = await this.fetchWithRetry(blobUrl, {
+        headers: { Authorization: `Bearer ${token}` },
+      })
+
+      if (!blobRes.ok) {
+        this.sourceUrlCache.set(cacheKey, null)
+        return null
+      }
+
+      const config = (await blobRes.json()) as {
+        config?: { Labels?: Record<string, string> }
+      }
+
+      const sourceUrl = config.config?.Labels?.['org.opencontainers.image.source'] || null
+      this.sourceUrlCache.set(cacheKey, sourceUrl)
+      return sourceUrl
+    } catch (error) {
+      logger.warn(`[ContainerRegistryService] Failed to get source URL for ${cacheKey}: ${error.message}`)
+      this.sourceUrlCache.set(cacheKey, null)
+      return null
+    }
+  }
+
+  /**
+   * Detect whether a GitHub/GitLab repo uses a 'v' prefix on release tags.
+   * Probes the GitHub API with the current tag to determine the convention,
+   * then caches the result per source URL.
+   */
+  async detectReleaseTagPrefix(sourceUrl: string, sampleTag: string): Promise<string> {
+    if (this.releaseTagPrefixCache.has(sourceUrl)) {
+      return this.releaseTagPrefixCache.get(sourceUrl)!
+    }
+
+    try {
+      const url = new URL(sourceUrl)
+      if (url.hostname !== 'github.com') {
+        this.releaseTagPrefixCache.set(sourceUrl, '')
+        return ''
+      }
+
+      const cleanPath = url.pathname.replace(/\.git$/, '').replace(/\/$/, '')
+      const strippedTag = sampleTag.replace(/^v/, '')
+      const vTag = `v${strippedTag}`
+
+      // Try both variants against GitHub's API — the one that 200s tells us the convention
+      // Try v-prefixed first since it's more common
+      const vRes = await this.fetchWithRetry(
+        `https://api.github.com/repos${cleanPath}/releases/tags/${vTag}`,
+        { headers: { Accept: 'application/vnd.github.v3+json', 'User-Agent': 'ProjectNomad' } },
+        1
+      )
+      if (vRes.ok) {
+        this.releaseTagPrefixCache.set(sourceUrl, 'v')
+        return 'v'
+      }
+
+      const plainRes = await this.fetchWithRetry(
+        `https://api.github.com/repos${cleanPath}/releases/tags/${strippedTag}`,
+        { headers: { Accept: 'application/vnd.github.v3+json', 'User-Agent': 'ProjectNomad' } },
+        1
+      )
+      if (plainRes.ok) {
+        this.releaseTagPrefixCache.set(sourceUrl, '')
+        return ''
+      }
+    } catch {
+      // On error, fall through to default
+    }
+
+    // Default: no prefix modification
+    this.releaseTagPrefixCache.set(sourceUrl, '')
+    return ''
+  }
+
+  /**
+   * Build a release URL for a specific tag given a source repository URL and
+   * the detected release tag prefix convention.
+   * Supports GitHub and GitLab URL patterns.
+   */
+  buildReleaseUrl(sourceUrl: string, tag: string, releaseTagPrefix: string): string | undefined {
+    try {
+      const url = new URL(sourceUrl)
+      if (url.hostname === 'github.com' || url.hostname.includes('gitlab')) {
+        const cleanPath = url.pathname.replace(/\.git$/, '').replace(/\/$/, '')
+        const strippedTag = tag.replace(/^v/, '')
+        const releaseTag = releaseTagPrefix ? `${releaseTagPrefix}${strippedTag}` : strippedTag
+        return `${url.origin}${cleanPath}/releases/tag/${releaseTag}`
+      }
+    } catch {
+      // Invalid URL, skip
+    }
+    return undefined
+  }
+
+  /**
+   * Filter and sort tags to find compatible updates for a service.
+   */
+  filterCompatibleUpdates(
+    tags: string[],
+    currentTag: string,
+    majorVersion: number
+  ): string[] {
+    return tags
+      .filter((tag) => {
+        // Must match semver pattern
+        if (!SEMVER_TAG_PATTERN.test(tag)) return false
+
+        // Reject known non-version tags
+        if (REJECTED_TAGS.has(tag.toLowerCase())) return false
+
+        // Reject platform suffixes
+        if (PLATFORM_SUFFIXES.some((suffix) => tag.toLowerCase().endsWith(suffix))) return false
+
+        // Must be same major version
+        if (parseMajorVersion(tag) !== majorVersion) return false
+
+        // Must be newer than current
+        return isNewerVersion(tag, currentTag)
+      })
+      .sort((a, b) => (isNewerVersion(a, b) ? -1 : 1)) // Newest first
+  }
+
+  /**
+   * High-level method to get available updates for a service.
+   * Returns a sorted list of compatible newer versions (newest first).
+   */
+  async getAvailableUpdates(
+    containerImage: string,
+    hostArch: string,
+    fallbackSourceRepo?: string | null
+  ): Promise<AvailableUpdate[]> {
+    const parsed = this.parseImageReference(containerImage)
+    const currentTag = parsed.tag
+
+    if (currentTag === 'latest') {
+      logger.warn(
+        `[ContainerRegistryService] Cannot check updates for ${containerImage} — using :latest tag`
+      )
+      return []
+    }
+
+    const majorVersion = parseMajorVersion(currentTag)
+
+    // Fetch tags and source URL in parallel
+    const [tags, ociSourceUrl] = await Promise.all([
+      this.listTags(parsed),
+      this.getSourceUrl(parsed),
+    ])
+
+    // OCI label takes precedence, fall back to DB-stored source_repo
+    const sourceUrl = ociSourceUrl || fallbackSourceRepo || null
+
+    const compatible = this.filterCompatibleUpdates(tags, currentTag, majorVersion)
+
+    // Detect release tag prefix convention (e.g. 'v' vs no prefix) if we have a source URL
+    let releaseTagPrefix = ''
+    if (sourceUrl) {
+      releaseTagPrefix = await this.detectReleaseTagPrefix(sourceUrl, currentTag)
+    }
+
+    // Check architecture support for the top candidates (limit checks to save API calls)
+    const maxArchChecks = 10
+    const results: AvailableUpdate[] = []
+
+    for (const tag of compatible.slice(0, maxArchChecks)) {
+      const supported = await this.checkArchSupport(parsed, tag, hostArch)
+      if (supported) {
+        results.push({
+          tag,
+          isLatest: results.length === 0,
+          releaseUrl: sourceUrl ? this.buildReleaseUrl(sourceUrl, tag, releaseTagPrefix) : undefined,
+        })
+      }
+    }
+
+    // For remaining tags (beyond arch check limit), include them but mark as not latest
+    for (const tag of compatible.slice(maxArchChecks)) {
+      results.push({
+        tag,
+        isLatest: false,
+        releaseUrl: sourceUrl ? this.buildReleaseUrl(sourceUrl, tag, releaseTagPrefix) : undefined,
+      })
+    }
+
+    return results
+  }
+
+  /**
+   * Fetch with retry and exponential backoff for rate limiting.
+   */
+  private async fetchWithRetry(
+    url: string,
+    init?: RequestInit,
+    maxRetries = 3
+  ): Promise<Response> {
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+      const response = await fetch(url, init)
+
+      if (response.status === 429 && attempt < maxRetries) {
+        const retryAfter = response.headers.get('retry-after')
+        const delay = retryAfter
+          ? parseInt(retryAfter, 10) * 1000
+          : Math.pow(2, attempt) * 1000
+        logger.warn(
+          `[ContainerRegistryService] Rate limited on ${url}, retrying in ${delay}ms`
+        )
+        await new Promise((resolve) => setTimeout(resolve, delay))
+        continue
+      }
+
+      return response
+    }
+
+    throw new Error(`Failed to fetch ${url} after ${maxRetries} retries`)
+  }
+}

admin/app/services/content_auto_update_service.ts

@@ -0,0 +1,550 @@
+import logger from '@adonisjs/core/services/logger'
+import { DateTime } from 'luxon'
+import KVStore from '#models/kv_store'
+import InstalledResource from '#models/installed_resource'
+import { DownloadService } from '#services/download_service'
+import { CollectionUpdateService } from '#services/collection_update_service'
+import {
+  KiwixCatalogService,
+  reconcileResourceUpdateState,
+  type CatalogResult,
+} from '#services/kiwix_catalog_service'
+import { isWithinWindow, parseWindowMinutes } from '../utils/update_window.js'
+import { recordResourceUpdateFailure } from '../utils/content_auto_update_backoff.js'
+import type { Blocker, PreflightResult } from '../utils/image_disk_preflight.js'
+
+/**
+ * Content auto-update is opt-in via a single global master switch and runs on
+ * its OWN window + per-window data cap (deliberately separate from the core/app
+ * `autoUpdate.*` window, since ZIM downloads are multi-GB and bandwidth
+ * sensitive). Defaults err toward an overnight window with no cap; the UI
+ * strongly recommends setting a cap.
+ */
+const DEFAULT_WINDOW_START = '02:00'
+const DEFAULT_WINDOW_END = '05:00'
+const DEFAULT_COOLOFF_HOURS = 72
+
+/** Whole-feature failures (e.g. catalog unreachable) before it self-disables. */
+const MAX_FEATURE_FAILURES = 3
+
+export interface ContentAutoUpdateConfig {
+  /** Global master switch (`contentAutoUpdate.enabled`). */
+  enabled: boolean
+  windowStart: string
+  windowEnd: string
+  cooloffHours: number
+  /** Max NEW bytes initiated per window instance. 0 = unlimited. */
+  maxBytesPerWindow: number
+}
+
+/** Per-resource eligibility verdict (drives both selection and the status UI). */
+export interface ContentEligibility {
+  eligible: boolean
+  reason: string
+  cooloffRemainingHours: number | null
+}
+
+/** An eligible resource paired with the catalog facts needed to download it. */
+export interface ContentCandidate {
+  resource: InstalledResource
+  version: string
+  download_url: string
+  size_bytes: number
+  installed_at: DateTime
+}
+
+/** Outcome of the cap-bounded greedy selection. */
+export interface ContentSelection {
+  selected: ContentCandidate[]
+  /** Single files larger than the whole cap — never auto-started (manual only). */
+  skippedOversize: ContentCandidate[]
+  /** Fit the cap but not this window's remaining budget — retried next window. */
+  deferred: ContentCandidate[]
+}
+
+export interface ContentAutoUpdateResourceStatus {
+  resource_id: string
+  resource_type: 'zim' | 'map'
+  current_version: string
+  available_update_version: string | null
+  size_bytes: number | null
+  eligible: boolean
+  reason: string
+  cooloff_remaining_hours: number | null
+  exceeds_cap: boolean
+  consecutive_failures: number
+  auto_disabled_reason: string | null
+}
+
+export interface ContentAutoUpdateStatus extends ContentAutoUpdateConfig {
+  withinWindow: boolean
+  windowBytesUsed: number
+  lastAttemptAt: string | null
+  lastResult: string | null
+  lastError: string | null
+  autoDisabledReason: string | null
+  resources: ContentAutoUpdateResourceStatus[]
+}
+
+/**
+ * Decision + safety layer for automatic content (ZIM/map) updates. This is the
+ * content-side counterpart to {@link AppAutoUpdateService}: it decides *whether*
+ * each installed resource with an available update should be downloaded now
+ * (master switch on + in the content window + past cool-off + within the data
+ * cap) and then drives the existing manual download path
+ * ({@link CollectionUpdateService.applyUpdate} → {@link RunDownloadJob}).
+ *
+ * It never installs synchronously — it dispatches resumable download jobs and
+ * lets the existing job-completion path advance the installed version and
+ * rebuild the Kiwix library.
+ */
+export class ContentAutoUpdateService {
+  constructor(
+    private downloadService: DownloadService,
+    private catalog: KiwixCatalogService = new KiwixCatalogService(),
+    private collectionUpdateService: CollectionUpdateService = new CollectionUpdateService()
+  ) {}
+
+  /** Read the master switch plus the content-specific window/cool-off/cap. */
+  async getConfig(): Promise<ContentAutoUpdateConfig> {
+    const [enabled, windowStart, windowEnd, cooloffHours, maxBytes] = await Promise.all([
+      KVStore.getValue('contentAutoUpdate.enabled'),
+      KVStore.getValue('contentAutoUpdate.windowStart'),
+      KVStore.getValue('contentAutoUpdate.windowEnd'),
+      KVStore.getValue('contentAutoUpdate.cooloffHours'),
+      KVStore.getValue('contentAutoUpdate.maxBytesPerWindow'),
+    ])
+
+    const parsedCooloff = Number(cooloffHours)
+    const parsedCap = Number(maxBytes)
+    return {
+      enabled: enabled ?? false,
+      windowStart: windowStart || DEFAULT_WINDOW_START,
+      windowEnd: windowEnd || DEFAULT_WINDOW_END,
+      // `Number(null) === 0`, so an unset value must fall through to the default
+      // rather than silently resolving to a zero cool-off. An explicit 0 is honored.
+      cooloffHours:
+        cooloffHours !== null && Number.isFinite(parsedCooloff) && parsedCooloff >= 0
+          ? parsedCooloff
+          : DEFAULT_COOLOFF_HOURS,
+      maxBytesPerWindow:
+        maxBytes !== null && Number.isFinite(parsedCap) && parsedCap >= 0 ? parsedCap : 0,
+    }
+  }
+
+  /**
+   * Pure per-resource eligibility verdict. A resource is eligible when it has a
+   * detected newer version, is not self-disabled, and is past its cool-off
+   * (measured from first-detected). Version comparison is a lexicographic
+   * compare of the YYYY-MM stamps, which sorts chronologically.
+   */
+  resourceEligibility(
+    resource: InstalledResource,
+    cooloffHours: number,
+    now: DateTime
+  ): ContentEligibility {
+    if (!resource.available_update_version) {
+      return { eligible: false, reason: 'Up to date', cooloffRemainingHours: null }
+    }
+    if (resource.auto_update_disabled_reason) {
+      return {
+        eligible: false,
+        reason: 'Auto-update disabled after repeated failures',
+        cooloffRemainingHours: null,
+      }
+    }
+    if (!(resource.available_update_version > resource.version)) {
+      return { eligible: false, reason: 'Up to date', cooloffRemainingHours: null }
+    }
+    if (!resource.available_update_first_seen_at) {
+      return { eligible: false, reason: 'Cool-off pending', cooloffRemainingHours: cooloffHours }
+    }
+
+    const ageHours = now.diff(resource.available_update_first_seen_at, 'hours').hours
+    const remaining = cooloffHours - ageHours
+    if (remaining > 0) {
+      const rounded = Math.ceil(remaining)
+      return {
+        eligible: false,
+        reason: `In cool-off (${rounded}h remaining)`,
+        cooloffRemainingHours: rounded,
+      }
+    }
+
+    return {
+      eligible: true,
+      reason: `Eligible → ${resource.available_update_version}`,
+      cooloffRemainingHours: 0,
+    }
+  }
+
+  /**
+   * Pure cap-bounded greedy selection. Oldest-installed first (stale content is
+   * prioritized), tie-broken smallest-first for predictability.
+   *
+   * - size unknown (0) → deferred (can't budget safely)
+   * - size > the WHOLE cap → skippedOversize (never auto-started; manual only)
+   * - size ≤ remaining budget → selected
+   * - otherwise → deferred (fits the cap, not this window)
+   */
+  selectUnderCap(
+    candidates: ContentCandidate[],
+    capBytes: number,
+    usedBytes: number
+  ): ContentSelection {
+    const cap = capBytes > 0 ? capBytes : Number.POSITIVE_INFINITY
+    let remaining = Math.max(0, cap - usedBytes)
+
+    const selected: ContentCandidate[] = []
+    const skippedOversize: ContentCandidate[] = []
+    const deferred: ContentCandidate[] = []
+
+    const ordered = [...candidates].sort((a, b) => {
+      const at = a.installed_at?.toMillis?.() ?? 0
+      const bt = b.installed_at?.toMillis?.() ?? 0
+      if (at !== bt) return at - bt
+      return a.size_bytes - b.size_bytes
+    })
+
+    for (const candidate of ordered) {
+      if (candidate.size_bytes <= 0) {
+        deferred.push(candidate)
+      } else if (candidate.size_bytes > cap) {
+        skippedOversize.push(candidate)
+      } else if (candidate.size_bytes <= remaining) {
+        selected.push(candidate)
+        remaining -= candidate.size_bytes
+      } else {
+        deferred.push(candidate)
+      }
+    }
+
+    return { selected, skippedOversize, deferred }
+  }
+
+  /**
+   * Run-wide pre-flight: never auto-update content while ANY download is already
+   * running. Because content downloads are multi-GB and resumable, an in-flight
+   * download from a prior window naturally blocks new starts here — exactly the
+   * "let in-flight finish, don't start new" behavior we want. Transient → `skip`.
+   */
+  async runGlobalPreflight(): Promise<PreflightResult> {
+    const blockers: Blocker[] = []
+    try {
+      const downloads = await this.downloadService.listDownloadJobs()
+      const active = downloads.filter(
+        (d) => !!d.status && ['waiting', 'active', 'delayed'].includes(d.status)
+      )
+      if (active.length > 0) {
+        blockers.push({ reason: `${active.length} download(s) in progress`, severity: 'skip' })
+      }
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error)
+      logger.warn(`[ContentAutoUpdateService] Could not check active downloads: ${message}`)
+    }
+    return { ok: blockers.length === 0, blockers }
+  }
+
+  /**
+   * Entry point invoked by ContentAutoUpdateJob. Gates on the master switch +
+   * window, runs the local catalog check, then downloads as many eligible
+   * resources as fit under the per-window data cap.
+   */
+  async attempt(): Promise<{ started: number; reason: string }> {
+    const config = await this.getConfig()
+    const now = DateTime.now()
+
+    if (!config.enabled) {
+      return { started: 0, reason: 'Content auto-update is disabled' }
+    }
+    if (!isWithinWindow(config.windowStart, config.windowEnd, now)) {
+      const reason = `Outside update window (${config.windowStart}-${config.windowEnd})`
+      await this.recordRun(reason)
+      return { started: 0, reason }
+    }
+
+    try {
+      // Reset the per-window budget once per window instance (the cron fires
+      // hourly but a window can span several hours).
+      await this.maybeResetWindowBudget(config, now)
+
+      // Local catalog check + persist available-update state for every resource.
+      const installed = await InstalledResource.all()
+      const latestByKey = await this.catalog.getLatestForResources(
+        installed.map((r) => ({ resource_id: r.resource_id, resource_type: r.resource_type }))
+      )
+      for (const resource of installed) {
+        const latest = latestByKey.get(`${resource.resource_type}:${resource.resource_id}`) ?? null
+        await reconcileResourceUpdateState(resource, latest, now)
+      }
+
+      const eligible = installed.filter(
+        (r) => this.resourceEligibility(r, config.cooloffHours, now).eligible
+      )
+      if (eligible.length === 0) {
+        await this.recordFeatureSuccess()
+        const reason = 'No eligible content updates'
+        await this.recordRun(reason)
+        return { started: 0, reason }
+      }
+
+      const global = await this.runGlobalPreflight()
+      if (!global.ok) {
+        await this.recordFeatureSuccess()
+        const reason = `Pre-flight blocked: ${global.blockers.map((b) => b.reason).join('; ')}`
+        await this.recordRun(reason)
+        return { started: 0, reason }
+      }
+
+      const candidates: ContentCandidate[] = eligible.map((r) => {
+        const latest = latestByKey.get(`${r.resource_type}:${r.resource_id}`) as CatalogResult
+        return {
+          resource: r,
+          version: latest.version,
+          download_url: latest.download_url,
+          size_bytes: r.available_update_size_bytes ?? latest.size_bytes ?? 0,
+          installed_at: r.installed_at,
+        }
+      })
+
+      const usedBytes = await this.getWindowBytesUsed()
+      const { selected, skippedOversize, deferred } = this.selectUnderCap(
+        candidates,
+        config.maxBytesPerWindow,
+        usedBytes
+      )
+
+      let started = 0
+      let failed = 0
+      let initiatedBytes = 0
+      for (const candidate of selected) {
+        const result = await this.collectionUpdateService.applyUpdate(
+          {
+            resource_id: candidate.resource.resource_id,
+            resource_type: candidate.resource.resource_type,
+            installed_version: candidate.resource.version,
+            latest_version: candidate.version,
+            download_url: candidate.download_url,
+            size_bytes: candidate.size_bytes || undefined,
+          },
+          { auto: true }
+        )
+
+        if (result.success) {
+          // Success is NOT recorded here: applyUpdate only enqueues a resumable
+          // download. The per-resource backoff is cleared once the download
+          // actually completes (RunDownloadJob.onComplete) and incremented when it
+          // fails terminally (the worker `failed` handler). Recording success on
+          // dispatch would reset the counter every window and defeat self-disable.
+          initiatedBytes += candidate.size_bytes
+          started++
+          logger.info(
+            `[ContentAutoUpdateService] Started ${candidate.resource.resource_id} → ${candidate.version}`
+          )
+        } else {
+          // A failure to even enqueue is a genuine auto-update failure; no job runs,
+          // so no terminal `failed` event will follow — count it here.
+          await recordResourceUpdateFailure(candidate.resource, result.error ?? 'dispatch failed')
+          failed++
+        }
+      }
+
+      if (initiatedBytes > 0) {
+        await this.addWindowBytesUsed(initiatedBytes)
+      }
+
+      const parts = [`${started} started`]
+      if (failed) parts.push(`${failed} failed`)
+      if (skippedOversize.length) parts.push(`${skippedOversize.length} skipped (exceeds cap)`)
+      if (deferred.length) parts.push(`${deferred.length} deferred (over budget)`)
+      const reason = parts.join(', ')
+
+      await this.recordFeatureSuccess()
+      await this.recordRun(reason)
+      logger.info(`[ContentAutoUpdateService] Run complete: ${reason}`)
+      return { started, reason }
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error)
+      await this.recordFeatureFailure(message)
+      await this.recordRun(`Failed: ${message}`)
+      logger.error(`[ContentAutoUpdateService] Run failed: ${message}`)
+      return { started: 0, reason: `Failed: ${message}` }
+    }
+  }
+
+  /**
+   * Evaluate what the next run *would* do, without hitting the network,
+   * persisting state, or dispatching anything. Operates on the available-update
+   * state already persisted by the last check (manual or auto), so run a "Check
+   * for Content Updates" first if you want fresh catalog data. Used by the
+   * `content-auto-update:dry-run` command.
+   */
+  async dryRun(
+    overrides: {
+      now?: DateTime
+      forceEnabled?: boolean
+      cooloffHours?: number
+      windowStart?: string
+      windowEnd?: string
+      maxBytesPerWindow?: number
+      windowBytesUsed?: number
+    } = {}
+  ): Promise<{
+    enabled: boolean
+    withinWindow: boolean
+    config: ContentAutoUpdateConfig
+    eligibleCount: number
+    selection: ContentSelection
+  }> {
+    const base = await this.getConfig()
+    const config: ContentAutoUpdateConfig = {
+      enabled: overrides.forceEnabled ? true : base.enabled,
+      windowStart: overrides.windowStart ?? base.windowStart,
+      windowEnd: overrides.windowEnd ?? base.windowEnd,
+      cooloffHours: overrides.cooloffHours ?? base.cooloffHours,
+      maxBytesPerWindow: overrides.maxBytesPerWindow ?? base.maxBytesPerWindow,
+    }
+    const now = overrides.now ?? DateTime.now()
+    const withinWindow = isWithinWindow(config.windowStart, config.windowEnd, now)
+
+    const pending = await InstalledResource.query().whereNotNull('available_update_version')
+    const eligible = pending.filter(
+      (r) => this.resourceEligibility(r, config.cooloffHours, now).eligible
+    )
+    const candidates: ContentCandidate[] = eligible.map((r) => ({
+      resource: r,
+      version: r.available_update_version!,
+      download_url: '(dry-run)',
+      size_bytes: r.available_update_size_bytes ?? 0,
+      installed_at: r.installed_at,
+    }))
+
+    const usedBytes = overrides.windowBytesUsed ?? (await this.getWindowBytesUsed())
+    const selection = this.selectUnderCap(candidates, config.maxBytesPerWindow, usedBytes)
+
+    return {
+      enabled: config.enabled,
+      withinWindow,
+      config,
+      eligibleCount: eligible.length,
+      selection,
+    }
+  }
+
+  // ── Per-window budget ─────────────────────────────────────────────────────────
+
+  /** Most-recent window-open boundary as an absolute timestamp (handles wrap). */
+  windowStartBoundary(windowStart: string, now: DateTime): DateTime {
+    const minutes = parseWindowMinutes(windowStart) ?? 0
+    const todayStart = now.startOf('day').plus({ minutes })
+    return now >= todayStart ? todayStart : todayStart.minus({ days: 1 })
+  }
+
+  /** Reset the window budget exactly once per entry into the window. */
+  private async maybeResetWindowBudget(
+    config: ContentAutoUpdateConfig,
+    now: DateTime
+  ): Promise<void> {
+    const boundary = this.windowStartBoundary(config.windowStart, now)
+    const resetAtRaw = await KVStore.getValue('contentAutoUpdate.windowResetAt')
+    const resetAt = resetAtRaw ? DateTime.fromISO(resetAtRaw) : null
+    if (!resetAt || !resetAt.isValid || resetAt < boundary) {
+      await KVStore.setValue('contentAutoUpdate.windowBytesUsed', '0')
+      await KVStore.setValue('contentAutoUpdate.windowResetAt', now.toISO()!)
+    }
+  }
+
+  private async getWindowBytesUsed(): Promise<number> {
+    const raw = await KVStore.getValue('contentAutoUpdate.windowBytesUsed')
+    const num = Number(raw)
+    return Number.isFinite(num) && num > 0 ? num : 0
+  }
+
+  private async addWindowBytesUsed(bytes: number): Promise<void> {
+    const used = await this.getWindowBytesUsed()
+    await KVStore.setValue('contentAutoUpdate.windowBytesUsed', String(used + bytes))
+  }
+
+  // ── Backoff + run recording ───────────────────────────────────────────────────
+  // Per-resource backoff lives in ../utils/content_auto_update_backoff.ts so the
+  // job-completion path and the worker `failed` handler can share it without an
+  // import cycle. The feature-level backoff below stays here.
+
+  /** Clear the feature-level backoff after a clean run. */
+  private async recordFeatureSuccess(): Promise<void> {
+    await KVStore.setValue('contentAutoUpdate.consecutiveFailures', '0')
+    await KVStore.clearValue('contentAutoUpdate.lastError')
+  }
+
+  /** Record a whole-feature failure and self-disable the feature at the threshold. */
+  private async recordFeatureFailure(reason: string): Promise<void> {
+    const raw = await KVStore.getValue('contentAutoUpdate.consecutiveFailures')
+    const failures = (Number(raw) || 0) + 1
+    await KVStore.setValue('contentAutoUpdate.consecutiveFailures', String(failures))
+    await KVStore.setValue('contentAutoUpdate.lastError', reason)
+    if (failures >= MAX_FEATURE_FAILURES) {
+      await KVStore.setValue('contentAutoUpdate.enabled', false)
+      await KVStore.setValue(
+        'contentAutoUpdate.autoDisabledReason',
+        `Content auto-update disabled after ${failures} consecutive failures. Last error: ${reason}`
+      )
+      logger.error(
+        `[ContentAutoUpdateService] Feature auto-disabled after ${failures} consecutive failures`
+      )
+    }
+  }
+
+  private async recordRun(reason: string): Promise<void> {
+    await KVStore.setValue('contentAutoUpdate.lastAttemptAt', DateTime.now().toISO()!)
+    await KVStore.setValue('contentAutoUpdate.lastResult', reason)
+  }
+
+  // ── Status snapshot ───────────────────────────────────────────────────────────
+
+  /** Full state snapshot for the settings UI (resources with pending updates). */
+  async getStatus(): Promise<ContentAutoUpdateStatus> {
+    const config = await this.getConfig()
+    const now = DateTime.now()
+
+    const pending = await InstalledResource.query().whereNotNull('available_update_version')
+    const resources: ContentAutoUpdateResourceStatus[] = pending.map((resource) => {
+      const verdict = this.resourceEligibility(resource, config.cooloffHours, now)
+      const size = resource.available_update_size_bytes ?? null
+      const exceedsCap =
+        config.maxBytesPerWindow > 0 && size !== null && size > config.maxBytesPerWindow
+      return {
+        resource_id: resource.resource_id,
+        resource_type: resource.resource_type,
+        current_version: resource.version,
+        available_update_version: resource.available_update_version,
+        size_bytes: size,
+        eligible: verdict.eligible && !exceedsCap,
+        reason: exceedsCap ? 'Exceeds data cap — update manually' : verdict.reason,
+        cooloff_remaining_hours: verdict.cooloffRemainingHours,
+        exceeds_cap: exceedsCap,
+        consecutive_failures: resource.auto_update_consecutive_failures || 0,
+        auto_disabled_reason: resource.auto_update_disabled_reason,
+      }
+    })
+
+    const [lastAttemptAt, lastResult, lastError, autoDisabledReason, windowBytesUsed] =
+      await Promise.all([
+        KVStore.getValue('contentAutoUpdate.lastAttemptAt'),
+        KVStore.getValue('contentAutoUpdate.lastResult'),
+        KVStore.getValue('contentAutoUpdate.lastError'),
+        KVStore.getValue('contentAutoUpdate.autoDisabledReason'),
+        this.getWindowBytesUsed(),
+      ])
+
+    return {
+      ...config,
+      withinWindow: isWithinWindow(config.windowStart, config.windowEnd, now),
+      windowBytesUsed,
+      lastAttemptAt: lastAttemptAt || null,
+      lastResult: lastResult || null,
+      lastError: lastError || null,
+      autoDisabledReason: autoDisabledReason || null,
+      resources,
+    }
+  }
+}

admin/app/services/countries_service.ts

@@ -0,0 +1,308 @@
+import { access, readFile, writeFile, mkdir } from 'fs/promises'
+import { join, resolve } from 'path'
+import { createHash } from 'crypto'
+import { tmpdir } from 'os'
+import logger from '@adonisjs/core/services/logger'
+import type { Country, CountryCode, CountryGroup } from '../../types/maps.js'
+
+interface NEFeature {
+  type: 'Feature'
+  properties: Record<string, any>
+  geometry: unknown
+}
+
+interface NEFeatureCollection {
+  type: 'FeatureCollection'
+  features: NEFeature[]
+}
+
+const COUNTRY_GEOJSON_PATH = join(
+  process.cwd(),
+  'resources',
+  'geodata',
+  'ne_50m_admin_0_countries.geojson'
+)
+
+// Natural Earth country polygons are land-only (no territorial waters), so a
+// strict intersect leaves tiles fully over the ocean out of the extract —
+// coastal cities render as grey off their coast. Inflate each polygon outward
+// by ~11 km to pull in adjacent tiles without ballooning the extract size.
+const REGION_BUFFER_DEGREES = 0.1
+
+const GROUP_ORDER = [
+  'north-america',
+  'south-america',
+  'europe',
+  'africa',
+  'asia',
+  'oceania',
+]
+
+const GROUP_META: Record<string, { id: string; name: string; description: string }> = {
+  'North America': {
+    id: 'north-america',
+    name: 'North America',
+    description: 'All countries in North America and the Caribbean.',
+  },
+  'South America': {
+    id: 'south-america',
+    name: 'South America',
+    description: 'All countries in South America.',
+  },
+  Europe: {
+    id: 'europe',
+    name: 'Europe',
+    description: 'All countries in Europe.',
+  },
+  Africa: {
+    id: 'africa',
+    name: 'Africa',
+    description: 'All countries in Africa.',
+  },
+  Asia: {
+    id: 'asia',
+    name: 'Asia',
+    description: 'All countries in Asia.',
+  },
+  Oceania: {
+    id: 'oceania',
+    name: 'Oceania',
+    description: 'Australia, New Zealand, and Pacific island nations.',
+  },
+}
+
+export class CountriesService {
+  private static instance: CountriesService | null = null
+  private loadPromise: Promise<void> | null = null
+  private countries: Country[] = []
+  private byCode: Map<CountryCode, { country: Country; feature: NEFeature }> = new Map()
+  private groups: CountryGroup[] = []
+
+  static getInstance(): CountriesService {
+    if (!this.instance) {
+      this.instance = new CountriesService()
+    }
+    return this.instance
+  }
+
+  private async ensureLoaded(): Promise<void> {
+    if (this.byCode.size > 0) return
+    if (!this.loadPromise) {
+      this.loadPromise = this.load()
+    }
+    await this.loadPromise
+  }
+
+  private async load(): Promise<void> {
+    const raw = await readFile(COUNTRY_GEOJSON_PATH, 'utf8')
+    const fc = JSON.parse(raw) as NEFeatureCollection
+
+    // Natural Earth reuses a sovereign state's ISO_A2 for its dependencies
+    // (e.g. AU covers both Australia and Australian territories). Sort so the
+    // sovereign mainland wins the ISO-code slot, and skip any subsequent
+    // same-code dependency — otherwise the "AU" entry ends up being some tiny
+    // island territory.
+    const sortedFeatures = [...fc.features].sort((a, b) => typeRank(a) - typeRank(b))
+
+    const countries: Country[] = []
+    const byCode = new Map<CountryCode, { country: Country; feature: NEFeature }>()
+    const groupCodes: Record<string, CountryCode[]> = {}
+
+    for (const feature of sortedFeatures) {
+      const p = feature.properties
+      const code = resolveIso2(p)
+      if (!code) continue
+      if (byCode.has(code)) continue
+
+      const continent = typeof p.CONTINENT === 'string' ? p.CONTINENT : 'Other'
+      if (continent === 'Antarctica' || continent === 'Seven seas (open ocean)') continue
+
+      const country: Country = {
+        code,
+        code3: resolveIso3(p) ?? code,
+        name: typeof p.NAME === 'string' ? p.NAME : code,
+        continent,
+        subregion: typeof p.SUBREGION === 'string' ? p.SUBREGION : continent,
+        population: typeof p.POP_EST === 'number' ? p.POP_EST : 0,
+      }
+
+      countries.push(country)
+      byCode.set(code, { country, feature })
+
+      if (GROUP_META[continent]) {
+        const groupId = GROUP_META[continent].id
+        if (!groupCodes[groupId]) groupCodes[groupId] = []
+        groupCodes[groupId].push(code)
+      }
+    }
+
+    countries.sort((a, b) => a.name.localeCompare(b.name))
+
+    const groups: CountryGroup[] = GROUP_ORDER.flatMap((groupId) => {
+      const meta = Object.values(GROUP_META).find((m) => m.id === groupId)
+      if (!meta) return []
+      const codes = (groupCodes[groupId] ?? []).slice().sort()
+      if (codes.length === 0) return []
+      return [{ id: meta.id, name: meta.name, description: meta.description, countries: codes }]
+    })
+
+    this.countries = countries
+    this.byCode = byCode
+    this.groups = groups
+
+    logger.info(
+      `[CountriesService] Loaded ${countries.length} countries across ${groups.length} groups`
+    )
+  }
+
+  async list(): Promise<Country[]> {
+    await this.ensureLoaded()
+    return this.countries
+  }
+
+  async listGroups(): Promise<CountryGroup[]> {
+    await this.ensureLoaded()
+    return this.groups
+  }
+
+  /** Throws when a supplied code does not map to a known country. */
+  async resolveCodes(codes: CountryCode[]): Promise<CountryCode[]> {
+    await this.ensureLoaded()
+    const normalized = [...new Set(codes.map((c) => c.toUpperCase()))].sort()
+    const unknown = normalized.filter((c) => !this.byCode.has(c))
+    if (unknown.length > 0) {
+      throw new Error(`Unknown country code(s): ${unknown.join(', ')}`)
+    }
+    return normalized
+  }
+
+  /**
+   * Filename is keyed on a hash of the sorted ISO codes + buffer size so
+   * repeated calls with the same selection reuse the same path, and bumping
+   * the buffer auto-invalidates stale files.
+   */
+  async writeRegionFile(codes: CountryCode[]): Promise<string> {
+    await this.ensureLoaded()
+    const resolved = await this.resolveCodes(codes)
+    const key = `b${REGION_BUFFER_DEGREES}:${resolved.join(',')}`
+    const hash = createHash('sha1').update(key).digest('hex').slice(0, 12)
+
+    const dir = resolve(tmpdir(), 'nomad-pmtiles-regions')
+    await mkdir(dir, { recursive: true })
+    const filepath = join(dir, `region-${hash}.geojson`)
+
+    try {
+      await access(filepath)
+      return filepath
+    } catch {}
+
+    const fc = {
+      type: 'FeatureCollection',
+      features: resolved.map((code) => {
+        const entry = this.byCode.get(code)!
+        return {
+          type: 'Feature',
+          properties: { iso: code, name: entry.country.name },
+          geometry: bufferGeometry(entry.feature.geometry, REGION_BUFFER_DEGREES),
+        }
+      }),
+    }
+
+    await writeFile(filepath, JSON.stringify(fc))
+    return filepath
+  }
+}
+
+function typeRank(f: NEFeature): number {
+  const t = typeof f.properties.TYPE === 'string' ? f.properties.TYPE : ''
+  if (t === 'Sovereign country') return 0
+  if (t === 'Country') return 1
+  if (t === 'Sovereignty') return 2
+  if (t === 'Disputed') return 3
+  if (t === 'Dependency') return 4
+  return 5
+}
+
+function resolveIso2(p: Record<string, any>): CountryCode | null {
+  // Natural Earth's ISO_A2 sometimes holds political escapes like "CN-TW" for
+  // Taiwan or "-99" for countries involved in disputes. Only accept clean
+  // 2-letter codes; fall back to ISO_A2_EH (which reliably has the real code).
+  const primary = typeof p.ISO_A2 === 'string' ? p.ISO_A2 : null
+  if (primary && /^[A-Z]{2}$/i.test(primary)) return primary.toUpperCase()
+  const fallback = typeof p.ISO_A2_EH === 'string' ? p.ISO_A2_EH : null
+  if (fallback && /^[A-Z]{2}$/i.test(fallback)) return fallback.toUpperCase()
+  return null
+}
+
+/**
+ * Inflate each polygon ring outward by `buffer` degrees via per-vertex
+ * averaged-normal offset. Not geodesically accurate — but at small buffers
+ * (<= 0.2°) it's within a few percent of a proper geodesic buffer at
+ * country scale, which is plenty for tile-inclusion purposes.
+ */
+function bufferGeometry(geometry: unknown, buffer: number): unknown {
+  const geom = geometry as { type: string; coordinates: any }
+  if (geom?.type === 'Polygon') {
+    return { type: 'Polygon', coordinates: bufferPolygonRings(geom.coordinates, buffer) }
+  }
+  if (geom?.type === 'MultiPolygon') {
+    return {
+      type: 'MultiPolygon',
+      coordinates: geom.coordinates.map((poly: number[][][]) =>
+        bufferPolygonRings(poly, buffer)
+      ),
+    }
+  }
+  return geometry
+}
+
+function bufferPolygonRings(rings: number[][][], buffer: number): number[][][] {
+  return rings.map((ring) => bufferRing(ring, buffer))
+}
+
+function bufferRing(ring: number[][], buffer: number): number[][] {
+  if (ring.length < 4) return ring
+  const sign = signedArea(ring) > 0 ? 1 : -1
+  const n = ring.length - 1
+  const out: number[][] = []
+  for (let i = 0; i < n; i++) {
+    const prev = ring[(i - 1 + n) % n]
+    const curr = ring[i]
+    const next = ring[(i + 1) % n]
+    const e1x = curr[0] - prev[0]
+    const e1y = curr[1] - prev[1]
+    const e2x = next[0] - curr[0]
+    const e2y = next[1] - curr[1]
+    const l1 = Math.hypot(e1x, e1y) || 1
+    const l2 = Math.hypot(e2x, e2y) || 1
+    const n1x = (e1y / l1) * sign
+    const n1y = (-e1x / l1) * sign
+    const n2x = (e2y / l2) * sign
+    const n2y = (-e2x / l2) * sign
+    const sumX = n1x + n2x
+    const sumY = n1y + n2y
+    const sl = Math.hypot(sumX, sumY) || 1
+    out.push([curr[0] + (sumX / sl) * buffer, curr[1] + (sumY / sl) * buffer])
+  }
+  out.push(out[0])
+  return out
+}
+
+function signedArea(ring: number[][]): number {
+  let a = 0
+  for (let i = 0; i < ring.length - 1; i++) {
+    a += ring[i][0] * ring[i + 1][1] - ring[i + 1][0] * ring[i][1]
+  }
+  return a / 2
+}
+
+function resolveIso3(p: Record<string, any>): string | null {
+  const primary = typeof p.ISO_A3 === 'string' ? p.ISO_A3 : null
+  if (primary && primary !== '-99') return primary.toUpperCase()
+  const fallback = typeof p.ISO_A3_EH === 'string' ? p.ISO_A3_EH : null
+  if (fallback && fallback !== '-99') return fallback.toUpperCase()
+  const adm = typeof p.ADM0_A3 === 'string' ? p.ADM0_A3 : null
+  if (adm && adm !== '-99') return adm.toUpperCase()
+  return null
+}
+

admin/app/services/custom_app_guard.ts

@@ -0,0 +1,167 @@
+import { dirname, normalize } from 'node:path'
+import env from '#start/env'
+
+/**
+ * Security guardrails for user-defined ("custom app") containers.
+ *
+ * project-nomad runs containers as host siblings via the mounted Docker socket (DooD), so a
+ * misconfigured bind mount or image is a real host-takeover vector. The posture here is
+ * "guardrails with warnings": hard-block the genuinely catastrophic, warn-but-allow the merely
+ * risky so a trusted admin keeps their power without an easy foot-gun.
+ */
+
+export interface GuardEvaluation {
+  /** Hard rejections — the install cannot proceed until these are fixed. */
+  blocked: string[]
+  /** Advisory warnings — overridable via the "install anyway" force flag. */
+  warnings: string[]
+}
+
+/** Absolute host directories that must never be bind-mounted into a custom container. */
+const SYSTEM_BLOCK_PREFIXES = ['/etc', '/proc', '/sys', '/boot', '/dev', '/run', '/var/run']
+
+/** Registries we ship curated apps from; anything else is allowed but warned on. */
+const TRUSTED_REGISTRIES = ['docker.io', 'registry-1.docker.io', 'ghcr.io', 'lscr.io', 'quay.io']
+
+/** Resolve the managed storage root (where bind mounts are expected to live). */
+export function getStorageRoot(): string {
+  return normalize(env.get('NOMAD_STORAGE_PATH', '/opt/project-nomad/storage')).replace(/\/+$/, '')
+}
+
+/** Normalize an absolute path: collapse `..`/`.` segments and strip any trailing slash. */
+function normalizeHostPath(p: string): string {
+  return normalize(p).replace(/\/+$/, '') || '/'
+}
+
+/** True when `child` equals `ancestor` or sits beneath it. */
+function isWithin(child: string, ancestor: string): boolean {
+  return child === ancestor || child.startsWith(ancestor + '/')
+}
+
+/**
+ * Evaluate user-supplied bind mounts. Hard-blocks the Docker socket, core system directories,
+ * and any mount at or above project-nomad's own install tree (which would expose its code/data).
+ * Warns on any host path outside the managed storage root.
+ */
+export function evaluateBindMounts(
+  volumes: { host_path: string; container_path: string }[]
+): GuardEvaluation {
+  const blocked: string[] = []
+  const warnings: string[] = []
+
+  const storageRoot = getStorageRoot()
+  // The install tree is the parent of the storage root (e.g. /opt/project-nomad). Mounting it —
+  // or any ancestor, up to and including `/` — would hand a container project-nomad's own files.
+  const installRoot = dirname(storageRoot)
+
+  for (const { host_path: hostPath, container_path: containerPath } of volumes) {
+    const host = normalizeHostPath(hostPath)
+
+    if (!hostPath.startsWith('/')) {
+      blocked.push(`Volume host path "${hostPath}" must be an absolute path.`)
+      continue
+    }
+    if (!containerPath.startsWith('/')) {
+      blocked.push(`Volume container path "${containerPath}" must be an absolute path.`)
+      continue
+    }
+
+    // A colon is Docker's bind delimiter (host:container:options). A path containing one would be
+    // re-split by Docker into a different mount than the one validated here — reject it outright so
+    // the checks below can't be bypassed by a parse-differential. (The validator blocks this too;
+    // this keeps the guard self-defending for any caller that skips validation.)
+    if (hostPath.includes(':') || containerPath.includes(':')) {
+      blocked.push(`Volume paths must not contain a colon (":"): "${hostPath}" → "${containerPath}".`)
+      continue
+    }
+
+    // The Docker socket is the most dangerous mount of all — full control of the host daemon.
+    if (host.endsWith('docker.sock') || /\/docker\.sock$/.test(host)) {
+      blocked.push(
+        `Mounting the Docker socket ("${hostPath}") is not allowed — it grants full host control.`
+      )
+      continue
+    }
+
+    // Core system directories.
+    if (host === '/' || SYSTEM_BLOCK_PREFIXES.some((p) => isWithin(host, p))) {
+      blocked.push(`Mounting system directory "${hostPath}" is not allowed.`)
+      continue
+    }
+
+    // At or above project-nomad's own install tree (covers `/`, `/opt`, `/opt/project-nomad`).
+    if (host === installRoot || isWithin(installRoot, host)) {
+      blocked.push(
+        `Mounting "${hostPath}" would expose project-nomad's own files and is not allowed.`
+      )
+      continue
+    }
+
+    // Anything outside the managed storage root is allowed but flagged.
+    if (!isWithin(host, storageRoot)) {
+      warnings.push(
+        `Volume "${hostPath}" is outside the managed storage root (${storageRoot}). Make sure you trust this image with access to that path.`
+      )
+    }
+  }
+
+  return { blocked, warnings }
+}
+
+/**
+ * Evaluate a Docker image reference. Hard-blocks malformed references; warns on moving tags
+ * (`:latest`/untagged) and images from registries outside the trusted set.
+ */
+export function evaluateImageReference(image: string): GuardEvaluation {
+  const blocked: string[] = []
+  const warnings: string[] = []
+
+  const ref = image.trim()
+  // Loose validity check: no whitespace/control chars, and a sane character set for an image ref.
+  if (!ref || /\s/.test(ref) || !/^[\w./:@-]+$/.test(ref)) {
+    blocked.push(`"${image}" is not a valid image reference.`)
+    return { blocked, warnings }
+  }
+
+  // Split off any digest, then any tag, to inspect the registry and tag.
+  const [nameAndTag] = ref.split('@')
+  const firstSegment = nameAndTag.split('/')[0]
+  const hasRegistryHost =
+    nameAndTag.includes('/') && (firstSegment.includes('.') || firstSegment.includes(':'))
+  const registry = hasRegistryHost ? firstSegment.split(':')[0] : 'docker.io'
+
+  if (!TRUSTED_REGISTRIES.includes(registry)) {
+    warnings.push(
+      `Image is from "${registry}", which is outside project-nomad's trusted registries. Only install images you trust.`
+    )
+  }
+
+  // Determine the tag (ignore a colon that's part of a registry host:port in the first segment).
+  const remainder = hasRegistryHost ? nameAndTag.slice(firstSegment.length + 1) : nameAndTag
+  const tag = remainder.includes(':') ? remainder.split(':').pop() : undefined
+  const hasDigest = ref.includes('@sha256:')
+  if (!hasDigest && (!tag || tag === 'latest')) {
+    warnings.push(
+      `Image "${image}" uses a moving tag (${tag ? ':latest' : 'no tag'}). Pin a specific version for reproducible installs.`
+    )
+  }
+
+  return { blocked, warnings }
+}
+
+/** Combine bind-mount and image evaluations into a single result. */
+export function evaluateCustomApp(input: {
+  image?: string
+  volumes?: { host_path: string; container_path: string }[]
+}): GuardEvaluation {
+  const bind = evaluateBindMounts(input.volumes ?? [])
+  const img = input.image ? evaluateImageReference(input.image) : { blocked: [], warnings: [] }
+  return {
+    blocked: [...bind.blocked, ...img.blocked],
+    warnings: [...bind.warnings, ...img.warnings],
+  }
+}
+
+/** Default resource caps applied to custom containers unless the user overrides them. */
+export const DEFAULT_MEMORY_MB = 1024
+export const DEFAULT_CPUS = 1

admin/app/services/docs_service.ts

@@ -3,9 +3,23 @@ import { streamToString } from '../../util/docs.js'
 import { getFile, getFileStatsIfExists, listDirectoryContentsRecursive } from '../utils/fs.js'
 import path from 'path'
 import InternalServerErrorException from '#exceptions/internal_server_error_exception'
+import logger from '@adonisjs/core/services/logger'
 
 export class DocsService {
   private docsPath = path.join(process.cwd(), 'docs')
+
+  private static readonly DOC_ORDER: Record<string, number> = {
+    'home': 1,
+    'getting-started': 2,
+    'use-cases': 3,
+    'supply-depot-apps': 4,
+    'community-add-ons': 5,
+    'updates': 6,
+    'faq': 7,
+    'about': 8,
+    'release-notes': 9,
+  }
+
   async getDocs() {
     const contents = await listDirectoryContentsRecursive(this.docsPath)
     const files: Array<{ title: string; slug: string }> = []
@@ -20,7 +34,11 @@ export class DocsService {
       }
     }
 
-    return files.sort((a, b) => a.title.localeCompare(b.title))
+    return files.sort((a, b) => {
+      const orderA = DocsService.DOC_ORDER[a.slug] ?? 999
+      const orderB = DocsService.DOC_ORDER[b.slug] ?? 999
+      return orderA - orderB
+    })
   }
 
   parse(content: string) {
@@ -32,13 +50,13 @@ export class DocsService {
       // Filter out attribute-undefined errors which may be caused by emojis and special characters
       const criticalErrors = errors.filter((e) => e.error.id !== 'attribute-undefined')
       if (criticalErrors.length > 0) {
-        console.error('Markdoc validation errors:', errors.map((e) => JSON.stringify(e.error)).join(', '))
+        logger.error('Markdoc validation errors:', errors.map((e) => JSON.stringify(e.error)).join(', '))
         throw new Error('Markdoc validation failed')
       }
 
       return Markdoc.transform(ast, config)
     } catch (error) {
-      console.log('Error parsing Markdoc content:', error)
+      logger.error('Error parsing Markdoc content:', error)
       throw new InternalServerErrorException(`Error parsing content: ${(error as Error).message}`)
     }
   }
@@ -51,12 +69,19 @@ export class DocsService {
 
       const filename = _filename.endsWith('.md') ? _filename : `${_filename}.md`
 
-      const fileExists = await getFileStatsIfExists(path.join(this.docsPath, filename))
+      // Prevent path traversal — resolved path must stay within the docs directory
+      const basePath = path.resolve(this.docsPath)
+      const fullPath = path.resolve(path.join(this.docsPath, filename))
+      if (!fullPath.startsWith(basePath + path.sep)) {
+        throw new Error('Invalid document slug')
+      }
+
+      const fileExists = await getFileStatsIfExists(fullPath)
       if (!fileExists) {
         throw new Error(`File not found: ${filename}`)
       }
 
-      const fileStream = await getFile(path.join(this.docsPath, filename), 'stream')
+      const fileStream = await getFile(fullPath, 'stream')
       if (!fileStream) {
         throw new Error(`Failed to read file stream: ${filename}`)
       }
@@ -67,9 +92,18 @@ export class DocsService {
     }
   }
 
+  private static readonly TITLE_OVERRIDES: Record<string, string> = {
+    'faq': 'FAQ',
+    'community-add-ons': 'Community Add-Ons',
+  }
+
   private prettify(filename: string) {
+    const slug = filename.replace(/\.md$/, '')
+    if (DocsService.TITLE_OVERRIDES[slug]) {
+      return DocsService.TITLE_OVERRIDES[slug]
+    }
     // Remove hyphens, underscores, and file extension
-    const cleaned = filename.replace(/_/g, ' ').replace(/\.md$/, '').replace(/-/g, ' ')
+    const cleaned = slug.replace(/_/g, ' ').replace(/-/g, ' ')
     // Convert to Title Case
     const titleCased = cleaned.replace(/\b\w/g, (char) => char.toUpperCase())
     return titleCased.charAt(0).toUpperCase() + titleCased.slice(1)
@@ -115,6 +149,58 @@ export class DocsService {
             class: { type: String }
           }
         },
+        table: {
+          render: 'Table',
+        },
+        thead: {
+          render: 'TableHead',
+        },
+        tbody: {
+          render: 'TableBody',
+        },
+        tr: {
+          render: 'TableRow',
+        },
+        th: {
+          render: 'TableHeader',
+        },
+        td: {
+          render: 'TableCell',
+        },
+        paragraph: {
+          render: 'Paragraph',
+        },
+        image: {
+          render: 'Image',
+          attributes: {
+            src: { type: String, required: true },
+            alt: { type: String },
+            title: { type: String },
+          },
+        },
+        link: {
+          render: 'Link',
+          attributes: {
+            href: { type: String, required: true },
+            title: { type: String },
+          },
+        },
+        fence: {
+          render: 'CodeBlock',
+          attributes: {
+            content: { type: String },
+            language: { type: String },
+          },
+        },
+        code: {
+          render: 'InlineCode',
+          attributes: {
+            content: { type: String },
+          },
+        },
+        hr: {
+          render: 'HorizontalRule',
+        },
       },
     }
   }

admin/app/services/download_service.ts

@@ -1,25 +1,320 @@
 import { inject } from '@adonisjs/core'
 import { QueueService } from './queue_service.js'
 import { RunDownloadJob } from '#jobs/run_download_job'
-import { DownloadJobWithProgress } from '../../types/downloads.js'
+import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job'
+import type { RunExtractPmtilesJobParams } from '#jobs/run_extract_pmtiles_job'
+import { DownloadModelJob } from '#jobs/download_model_job'
+import { DownloadJobWithProgress, DownloadProgressData } from '../../types/downloads.js'
+import type { Job, Queue } from 'bullmq'
 import { normalize } from 'path'
+import { deleteFileIfExists } from '../utils/fs.js'
+import transmit from '@adonisjs/transmit/services/main'
+import { BROADCAST_CHANNELS } from '../../constants/broadcast.js'
+
+type FileJobState = 'waiting' | 'active' | 'delayed' | 'failed'
+type TaggedJob = { job: Job; state: FileJobState }
 
 @inject()
 export class DownloadService {
   constructor(private queueService: QueueService) {}
 
-  async listDownloadJobs(filetype?: string): Promise<DownloadJobWithProgress[]> {
-    const queue = this.queueService.getQueue(RunDownloadJob.queue)
-    const jobs = await queue.getJobs(['waiting', 'active', 'delayed'])
+  private parseProgress(progress: any): { percent: number; downloadedBytes?: number; totalBytes?: number; lastProgressTime?: number } {
+    if (typeof progress === 'object' && progress !== null && 'percent' in progress) {
+      const p = progress as DownloadProgressData
+      return {
+        percent: p.percent,
+        downloadedBytes: p.downloadedBytes,
+        totalBytes: p.totalBytes,
+        lastProgressTime: p.lastProgressTime,
+      }
+    }
+    // Backward compat: plain integer from in-flight jobs during upgrade
+    return { percent: parseInt(String(progress), 10) || 0 }
+  }
 
-    return jobs
-      .map((job) => ({
+  /** Fetch all non-completed jobs from a queue, tagged with their current BullMQ state */
+  private async fetchJobsWithStates(queueName: string): Promise<TaggedJob[]> {
+    const queue = this.queueService.getQueue(queueName)
+    const [waiting, active, delayed, failed] = await Promise.all([
+      queue.getJobs(['waiting']),
+      queue.getJobs(['active']),
+      queue.getJobs(['delayed']),
+      queue.getJobs(['failed']),
+    ])
+    return [
+      ...waiting.map((j) => ({ job: j, state: 'waiting' as const })),
+      ...active.map((j) => ({ job: j, state: 'active' as const })),
+      ...delayed.map((j) => ({ job: j, state: 'delayed' as const })),
+      ...failed.map((j) => ({ job: j, state: 'failed' as const })),
+    ]
+  }
+
+  async listDownloadJobs(filetype?: string): Promise<DownloadJobWithProgress[]> {
+    const modelQueue = this.queueService.getQueue(DownloadModelJob.queue)
+    const [fileTagged, extractTagged, modelJobs] = await Promise.all([
+      this.fetchJobsWithStates(RunDownloadJob.queue),
+      this.fetchJobsWithStates(RunExtractPmtilesJob.queue),
+      modelQueue.getJobs(['waiting', 'active', 'delayed', 'failed']),
+    ])
+
+    const fileDownloads = fileTagged.map(({ job, state }) => {
+      const parsed = this.parseProgress(job.progress)
+      return {
         jobId: job.id!.toString(),
         url: job.data.url,
-        progress: parseInt(job.progress.toString(), 10),
+        progress: parsed.percent,
         filepath: normalize(job.data.filepath),
         filetype: job.data.filetype,
-      }))
-      .filter((job) => !filetype || job.filetype === filetype)
+        title: job.data.title || undefined,
+        downloadedBytes: parsed.downloadedBytes,
+        totalBytes: parsed.totalBytes || job.data.totalBytes || undefined,
+        lastProgressTime: parsed.lastProgressTime,
+        status: state,
+        failedReason: job.failedReason || undefined,
+      }
+    })
+
+    const extractDownloads = extractTagged.map(({ job, state }) => {
+      const parsed = this.parseProgress(job.progress)
+      return {
+        jobId: job.id!.toString(),
+        url: job.data.sourceUrl,
+        progress: parsed.percent,
+        filepath: normalize(job.data.outputFilepath),
+        filetype: job.data.filetype || 'map',
+        title: job.data.title || undefined,
+        downloadedBytes: parsed.downloadedBytes,
+        totalBytes: parsed.totalBytes || job.data.estimatedBytes || undefined,
+        lastProgressTime: parsed.lastProgressTime,
+        status: state,
+        failedReason: job.failedReason || undefined,
+      }
+    })
+
+    const modelDownloads = modelJobs.map((job) => ({
+      jobId: job.id!.toString(),
+      url: job.data.modelName || 'Unknown Model',
+      progress: parseInt(job.progress.toString(), 10),
+      filepath: job.data.modelName || 'Unknown Model',
+      filetype: 'model',
+      status: (job.failedReason ? 'failed' : 'active') as 'active' | 'failed',
+      failedReason: job.failedReason || undefined,
+    }))
+
+    const allDownloads = [...fileDownloads, ...extractDownloads, ...modelDownloads]
+    const filtered = allDownloads.filter((job) => !filetype || job.filetype === filetype)
+
+    return filtered.sort((a, b) => {
+      if (a.status === 'failed' && b.status !== 'failed') return 1
+      if (a.status !== 'failed' && b.status === 'failed') return -1
+      return b.progress - a.progress
+    })
+  }
+
+  async removeFailedJob(jobId: string): Promise<void> {
+    for (const queueName of [
+      RunDownloadJob.queue,
+      RunExtractPmtilesJob.queue,
+      DownloadModelJob.queue,
+    ]) {
+      const queue = this.queueService.getQueue(queueName)
+      const job = await queue.getJob(jobId)
+      if (job) {
+        try {
+          await job.remove()
+        } catch {
+          // Job may be locked by the worker after cancel. Remove the stale lock and retry.
+          try {
+            const client = await queue.client
+            await client.del(`bull:${queueName}:${jobId}:lock`)
+            await job.remove()
+          } catch {
+            // Last resort: already removed or truly stuck
+          }
+        }
+        return
+      }
+    }
+  }
+
+  async cancelJob(jobId: string): Promise<{ success: boolean; message: string }> {
+    const queue = this.queueService.getQueue(RunDownloadJob.queue)
+    const job = await queue.getJob(jobId)
+
+    if (job) {
+      return await this._cancelFileDownloadJob(jobId, job, queue)
+    }
+
+    const extractQueue = this.queueService.getQueue(RunExtractPmtilesJob.queue)
+    const extractJob = await extractQueue.getJob(jobId)
+
+    if (extractJob) {
+      return await this._cancelExtractJob(jobId, extractJob, extractQueue)
+    }
+
+    const modelQueue = this.queueService.getQueue(DownloadModelJob.queue)
+    const modelJob = await modelQueue.getJob(jobId)
+
+    if (modelJob) {
+      return await this._cancelModelDownloadJob(jobId, modelJob, modelQueue)
+    }
+
+    return { success: true, message: 'Job not found (may have already completed)' }
+  }
+
+  private async _cancelExtractJob(
+    jobId: string,
+    job: Job<RunExtractPmtilesJobParams>,
+    queue: Queue<RunExtractPmtilesJobParams>
+  ): Promise<{ success: boolean; message: string }> {
+    const outputFilepath = job.data.outputFilepath
+
+    await RunExtractPmtilesJob.signalCancel(jobId)
+
+    // Same-process fallback when worker and API share a process
+    RunExtractPmtilesJob.childProcesses.get(jobId)?.kill('SIGTERM')
+    RunExtractPmtilesJob.childProcesses.delete(jobId)
+
+    await this._pollForTerminalState(job, jobId)
+    await this._removeJobWithLockFallback(job, queue, RunExtractPmtilesJob.queue, jobId)
+
+    if (outputFilepath) {
+      try {
+        await deleteFileIfExists(outputFilepath)
+      } catch {
+        // File may not exist yet (subprocess may not have opened it)
+      }
+    }
+
+    return { success: true, message: 'Extract cancelled and partial file deleted' }
+  }
+
+  /** Cancel a content download (zim, map, pmtiles, etc.) */
+  private async _cancelFileDownloadJob(
+    jobId: string,
+    job: any,
+    queue: any
+  ): Promise<{ success: boolean; message: string }> {
+    const filepath = job.data.filepath
+
+    // Signal the worker process to abort the download via Redis
+    await RunDownloadJob.signalCancel(jobId)
+
+    // Also try in-memory abort (works if worker is in same process)
+    RunDownloadJob.abortControllers.get(jobId)?.abort('user-cancel')
+    RunDownloadJob.abortControllers.delete(jobId)
+
+    await this._pollForTerminalState(job, jobId)
+    await this._removeJobWithLockFallback(job, queue, RunDownloadJob.queue, jobId)
+
+    // Delete the partial file from disk
+    if (filepath) {
+      try {
+        await deleteFileIfExists(filepath)
+        // Also try .tmp in case PR #448 staging is merged
+        await deleteFileIfExists(filepath + '.tmp')
+      } catch {
+        // File may not exist yet (waiting job)
+      }
+    }
+
+    // If this was a Wikipedia download, update selection status to failed
+    // (the worker's failed event may not fire if we removed the job first)
+    if (job.data.filetype === 'zim' && job.data.url?.includes('wikipedia_en_')) {
+      try {
+        const { DockerService } = await import('#services/docker_service')
+        const { ZimService } = await import('#services/zim_service')
+        const dockerService = new DockerService()
+        const zimService = new ZimService(dockerService)
+        await zimService.onWikipediaDownloadComplete(job.data.url, false)
+      } catch {
+        // Best effort
+      }
+    }
+
+    return { success: true, message: 'Download cancelled and partial file deleted' }
+  }
+
+  /** Cancel an Ollama model download — mirrors the file cancel pattern but skips file cleanup */
+  private async _cancelModelDownloadJob(
+    jobId: string,
+    job: any,
+    queue: any
+  ): Promise<{ success: boolean; message: string }> {
+    const modelName: string = job.data?.modelName ?? 'unknown'
+
+    // Signal the worker process to abort the pull via Redis
+    await DownloadModelJob.signalCancel(jobId)
+
+    // Also try in-memory abort (works if worker is in same process)
+    DownloadModelJob.abortControllers.get(jobId)?.abort('user-cancel')
+    DownloadModelJob.abortControllers.delete(jobId)
+
+    await this._pollForTerminalState(job, jobId)
+    await this._removeJobWithLockFallback(job, queue, DownloadModelJob.queue, jobId)
+
+    // Broadcast a cancelled event so the frontend hook clears the entry. We use percent: -2
+    // (distinct from -1 = error) so the hook can route it to a 2s auto-clear instead of the
+    // 15s error display. The frontend ALSO removes the entry optimistically from the API
+    // response, so this is belt-and-suspenders for cases where the SSE arrives first.
+    transmit.broadcast(BROADCAST_CHANNELS.OLLAMA_MODEL_DOWNLOAD, {
+      model: modelName,
+      jobId,
+      percent: -2,
+      status: 'cancelled',
+      timestamp: new Date().toISOString(),
+    })
+
+    // Note on partial blob cleanup: Ollama manages model blobs internally at
+    // /root/.ollama/models/blobs/. We deliberately do NOT call /api/delete here — Ollama's
+    // expected behavior is to retain partial blobs so a re-pull resumes from where it left
+    // off. If the user wants to reclaim that space, they can re-pull and let it complete,
+    // or delete the partially-downloaded model from the AI Settings page.
+    return { success: true, message: 'Model download cancelled' }
+  }
+
+  /** Wait up to 4s (250ms intervals) for the job to reach a terminal state */
+  private async _pollForTerminalState(job: any, jobId: string): Promise<void> {
+    const POLL_INTERVAL_MS = 250
+    const POLL_TIMEOUT_MS = 4000
+    const deadline = Date.now() + POLL_TIMEOUT_MS
+
+    while (Date.now() < deadline) {
+      await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS))
+      try {
+        const state = await job.getState()
+        if (state === 'failed' || state === 'completed' || state === 'unknown') {
+          return
+        }
+      } catch {
+        return // getState() throws if job is already gone
+      }
+    }
+
+    console.warn(
+      `[DownloadService] cancelJob: job ${jobId} did not reach terminal state within timeout, removing anyway`
+    )
+  }
+
+  /** Remove a BullMQ job, clearing a stale worker lock if the first attempt fails */
+  private async _removeJobWithLockFallback(
+    job: any,
+    queue: any,
+    queueName: string,
+    jobId: string
+  ): Promise<void> {
+    try {
+      await job.remove()
+    } catch {
+      // Lock contention fallback: clear lock and retry once
+      try {
+        const client = await queue.client
+        await client.del(`bull:${queueName}:${jobId}:lock`)
+        const updatedJob = await queue.getJob(jobId)
+        if (updatedJob) await updatedJob.remove()
+      } catch {
+        // Best effort - job will be cleaned up on next dismiss attempt
+      }
+    }
   }
 }

admin/app/services/kiwix_catalog_service.ts

@@ -0,0 +1,338 @@
+import axios from 'axios'
+import { XMLParser } from 'fast-xml-parser'
+import { DateTime } from 'luxon'
+import logger from '@adonisjs/core/services/logger'
+import InstalledResource from '#models/installed_resource'
+import { isRawListRemoteZimFilesResponse } from '../../util/zim.js'
+
+/**
+ * Local, in-process freshness check for installed content (Kiwix ZIM files +
+ * PMTiles maps). This replaces the former dependency on the external
+ * project-nomad-api `/api/v1/resources/check-updates` endpoint — every NOMAD
+ * instance now queries the upstream catalogs directly.
+ *
+ * Downloads have always gone straight to the Kiwix/GitHub mirrors regardless of
+ * who performed the check, so moving the check in-process only shifts the
+ * lightweight *catalog* lookup. To stay mirror-respectful the auto-updater gates
+ * these calls behind the update window and bounds their concurrency; sizes come
+ * from the catalog metadata so we avoid per-file HEAD requests.
+ *
+ * Robustness over the old API: ZIM lookups use the OPDS exact `name=` filter
+ * (no lossy keyword stripping) and every returned link is still validated
+ * against the authoritative `^<id>_YYYY-MM\.zim$` filename regex, so a substring
+ * match in the catalog can never resolve to the wrong book. Parsing is fully
+ * defensive — a malformed entry is skipped, never thrown.
+ */
+
+const KIWIX_CATALOG_URL = 'https://browse.library.kiwix.org/catalog/v2/entries'
+const GITHUB_PMTILES_URL =
+  'https://api.github.com/repos/Crosstalk-Solutions/project-nomad-maps/contents/pmtiles'
+
+const CATALOG_TIMEOUT_MS = 15000
+/** Bounded paginated fallback scan when the exact `name=` lookup comes up empty. */
+const KIWIX_PAGE_SIZE = 60
+const MAX_KIWIX_FETCHES = 5
+/** Concurrent ZIM catalog lookups — keep small to avoid hammering the mirror. */
+const ZIM_CHECK_CONCURRENCY = 4
+
+/** The newest available version of a single resource (a YYYY-MM date stamp). */
+export interface CatalogResult {
+  version: string
+  download_url: string
+  size_bytes: number
+}
+
+interface CatalogZimEntry {
+  name: string | null
+  download_url: string
+  file_name: string
+  size_bytes: number
+}
+
+interface GithubContentEntry {
+  name: string
+  download_url: string | null
+  size: number
+}
+
+function escapeRegex(input: string): string {
+  return input.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+}
+
+export class KiwixCatalogService {
+  private parser = new XMLParser({
+    ignoreAttributes: false,
+    attributeNamePrefix: '',
+    textNodeName: '#text',
+  })
+
+  /**
+   * Resolve the newest available version for a batch of installed resources.
+   * Returns a map keyed by `"<resource_type>:<resource_id>"`; resources with no
+   * available newer version (or a failed lookup) are simply absent.
+   *
+   * ZIMs are checked one OPDS request each (bounded concurrency). All maps are
+   * resolved from a single GitHub directory listing.
+   */
+  async getLatestForResources(
+    resources: Array<{ resource_id: string; resource_type: 'zim' | 'map' }>
+  ): Promise<Map<string, CatalogResult>> {
+    const result = new Map<string, CatalogResult>()
+    const zims = resources.filter((r) => r.resource_type === 'zim')
+    const maps = resources.filter((r) => r.resource_type === 'map')
+
+    await this.forEachWithConcurrency(zims, ZIM_CHECK_CONCURRENCY, async (r) => {
+      try {
+        const latest = await this.getLatestZim(r.resource_id)
+        if (latest) result.set(`zim:${r.resource_id}`, latest)
+      } catch (error) {
+        logger.warn(
+          `[KiwixCatalogService] ZIM check failed for ${r.resource_id}: ${error instanceof Error ? error.message : error}`
+        )
+      }
+    })
+
+    if (maps.length > 0) {
+      try {
+        const listing = await this.fetchMapListing()
+        for (const r of maps) {
+          const latest = this.pickNewestMap(listing, r.resource_id)
+          if (latest) result.set(`map:${r.resource_id}`, latest)
+        }
+      } catch (error) {
+        logger.warn(
+          `[KiwixCatalogService] Map listing fetch failed: ${error instanceof Error ? error.message : error}`
+        )
+      }
+    }
+
+    return result
+  }
+
+  /** Newest catalog version of a single ZIM book, or null if none/older. */
+  async getLatestZim(resourceId: string): Promise<CatalogResult | null> {
+    const pattern = new RegExp(`^${escapeRegex(resourceId)}_(\\d{4}-\\d{2})\\.zim$`)
+
+    // 1. Exact-name lookup (the robust path).
+    const named = await this.fetchZimEntries({ name: resourceId, count: 50, start: 0 })
+    const exact = this.pickNewestZim(named, pattern)
+    if (exact) return exact
+
+    // 2. Fallback: bounded keyword scan in case the catalog ignored `name=` or
+    //    indexes the book under a slightly different name.
+    return this.scanZimByQuery(resourceId, pattern)
+  }
+
+  /** Newest catalog version of a single PMTiles map, or null if none/older. */
+  async getLatestMap(resourceId: string): Promise<CatalogResult | null> {
+    const listing = await this.fetchMapListing()
+    return this.pickNewestMap(listing, resourceId)
+  }
+
+  // ── ZIM internals ───────────────────────────────────────────────────────────
+
+  private pickNewestZim(entries: CatalogZimEntry[], pattern: RegExp): CatalogResult | null {
+    let latest: CatalogResult | null = null
+    for (const entry of entries) {
+      const match = entry.file_name.match(pattern)
+      if (!match) continue
+      const version = match[1]
+      if (!latest || version > latest.version) {
+        latest = { version, download_url: entry.download_url, size_bytes: entry.size_bytes }
+      }
+    }
+    return latest
+  }
+
+  private async scanZimByQuery(
+    resourceId: string,
+    pattern: RegExp
+  ): Promise<CatalogResult | null> {
+    let start = 0
+    let total = 0
+    let latest: CatalogResult | null = null
+
+    for (let i = 0; i < MAX_KIWIX_FETCHES; i++) {
+      const { entries, totalResults } = await this.fetchZimEntriesPage({
+        q: resourceId,
+        count: KIWIX_PAGE_SIZE,
+        start,
+      })
+      total = totalResults
+      if (entries.length === 0) break
+      start += entries.length
+
+      const candidate = this.pickNewestZim(entries, pattern)
+      if (candidate && (!latest || candidate.version > latest.version)) {
+        latest = candidate
+      }
+      if (start >= total) break
+    }
+    return latest
+  }
+
+  private async fetchZimEntries(params: {
+    name?: string
+    q?: string
+    count: number
+    start: number
+  }): Promise<CatalogZimEntry[]> {
+    const { entries } = await this.fetchZimEntriesPage(params)
+    return entries
+  }
+
+  private async fetchZimEntriesPage(params: {
+    name?: string
+    q?: string
+    count: number
+    start: number
+  }): Promise<{ entries: CatalogZimEntry[]; totalResults: number }> {
+    const res = await axios.get(KIWIX_CATALOG_URL, {
+      params: {
+        start: params.start,
+        count: params.count,
+        lang: 'eng',
+        ...(params.name ? { name: params.name } : {}),
+        ...(params.q ? { q: params.q } : {}),
+      },
+      responseType: 'text',
+      timeout: CATALOG_TIMEOUT_MS,
+    })
+    return this.parseZimEntries(res.data)
+  }
+
+  private parseZimEntries(xml: string): { entries: CatalogZimEntry[]; totalResults: number } {
+    let parsed: any
+    try {
+      parsed = this.parser.parse(xml)
+    } catch {
+      return { entries: [], totalResults: 0 }
+    }
+    if (!isRawListRemoteZimFilesResponse(parsed)) {
+      return { entries: [], totalResults: 0 }
+    }
+
+    const feed = parsed.feed
+    const totalResults = Number(feed?.totalResults)
+    const rawEntries = feed?.entry
+      ? Array.isArray(feed.entry)
+        ? feed.entry
+        : [feed.entry]
+      : []
+
+    const entries: CatalogZimEntry[] = []
+    for (const raw of rawEntries) {
+      if (!raw || typeof raw !== 'object') continue
+      const links = Array.isArray(raw.link) ? raw.link : raw.link ? [raw.link] : []
+      const downloadLink = links.find(
+        (link: any) =>
+          link &&
+          typeof link === 'object' &&
+          link.type === 'application/x-zim' &&
+          typeof link.href === 'string'
+      )
+      if (!downloadLink) continue
+
+      // The OPDS href ends with `.meta4`; strip it to get the real .zim URL.
+      const href: string = downloadLink.href
+      const download_url = href.endsWith('.meta4') ? href.slice(0, -'.meta4'.length) : href
+      const file_name = download_url.split('/').pop() || ''
+      if (!file_name) continue
+
+      const size_bytes = Number.parseInt(downloadLink.length, 10) || 0
+      entries.push({
+        name: typeof raw.name === 'string' ? raw.name : null,
+        download_url,
+        file_name,
+        size_bytes,
+      })
+    }
+
+    return { entries, totalResults: Number.isFinite(totalResults) ? totalResults : 0 }
+  }
+
+  // ── Map internals ────────────────────────────────────────────────────────────
+
+  private async fetchMapListing(): Promise<GithubContentEntry[]> {
+    const res = await axios.get(GITHUB_PMTILES_URL, {
+      headers: { Accept: 'application/vnd.github+json' },
+      timeout: CATALOG_TIMEOUT_MS,
+    })
+    return Array.isArray(res.data) ? res.data : []
+  }
+
+  private pickNewestMap(listing: GithubContentEntry[], resourceId: string): CatalogResult | null {
+    const pattern = new RegExp(`^${escapeRegex(resourceId)}_(\\d{4}-\\d{2})\\.pmtiles$`)
+    let latest: CatalogResult | null = null
+    for (const file of listing) {
+      if (!file || typeof file.name !== 'string' || !file.download_url) continue
+      const match = file.name.match(pattern)
+      if (!match) continue
+      const version = match[1]
+      if (!latest || version > latest.version) {
+        latest = {
+          version,
+          download_url: file.download_url,
+          size_bytes: typeof file.size === 'number' ? file.size : 0,
+        }
+      }
+    }
+    return latest
+  }
+
+  // ── Shared ───────────────────────────────────────────────────────────────────
+
+  private async forEachWithConcurrency<T>(
+    items: T[],
+    concurrency: number,
+    worker: (item: T) => Promise<void>
+  ): Promise<void> {
+    let cursor = 0
+    const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
+      while (cursor < items.length) {
+        const index = cursor++
+        await worker(items[index])
+      }
+    })
+    await Promise.all(runners)
+  }
+}
+
+/**
+ * Persist the latest-known available-update state onto an installed resource.
+ * Shared by the manual check and the auto-updater so both keep the cool-off
+ * anchor consistent.
+ *
+ * The first-seen anchor is reset **only** when the available version string
+ * actually changes, so a manual "Check for updates" never resets the auto
+ * cool-off clock. State is cleared entirely once the resource is current (the
+ * update got installed, or the upstream release was withdrawn).
+ */
+export async function reconcileResourceUpdateState(
+  resource: InstalledResource,
+  latest: CatalogResult | null,
+  now: DateTime
+): Promise<void> {
+  const hasUpdate = latest !== null && latest.version > resource.version
+
+  if (hasUpdate) {
+    if (resource.available_update_version !== latest!.version) {
+      resource.available_update_version = latest!.version
+      resource.available_update_first_seen_at = now
+    }
+    // Keep the cached size fresh even when the version is unchanged (the catalog
+    // may report a size it lacked on a previous check).
+    const size = latest!.size_bytes || null
+    if (resource.available_update_size_bytes !== size) {
+      resource.available_update_size_bytes = size
+    }
+  } else if (resource.available_update_version !== null) {
+    resource.available_update_version = null
+    resource.available_update_size_bytes = null
+    resource.available_update_first_seen_at = null
+  }
+
+  if (resource.$isDirty) {
+    await resource.save()
+  }
+}

admin/app/services/kiwix_library_service.ts

@@ -0,0 +1,364 @@
+import { XMLBuilder, XMLParser } from 'fast-xml-parser'
+import { readFile, writeFile, rename, readdir } from 'fs/promises'
+import { join } from 'path'
+import { Archive } from '@openzim/libzim'
+import { KIWIX_LIBRARY_XML_PATH, ZIM_STORAGE_PATH, ensureDirectoryExists, isValidZimFile } from '../utils/fs.js'
+import logger from '@adonisjs/core/services/logger'
+import { randomUUID } from 'node:crypto'
+
+const CONTAINER_DATA_PATH = '/data'
+const XML_DECLARATION = '<?xml version="1.0" encoding="UTF-8"?>\n'
+
+interface KiwixBook {
+  id: string
+  path: string
+  title: string
+  description?: string
+  language?: string
+  creator?: string
+  publisher?: string
+  name?: string
+  flavour?: string
+  tags?: string
+  faviconMimeType?: string
+  favicon?: string
+  date?: string
+  articleCount?: number
+  mediaCount?: number
+  size?: number
+}
+
+export class KiwixLibraryService {
+  getLibraryFilePath(): string {
+    return join(process.cwd(), KIWIX_LIBRARY_XML_PATH)
+  }
+
+  containerLibraryPath(): string {
+    return '/data/kiwix-library.xml'
+  }
+
+  private _filenameToTitle(filename: string): string {
+    const withoutExt = filename.endsWith('.zim') ? filename.slice(0, -4) : filename
+    const parts = withoutExt.split('_')
+    // Drop last segment if it looks like a date (YYYY-MM)
+    const lastPart = parts[parts.length - 1]
+    const isDate = /^\d{4}-\d{2}$/.test(lastPart)
+    const titleParts = isDate && parts.length > 1 ? parts.slice(0, -1) : parts
+    return titleParts.map((p) => p.charAt(0).toUpperCase() + p.slice(1)).join(' ')
+  }
+
+  /**
+   * Reads all kiwix-manage-compatible metadata from a ZIM file, including the internal UUID,
+   * rich text fields, and the base64-encoded favicon. Kiwix-serve uses the UUID for OPDS
+   * catalog entries and illustration URLs (/catalog/v2/illustration/{uuid}).
+   *
+   * Returns null on any error so callers can fall back gracefully.
+   */
+  private async _readZimMetadata(zimFilePath: string): Promise<Partial<KiwixBook> | null> {
+    try {
+      if (!(await isValidZimFile(zimFilePath))) {
+        logger.warn(`[KiwixLibraryService] Skipping invalid/corrupted ZIM file: ${zimFilePath}`)
+        return null
+      }
+      const archive = new Archive(zimFilePath)
+
+      const getMeta = (key: string): string | undefined => {
+        try {
+          return archive.getMetadata(key) || undefined
+        } catch {
+          return undefined
+        }
+      }
+
+      let favicon: string | undefined
+      let faviconMimeType: string | undefined
+      try {
+        if (archive.illustrationSizes.size > 0) {
+          const size = archive.illustrationSizes.has(48)
+            ? 48
+            : ([...archive.illustrationSizes][0] as number)
+          const item = archive.getIllustrationItem(size)
+          favicon = item.data.data.toString('base64')
+          faviconMimeType = item.mimetype || undefined
+        }
+      } catch {
+        // ZIM has no illustration — that's fine
+      }
+
+      const rawFilesize =
+        typeof archive.filesize === 'bigint' ? Number(archive.filesize) : archive.filesize
+
+      return {
+        id: archive.uuid || undefined,
+        title: getMeta('Title'),
+        description: getMeta('Description'),
+        language: getMeta('Language'),
+        creator: getMeta('Creator'),
+        publisher: getMeta('Publisher'),
+        name: getMeta('Name'),
+        flavour: getMeta('Flavour'),
+        tags: getMeta('Tags'),
+        date: getMeta('Date'),
+        articleCount: archive.articleCount,
+        mediaCount: archive.mediaCount,
+        size: Math.floor(rawFilesize / 1024),
+        favicon,
+        faviconMimeType,
+      }
+    } catch {
+      return null
+    }
+  }
+
+  private _buildXml(books: KiwixBook[]): string {
+    const builder = new XMLBuilder({
+      ignoreAttributes: false,
+      attributeNamePrefix: '@_',
+      format: true,
+      suppressEmptyNode: false,
+    })
+
+    const obj: Record<string, any> = {
+      library: {
+        '@_version': '20110515',
+        ...(books.length > 0 && {
+          book: books.map((b) => ({
+            '@_id': b.id,
+            '@_path': b.path,
+            '@_title': b.title,
+            ...(b.description !== undefined && { '@_description': b.description }),
+            ...(b.language !== undefined && { '@_language': b.language }),
+            ...(b.creator !== undefined && { '@_creator': b.creator }),
+            ...(b.publisher !== undefined && { '@_publisher': b.publisher }),
+            ...(b.name !== undefined && { '@_name': b.name }),
+            ...(b.flavour !== undefined && { '@_flavour': b.flavour }),
+            ...(b.tags !== undefined && { '@_tags': b.tags }),
+            ...(b.faviconMimeType !== undefined && { '@_faviconMimeType': b.faviconMimeType }),
+            ...(b.favicon !== undefined && { '@_favicon': b.favicon }),
+            ...(b.date !== undefined && { '@_date': b.date }),
+            ...(b.articleCount !== undefined && { '@_articleCount': b.articleCount }),
+            ...(b.mediaCount !== undefined && { '@_mediaCount': b.mediaCount }),
+            ...(b.size !== undefined && { '@_size': b.size }),
+          })),
+        }),
+      },
+    }
+
+    return XML_DECLARATION + builder.build(obj)
+  }
+
+  private async _atomicWrite(content: string): Promise<void> {
+    const filePath = this.getLibraryFilePath()
+    const tmpPath = `${filePath}.tmp.${randomUUID()}`
+    await writeFile(tmpPath, content, 'utf-8')
+    await rename(tmpPath, filePath)
+  }
+
+  private _parseExistingBooks(xmlContent: string): KiwixBook[] {
+    const parser = new XMLParser({
+      ignoreAttributes: false,
+      attributeNamePrefix: '@_',
+      isArray: (name) => name === 'book',
+    })
+
+    const parsed = parser.parse(xmlContent)
+    const books: any[] = parsed?.library?.book ?? []
+
+    return books
+      .map((b) => ({
+        id: b['@_id'] ?? '',
+        path: b['@_path'] ?? '',
+        title: b['@_title'] ?? '',
+        description: b['@_description'],
+        language: b['@_language'],
+        creator: b['@_creator'],
+        publisher: b['@_publisher'],
+        name: b['@_name'],
+        flavour: b['@_flavour'],
+        tags: b['@_tags'],
+        faviconMimeType: b['@_faviconMimeType'],
+        favicon: b['@_favicon'],
+        date: b['@_date'],
+        articleCount:
+          b['@_articleCount'] !== undefined ? Number(b['@_articleCount']) : undefined,
+        mediaCount: b['@_mediaCount'] !== undefined ? Number(b['@_mediaCount']) : undefined,
+        size: b['@_size'] !== undefined ? Number(b['@_size']) : undefined,
+      }))
+      .filter((b) => b.id && b.path)
+  }
+
+  /**
+   * Returns the number of books currently listed in the library XML, or 0 if the
+   * file doesn't exist yet. Used to report a before/after delta on a manual rescan.
+   */
+  async getBookCount(): Promise<number> {
+    try {
+      const content = await readFile(this.getLibraryFilePath(), 'utf-8')
+      return this._parseExistingBooks(content).length
+    } catch (err: any) {
+      if (err.code === 'ENOENT') return 0
+      throw err
+    }
+  }
+
+  /**
+   * True if the library XML parses and has a <library> root. A truncated or
+   * corrupt file (e.g. an interrupted write) fails this even though it exists,
+   * so the caller can rebuild rather than leave Kiwix serving a broken library.
+   * An empty-but-well-formed library is considered valid (nothing to repair).
+   */
+  private _isValidLibraryXml(xmlContent: string): boolean {
+    try {
+      const parser = new XMLParser({
+        ignoreAttributes: false,
+        attributeNamePrefix: '@_',
+        isArray: (name) => name === 'book',
+      })
+      const parsed = parser.parse(xmlContent)
+      return parsed?.library !== undefined && parsed?.library !== null
+    } catch {
+      return false
+    }
+  }
+
+  /**
+   * Boot-time safety net: if the library XML is missing or unparseable, rebuild
+   * it from the ZIM files on disk so Kiwix (running in library mode with
+   * --monitorLibrary) doesn't come up serving an empty/broken library with no
+   * path to recovery. This covers files lost or corrupted outside the normal
+   * download flow (storage relocation, interrupted write, manual deletion).
+   *
+   * Returns true if a rebuild was performed. Filesystem errors other than
+   * "not found" are surfaced rather than masked by a rebuild.
+   */
+  async ensureLibraryXmlHealthy(): Promise<boolean> {
+    let content: string
+    try {
+      content = await readFile(this.getLibraryFilePath(), 'utf-8')
+    } catch (err: any) {
+      if (err?.code === 'ENOENT') {
+        logger.warn('[KiwixLibraryService] Library XML missing on startup; rebuilding from disk.')
+        await this.rebuildFromDisk()
+        return true
+      }
+      throw err
+    }
+
+    if (this._isValidLibraryXml(content)) return false
+
+    logger.warn('[KiwixLibraryService] Library XML present but invalid; rebuilding from disk.')
+    await this.rebuildFromDisk()
+    return true
+  }
+
+  async rebuildFromDisk(opts?: { excludeFilenames?: string[] }): Promise<number> {
+    const dirPath = join(process.cwd(), ZIM_STORAGE_PATH)
+    await ensureDirectoryExists(dirPath)
+
+    let entries: string[] = []
+    try {
+      entries = await readdir(dirPath)
+    } catch {
+      entries = []
+    }
+
+    const excludeSet = new Set(opts?.excludeFilenames ?? [])
+    const zimFiles = entries.filter((name) => name.endsWith('.zim') && !excludeSet.has(name))
+
+    const books: KiwixBook[] = []
+    for (const filename of zimFiles) {
+      const meta = await this._readZimMetadata(join(dirPath, filename))
+      if (meta === null) {
+        logger.warn(`[KiwixLibraryService] Skipping unreadable ZIM file: ${filename}`)
+        continue
+      }
+      const containerPath = `${CONTAINER_DATA_PATH}/${filename}`
+      books.push({
+        ...meta,
+        // Override fields that must be derived locally, not from ZIM metadata
+        id: meta?.id ?? filename.slice(0, -4),
+        path: containerPath,
+        title: meta?.title ?? this._filenameToTitle(filename),
+      })
+    }
+
+    const xml = this._buildXml(books)
+    await this._atomicWrite(xml)
+    logger.info(`[KiwixLibraryService] Rebuilt library XML with ${books.length} book(s).`)
+    return books.length
+  }
+
+  async addBook(filename: string): Promise<void> {
+    const zimFilename = filename.endsWith('.zim') ? filename : `${filename}.zim`
+    const containerPath = `${CONTAINER_DATA_PATH}/${zimFilename}`
+
+    const filePath = this.getLibraryFilePath()
+    let existingBooks: KiwixBook[] = []
+
+    try {
+      const content = await readFile(filePath, 'utf-8')
+      existingBooks = this._parseExistingBooks(content)
+    } catch (err: any) {
+      if (err.code === 'ENOENT') {
+        // XML doesn't exist yet — rebuild from disk; the completed download is already there
+        await this.rebuildFromDisk()
+        return
+      }
+      throw err
+    }
+
+    if (existingBooks.some((b) => b.path === containerPath)) {
+      logger.info(`[KiwixLibraryService] ${zimFilename} already in library, skipping.`)
+      return
+    }
+
+    const fullPath = join(process.cwd(), ZIM_STORAGE_PATH, zimFilename)
+    const meta = await this._readZimMetadata(fullPath)
+
+    if (meta === null) {
+      logger.error(`[KiwixLibraryService] Cannot add ${zimFilename}: file is invalid or corrupted.`)
+      return
+    }
+
+    existingBooks.push({
+      ...meta,
+      id: meta?.id ?? zimFilename.slice(0, -4),
+      path: containerPath,
+      title: meta?.title ?? this._filenameToTitle(zimFilename),
+    })
+
+    const xml = this._buildXml(existingBooks)
+    await this._atomicWrite(xml)
+    logger.info(`[KiwixLibraryService] Added ${zimFilename} to library XML.`)
+  }
+
+  async removeBook(filename: string): Promise<void> {
+    const zimFilename = filename.endsWith('.zim') ? filename : `${filename}.zim`
+    const containerPath = `${CONTAINER_DATA_PATH}/${zimFilename}`
+
+    const filePath = this.getLibraryFilePath()
+    let existingBooks: KiwixBook[] = []
+
+    try {
+      const content = await readFile(filePath, 'utf-8')
+      existingBooks = this._parseExistingBooks(content)
+    } catch (err: any) {
+      if (err.code === 'ENOENT') {
+        logger.warn(`[KiwixLibraryService] Library XML not found, nothing to remove.`)
+        return
+      }
+      throw err
+    }
+
+    const filtered = existingBooks.filter((b) => b.path !== containerPath)
+
+    if (filtered.length === existingBooks.length) {
+      logger.info(`[KiwixLibraryService] ${zimFilename} not found in library, nothing to remove.`)
+      return
+    }
+
+    const xml = this._buildXml(filtered)
+    await this._atomicWrite(xml)
+    logger.info(`[KiwixLibraryService] Removed ${zimFilename} from library XML.`)
+  }
+}

admin/app/services/map_service.ts

@@ -1,6 +1,5 @@
 import { BaseStylesFile, MapLayer } from '../../types/maps.js'
 import {
-  DownloadCollectionOperation,
   DownloadRemoteSuccessCallback,
   FileEntry,
 } from '../../types/files.js'
@@ -9,22 +8,54 @@ import { extract } from 'tar'
 import env from '#start/env'
 import {
   listDirectoryContentsRecursive,
-  listDirectoryContents,
   getFileStatsIfExists,
   deleteFileIfExists,
   getFile,
   ensureDirectoryExists,
 } from '../utils/fs.js'
-import { join } from 'path'
+import { join, resolve, sep } from 'path'
 import urlJoin from 'url-join'
-import axios from 'axios'
 import { RunDownloadJob } from '#jobs/run_download_job'
+import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job'
 import logger from '@adonisjs/core/services/logger'
-import { CuratedCollectionsFile, CuratedCollectionWithStatus } from '../../types/downloads.js'
-import CuratedCollection from '#models/curated_collection'
-import vine from '@vinejs/vine'
-import { curatedCollectionsFileSchema } from '#validators/curated_collections'
-import CuratedCollectionResource from '#models/curated_collection_resource'
+import { assertNotPrivateUrl } from '#validators/common'
+import InstalledResource from '#models/installed_resource'
+import { CollectionManifestService } from './collection_manifest_service.js'
+import { decideSupersededDeletion } from '../utils/superseded_resource.js'
+import type { CollectionWithStatus, MapsSpec } from '../../types/collections.js'
+import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps.js'
+import {
+  EXTRACT_DEFAULT_MAX_ZOOM,
+  EXTRACT_MAX_ZOOM,
+  EXTRACT_MIN_ZOOM,
+  PMTILES_BINARY_PATH,
+  WORLD_BASEMAP_FILENAME,
+  WORLD_BASEMAP_MAX_ZOOM,
+  WORLD_BASEMAP_SOURCE_NAME,
+  buildPmtilesExtractArgs,
+} from '../../constants/map_regions.js'
+import { CountriesService } from './countries_service.js'
+import { execFile } from 'child_process'
+import { createHash, randomBytes } from 'crypto'
+import { tmpdir } from 'os'
+import { promisify } from 'util'
+
+const execFileAsync = promisify(execFile)
+const DRY_RUN_TIMEOUT_MS = 60_000
+const DRY_RUN_MAX_BUFFER = 256 * 1024
+// Real extract of z0-5 world tiles; generous to tolerate slow/metered links
+// since a failure leaves the map grey for uncovered regions.
+const WORLD_BASEMAP_EXTRACT_TIMEOUT_MS = 5 * 60_000
+
+const PROTOMAPS_BUILDS_METADATA_URL = 'https://build-metadata.protomaps.dev/builds.json'
+const PROTOMAPS_BUILD_BASE_URL = 'https://build.protomaps.com'
+
+export interface ProtomapsBuildInfo {
+  url: string
+  date: string
+  size: number
+  key: string
+}
 
 const BASE_ASSETS_MIME_TYPES = [
   'application/gzip',
@@ -32,15 +63,11 @@ const BASE_ASSETS_MIME_TYPES = [
   'application/octet-stream',
 ]
 
-const COLLECTIONS_URL =
-  'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/collections/maps.json'
-
 const PMTILES_ATTRIBUTION =
   '<a href="https://github.com/protomaps/basemaps">Protomaps</a> © <a href="https://openstreetmap.org">OpenStreetMap</a>'
 const PMTILES_MIME_TYPES = ['application/vnd.pmtiles', 'application/octet-stream']
 
 interface IMapService {
-  downloadCollection: DownloadCollectionOperation
   downloadRemoteSuccessCallback: DownloadRemoteSuccessCallback
 }
 
@@ -50,10 +77,16 @@ export class MapService implements IMapService {
   private readonly basemapsAssetsDir = 'basemaps-assets'
   private readonly baseAssetsTarFile = 'base-assets.tar.gz'
   private readonly baseDirPath = join(process.cwd(), this.mapStoragePath)
+  private baseAssetsExistCache: boolean | null = null
+  private worldBasemapReady = false
+  private worldBasemapInFlight: Promise<void> | null = null
 
   async listRegions() {
     const files = (await this.listAllMapStorageItems()).filter(
-      (item) => item.type === 'file' && item.name.endsWith('.pmtiles')
+      (item) =>
+        item.type === 'file' &&
+        item.name.endsWith('.pmtiles') &&
+        item.name !== WORLD_BASEMAP_FILENAME
     )
 
     return {
@@ -93,37 +126,46 @@ export class MapService implements IMapService {
 
     await deleteFileIfExists(tempTarPath)
 
+    // Invalidate cache since we just downloaded new assets
+    this.baseAssetsExistCache = true
+
     return true
   }
 
-  async downloadCollection(slug: string) {
-    const collection = await CuratedCollection.query()
-      .where('slug', slug)
-      .andWhere('type', 'map')
-      .first()
-    if (!collection) {
-      return null
-    }
+  async downloadCollection(slug: string): Promise<string[] | null> {
+    const manifestService = new CollectionManifestService()
+    const spec = await manifestService.getSpecWithFallback<MapsSpec>('maps')
+    if (!spec) return null
 
-    const resources = await collection.related('resources').query().where('downloaded', false)
-    if (resources.length === 0) {
-      return null
-    }
+    const collection = spec.collections.find((c) => c.slug === slug)
+    if (!collection) return null
+
+    // Filter out already installed
+    const installed = await InstalledResource.query().where('resource_type', 'map')
+    const installedIds = new Set(installed.map((r) => r.resource_id))
+    const toDownload = collection.resources.filter((r) => !installedIds.has(r.id))
+
+    if (toDownload.length === 0) return null
 
-    const downloadUrls = resources.map((res) => res.url)
     const downloadFilenames: string[] = []
 
-    for (const url of downloadUrls) {
-      const existing = await RunDownloadJob.getByUrl(url)
-      if (existing) {
-        logger.warn(`[MapService] Download already in progress for URL ${url}, skipping.`)
+    for (const resource of toDownload) {
+      try {
+        assertNotPrivateUrl(resource.url)
+      } catch {
+        logger.warn(`[MapService] Blocked download from private/loopback URL: ${resource.url}`)
         continue
       }
 
-      // Extract the filename from the URL
-      const filename = url.split('/').pop()
+      const existing = await RunDownloadJob.getActiveByUrl(resource.url)
+      if (existing) {
+        logger.warn(`[MapService] Download already in progress for URL ${resource.url}, skipping.`)
+        continue
+      }
+
+      const filename = resource.url.split('/').pop()
       if (!filename) {
-        logger.warn(`[MapService] Could not determine filename from URL ${url}, skipping.`)
+        logger.warn(`[MapService] Could not determine filename from URL ${resource.url}, skipping.`)
         continue
       }
 
@@ -131,12 +173,18 @@ export class MapService implements IMapService {
       const filepath = join(process.cwd(), this.mapStoragePath, 'pmtiles', filename)
 
       await RunDownloadJob.dispatch({
-        url,
+        url: resource.url,
         filepath,
         timeout: 30000,
         allowedMimeTypes: PMTILES_MIME_TYPES,
         forceNew: true,
         filetype: 'map',
+        title: (resource as any).title || undefined,
+        resourceMetadata: {
+          resource_id: resource.id,
+          version: resource.version,
+          collection_ref: slug,
+        },
       })
     }
 
@@ -144,11 +192,66 @@ export class MapService implements IMapService {
   }
 
   async downloadRemoteSuccessCallback(urls: string[], _: boolean) {
-    const resources = await CuratedCollectionResource.query().whereIn('url', urls)
-    for (const resource of resources) {
-      resource.downloaded = true
-      await resource.save()
-      logger.info(`[MapService] Marked resource as downloaded: ${resource.url}`)
+    // Create InstalledResource entries for downloaded map files
+    for (const url of urls) {
+      const filename = url.split('/').pop()
+      if (!filename) continue
+
+      const parsed = CollectionManifestService.parseMapFilename(filename)
+      if (!parsed) continue
+
+      const pmtilesDir = join(process.cwd(), this.mapStoragePath, 'pmtiles')
+      const filepath = join(pmtilesDir, filename)
+      const stats = await getFileStatsIfExists(filepath)
+
+      try {
+        // Capture the prior install for this resource_id before updateOrCreate
+        // overwrites it, so we know the old file to clean up (#634).
+        const prior = await InstalledResource.query()
+          .where('resource_id', parsed.resource_id)
+          .where('resource_type', 'map')
+          .first()
+
+        const { DateTime } = await import('luxon')
+        await InstalledResource.updateOrCreate(
+          { resource_id: parsed.resource_id, resource_type: 'map' },
+          {
+            version: parsed.version,
+            url: url,
+            file_path: filepath,
+            file_size_bytes: stats ? Number(stats.size) : null,
+            installed_at: DateTime.now(),
+          }
+        )
+        logger.info(`[MapService] Created InstalledResource entry for: ${parsed.resource_id}`)
+
+        // Remove the superseded prior version's pmtiles file if every safety
+        // rail passes (see decideSupersededDeletion). Maps have no library index,
+        // so a direct delete of the recorded old file is sufficient.
+        const decision = decideSupersededDeletion({
+          existing: prior ? { file_path: prior.file_path, version: prior.version } : null,
+          newFilePath: filepath,
+          newVersion: parsed.version,
+          newFileExists: !!stats,
+          storageBaseDir: pmtilesDir,
+        })
+        if (decision.delete && decision.path) {
+          try {
+            await deleteFileIfExists(decision.path)
+            logger.info(
+              `[MapService] Removed superseded ${parsed.resource_id} file: ${decision.path}`
+            )
+          } catch (err) {
+            logger.warn(`[MapService] Failed to remove superseded file ${decision.path}:`, err)
+          }
+        } else if (decision.reason !== 'first_install' && decision.reason !== 'same_file') {
+          logger.info(
+            `[MapService] Kept prior ${parsed.resource_id} file (reason: ${decision.reason})`
+          )
+        }
+      } catch (error) {
+        logger.error(`[MapService] Failed to create InstalledResource for ${filename}:`, error)
+      }
     }
   }
 
@@ -158,7 +261,7 @@ export class MapService implements IMapService {
       throw new Error(`Invalid PMTiles file URL: ${url}. URL must end with .pmtiles`)
     }
 
-    const existing = await RunDownloadJob.getByUrl(url)
+    const existing = await RunDownloadJob.getActiveByUrl(url)
     if (existing) {
       throw new Error(`Download already in progress for URL ${url}`)
     }
@@ -170,6 +273,21 @@ export class MapService implements IMapService {
 
     const filepath = join(process.cwd(), this.mapStoragePath, 'pmtiles', filename)
 
+
+    // First, ensure base assets are present - regions depend on them
+    const baseAssetsExist = await this.ensureBaseAssets()
+    if (!baseAssetsExist) {
+      throw new Error(
+        'Base map assets are missing and could not be downloaded. Please check your connection and try again.'
+      )
+    }
+
+    // Parse resource metadata
+    const parsedFilename = CollectionManifestService.parseMapFilename(filename)
+    const resourceMetadata = parsedFilename
+      ? { resource_id: parsedFilename.resource_id, version: parsedFilename.version, collection_ref: null }
+      : undefined
+
     // Dispatch background job
     const result = await RunDownloadJob.dispatch({
       url,
@@ -178,6 +296,7 @@ export class MapService implements IMapService {
       allowedMimeTypes: PMTILES_MIME_TYPES,
       forceNew: true,
       filetype: 'map',
+      resourceMetadata,
     })
 
     if (!result.job) {
@@ -196,6 +315,7 @@ export class MapService implements IMapService {
     url: string
   ): Promise<{ filename: string; size: number } | { message: string }> {
     try {
+      assertNotPrivateUrl(url)
       const parsed = new URL(url)
       if (!parsed.pathname.endsWith('.pmtiles')) {
         throw new Error(`Invalid PMTiles file URL: ${url}. URL must end with .pmtiles`)
@@ -207,6 +327,7 @@ export class MapService implements IMapService {
       }
 
       // Perform a HEAD request to get the content length
+      const { default: axios } = await import('axios')
       const response = await axios.head(url)
 
       if (response.status !== 200) {
@@ -214,15 +335,16 @@ export class MapService implements IMapService {
       }
 
       const contentLength = response.headers['content-length']
-      const size = contentLength ? parseInt(contentLength, 10) : 0
+      const size = contentLength ? parseInt(contentLength.toString(), 10) : 0
 
       return { filename, size }
-    } catch (error) {
-      return { message: `Preflight check failed: ${error.message}` }
+    } catch (error: any) {
+      logger.error({ err: error }, '[MapService] Preflight check failed for URL')
+      return { message: 'Preflight check failed. Please verify the URL is valid and accessible.' }
     }
   }
 
-  async generateStylesJSON() {
+  async generateStylesJSON(host: string | null = null, protocol: string = 'http'): Promise<BaseStylesFile> {
     if (!(await this.checkBaseAssetsExist())) {
       throw new Error('Base map assets are missing from storage/maps')
     }
@@ -236,12 +358,15 @@ export class MapService implements IMapService {
     const rawStyles = JSON.parse(baseStyle.toString()) as BaseStylesFile
 
     const regions = (await this.listRegions()).files
-    const sources = this.generateSourcesArray(regions)
 
-    const localUrl = env.get('URL')
-    const withProtocol = localUrl.startsWith('http') ? localUrl : `http://${localUrl}`
-    const baseUrlPath = urlJoin(this.mapStoragePath, this.basemapsAssetsDir)
-    const baseUrl = new URL(baseUrlPath, withProtocol).toString()
+    /** If we have the host, use it to build public URLs, otherwise we'll fallback to defaults
+    * This is mainly useful because we need to know what host the user is accessing from in order to
+    * properly generate URLs in the styles file
+    * e.g. user is accessing from "example.com", but we would by default generate "localhost:8080/..." so maps would
+    * fail to load.
+    */
+    const sources = this.generateSourcesArray(host, regions, protocol)
+    const baseUrl = this.getPublicFileBaseUrl(host, this.basemapsAssetsDir, protocol)
 
     const styles = await this.generateStylesFile(
       rawStyles,
@@ -253,62 +378,112 @@ export class MapService implements IMapService {
     return styles
   }
 
-  async checkBaseAssetsExist() {
-    const storageContents = await this.listMapStorageItems()
-    const baseStyleItem = storageContents.find(
-      (item) => item.type === 'file' && item.name === this.baseStylesFile
-    )
-    const basemapsAssetsItem = storageContents.find(
-      (item) => item.type === 'directory' && item.name === this.basemapsAssetsDir
-    )
-
-    return !!baseStyleItem && !!basemapsAssetsItem
-  }
-
-  async listCuratedCollections(): Promise<CuratedCollectionWithStatus[]> {
-    const collections = await CuratedCollection.query().where('type', 'map').preload('resources')
-    return collections.map((collection) => ({
-      ...(collection.serialize() as CuratedCollection),
-      all_downloaded: collection.resources.every((res) => res.downloaded),
-    }))
+  async listCuratedCollections(): Promise<CollectionWithStatus[]> {
+    const manifestService = new CollectionManifestService()
+    return manifestService.getMapCollectionsWithStatus()
   }
 
   async fetchLatestCollections(): Promise<boolean> {
+    const manifestService = new CollectionManifestService()
+    return manifestService.fetchAndCacheSpec('maps')
+  }
+
+  async ensureBaseAssets(): Promise<boolean> {
+    const exists = await this.checkBaseAssetsExist()
+    if (!exists) {
+      const downloaded = await this.downloadBaseAssets()
+      if (!downloaded) return false
+    }
+
     try {
-      const response = await axios.get<CuratedCollectionsFile>(COLLECTIONS_URL)
+      await this.ensureWorldBasemap()
+    } catch (err) {
+      logger.warn(`[MapService] World basemap setup failed, continuing without it: ${err}`)
+    }
 
-      const validated = await vine.validate({
-        schema: curatedCollectionsFileSchema,
-        data: response.data,
+    return true
+  }
+
+  /**
+   * Extract a low-zoom global basemap once so the map isn't grey outside a
+   * regional extract's polygon. Cheap (~15 MB, a handful of HTTP range
+   * requests) and layered underneath regional sources at render time.
+   *
+   * Memoizes success in-process, and de-duplicates concurrent callers via a
+   * shared in-flight promise so two simultaneous `/maps` requests on a cold
+   * start don't both launch `pmtiles extract` against the same output path.
+   */
+  private async ensureWorldBasemap(): Promise<void> {
+    if (this.worldBasemapReady) return
+    if (this.worldBasemapInFlight) return this.worldBasemapInFlight
+    this.worldBasemapInFlight = this._setupWorldBasemap().finally(() => {
+      this.worldBasemapInFlight = null
+    })
+    return this.worldBasemapInFlight
+  }
+
+  private async _setupWorldBasemap(): Promise<void> {
+    const basePath = resolve(join(this.baseDirPath, 'pmtiles'))
+    const filepath = resolve(join(basePath, WORLD_BASEMAP_FILENAME))
+    if (!filepath.startsWith(basePath + sep)) {
+      throw new Error('Invalid world basemap path')
+    }
+
+    await ensureDirectoryExists(basePath)
+
+    const existing = await getFileStatsIfExists(filepath)
+    if (existing && Number(existing.size) > 0) {
+      this.worldBasemapReady = true
+      return
+    }
+
+    const info = await this.getGlobalMapInfo()
+    const args = buildPmtilesExtractArgs({
+      sourceUrl: info.url,
+      outputFilepath: filepath,
+      maxzoom: WORLD_BASEMAP_MAX_ZOOM,
+      downloadThreads: 4,
+    })
+
+    logger.info(
+      `[MapService] Extracting world basemap (z0-${WORLD_BASEMAP_MAX_ZOOM}) from ${info.url}`
+    )
+    try {
+      await execFileAsync(PMTILES_BINARY_PATH, args, {
+        timeout: WORLD_BASEMAP_EXTRACT_TIMEOUT_MS,
+        maxBuffer: DRY_RUN_MAX_BUFFER,
       })
-
-      for (const collection of validated.collections) {
-        const collectionResult = await CuratedCollection.updateOrCreate(
-          { slug: collection.slug },
-          {
-            ...collection,
-            type: 'map',
-          }
-        )
-        logger.info(`[MapService] Upserted curated collection: ${collection.slug}`)
-
-        await collectionResult.related('resources').createMany(collection.resources)
-        logger.info(
-          `[MapService] Upserted ${collection.resources.length} resources for collection: ${collection.slug}`
-        )
-      }
-
-      return true
-    } catch (error) {
-      console.error(error)
-      logger.error(`[MapService] Failed to download latest Kiwix collections:`, error)
-      return false
+      this.worldBasemapReady = true
+    } catch (err: any) {
+      await deleteFileIfExists(filepath)
+      throw new Error(
+        `pmtiles extract for world basemap failed: ${err.message}. stderr: ${err.stderr ?? ''}`
+      )
     }
   }
 
-  private async listMapStorageItems(): Promise<FileEntry[]> {
+  private async checkBaseAssetsExist(useCache: boolean = true): Promise<boolean> {
+    // Return cached result if available and caching is enabled
+    if (useCache && this.baseAssetsExistCache !== null) {
+      return this.baseAssetsExistCache
+    }
+
     await ensureDirectoryExists(this.baseDirPath)
-    return await listDirectoryContents(this.baseDirPath)
+
+    const baseStylePath = join(this.baseDirPath, this.baseStylesFile)
+    const basemapsAssetsPath = join(this.baseDirPath, this.basemapsAssetsDir)
+
+    const [baseStyleExists, basemapsAssetsExists] = await Promise.all([
+      getFileStatsIfExists(baseStylePath),
+      getFileStatsIfExists(basemapsAssetsPath),
+    ])
+
+    const exists = !!baseStyleExists && !!basemapsAssetsExists
+
+    // update cache
+    this.baseAssetsExistCache = exists
+
+    return exists
   }
 
   private async listAllMapStorageItems(): Promise<FileEntry[]> {
@@ -316,28 +491,76 @@ export class MapService implements IMapService {
     return await listDirectoryContentsRecursive(this.baseDirPath)
   }
 
-  private generateSourcesArray(regions: FileEntry[]): BaseStylesFile['sources'][] {
-    const localUrl = env.get('URL')
-    const sources: BaseStylesFile['sources'][] = []
+  /**
+   * Compare two map-file versions (YYYY-MM, or null for an undated legacy file). Returns >0 if
+   * `a` is newer than `b`, <0 if older, 0 if equal. A dated build is always newer than an undated
+   * legacy file; two dated builds compare lexicographically (correct for zero-padded YYYY-MM).
+   */
+  private static compareMapVersions(a: string | null, b: string | null): number {
+    if (a === b) return 0
+    if (a === null) return -1
+    if (b === null) return 1
+    return a < b ? -1 : 1
+  }
 
+  private generateSourcesArray(host: string | null, regions: FileEntry[], protocol: string = 'http'): BaseStylesFile['sources'][] {
+    const sources: BaseStylesFile['sources'][] = []
+    const baseUrl = this.getPublicFileBaseUrl(host, 'pmtiles', protocol)
+
+    // World basemap goes first so its layers render underneath regional extracts.
+    // Only emitted when ensureWorldBasemap() succeeded — otherwise the style would
+    // reference a file that doesn't exist and produce 404s on every tile request.
+    if (this.worldBasemapReady) {
+      const worldSource: BaseStylesFile['sources'] = {}
+      worldSource[WORLD_BASEMAP_SOURCE_NAME] = {
+        type: 'vector',
+        attribution: PMTILES_ATTRIBUTION,
+        url: `pmtiles://${urlJoin(baseUrl, WORLD_BASEMAP_FILENAME)}`,
+      }
+      sources.push(worldSource)
+    }
+
+    // Dedupe by region name, keeping only the newest file per region. The source name is the
+    // date-stripped region (e.g. both "washington.pmtiles" and "washington_2025-12.pmtiles" map
+    // to "washington"). Emitting both produces duplicate source keys and duplicate layer ids,
+    // which MapLibre rejects outright — blanking the ENTIRE map, not just that region. Old copies
+    // linger when a newer curated version installs (#634), so guard against it here so the style
+    // stays valid even if cleanup hasn't run. A dated build beats an undated legacy file; between
+    // two dated builds the later YYYY-MM wins (lexicographic compare is correct for that format).
+    const bestByRegion = new Map<string, { region: FileEntry; version: string | null }>()
     for (const region of regions) {
       if (region.type === 'file' && region.name.endsWith('.pmtiles')) {
-        const regionName = region.name.replace('.pmtiles', '')
-        const source: BaseStylesFile['sources'] = {}
-        const sourceUrl = new URL(
-          urlJoin(this.mapStoragePath, 'pmtiles', region.name),
-          localUrl.startsWith('http') ? localUrl : `http://${localUrl}`
-        ).toString()
-
-        source[regionName] = {
-          type: 'vector',
-          attribution: PMTILES_ATTRIBUTION,
-          url: `pmtiles://${sourceUrl}`,
+        const parsed = CollectionManifestService.parseMapFilename(region.name)
+        const regionName = parsed ? parsed.resource_id : region.name.replace('.pmtiles', '')
+        const version = parsed?.version ?? null
+        const existing = bestByRegion.get(regionName)
+        if (!existing || MapService.compareMapVersions(version, existing.version) > 0) {
+          if (existing) {
+            logger.warn(
+              `[MapService] Duplicate map region "${regionName}": using "${region.name}" over "${existing.region.name}" (keeping newest)`
+            )
+          }
+          bestByRegion.set(regionName, { region, version })
+        } else {
+          logger.warn(
+            `[MapService] Duplicate map region "${regionName}": skipping "${region.name}" in favor of "${existing.region.name}" (keeping newest)`
+          )
         }
-        sources.push(source)
       }
     }
 
+    for (const [regionName, { region }] of bestByRegion) {
+      const source: BaseStylesFile['sources'] = {}
+      const sourceUrl = urlJoin(baseUrl, region.name)
+
+      source[regionName] = {
+        type: 'vector',
+        attribution: PMTILES_ATTRIBUTION,
+        url: `pmtiles://${sourceUrl}`,
+      }
+      sources.push(source)
+    }
+
     return sources
   }
 
@@ -373,13 +596,283 @@ export class MapService implements IMapService {
     return template
   }
 
-  async delete(file: string): Promise<void> {
-    let fileName = file
-    if (!fileName.endsWith('.zim')) {
-      fileName += '.zim'
+  async getGlobalMapInfo(): Promise<ProtomapsBuildInfo> {
+    const { default: axios } = await import('axios')
+    const response = await axios.get(PROTOMAPS_BUILDS_METADATA_URL, { timeout: 15000 })
+    const builds = response.data as Array<{ key: string; size: number }>
+
+    if (!builds || builds.length === 0) {
+      throw new Error('No protomaps builds found')
     }
 
-    const fullPath = join(this.baseDirPath, fileName)
+    // Latest build first
+    const sorted = builds.sort((a, b) => b.key.localeCompare(a.key))
+    const latest = sorted[0]
+
+    const dateStr = latest.key.replace('.pmtiles', '')
+    const date = `${dateStr.slice(0, 4)}-${dateStr.slice(4, 6)}-${dateStr.slice(6, 8)}`
+
+    return {
+      url: `${PROTOMAPS_BUILD_BASE_URL}/${latest.key}`,
+      date,
+      size: latest.size,
+      key: latest.key,
+    }
+  }
+
+  async downloadGlobalMap(): Promise<{ filename: string; jobId?: string }> {
+    const info = await this.getGlobalMapInfo()
+
+    const existing = await RunDownloadJob.getByUrl(info.url)
+    if (existing) {
+      throw new Error(`Download already in progress for URL ${info.url}`)
+    }
+
+    const basePath = resolve(join(this.baseDirPath, 'pmtiles'))
+    const filepath = resolve(join(basePath, info.key))
+
+    // Prevent path traversal — resolved path must stay within the storage directory
+    if (!filepath.startsWith(basePath + sep)) {
+      throw new Error('Invalid filename')
+    }
+
+    // First, ensure base assets are present - the global map depends on them
+    const baseAssetsExist = await this.ensureBaseAssets()
+    if (!baseAssetsExist) {
+      throw new Error(
+        'Base map assets are missing and could not be downloaded. Please check your connection and try again.'
+      )
+    }
+
+    // forceNew: false so retries resume partial downloads
+    const result = await RunDownloadJob.dispatch({
+      url: info.url,
+      filepath,
+      timeout: 30000,
+      allowedMimeTypes: PMTILES_MIME_TYPES,
+      forceNew: false,
+      filetype: 'map',
+    })
+
+    if (!result.job) {
+      throw new Error('Failed to dispatch download job')
+    }
+
+    logger.info(`[MapService] Dispatched global map download job ${result.job.id}`)
+
+    return {
+      filename: info.key,
+      jobId: result.job?.id,
+    }
+  }
+
+  async listCountries(): Promise<Country[]> {
+    return CountriesService.getInstance().list()
+  }
+
+  async listCountryGroups(): Promise<CountryGroup[]> {
+    return CountriesService.getInstance().listGroups()
+  }
+
+  async extractPreflight(params: {
+    countries: CountryCode[]
+    maxzoom?: number
+  }): Promise<MapExtractPreflight> {
+    this.validateMaxzoom(params.maxzoom)
+    const countries = await CountriesService.getInstance().resolveCodes(params.countries)
+    const regionFilepath = await CountriesService.getInstance().writeRegionFile(countries)
+    const info = await this.getGlobalMapInfo()
+    return this.runDryRun(info, regionFilepath, params.maxzoom)
+  }
+
+  private async runDryRun(
+    info: { url: string; date: string; key: string },
+    regionFilepath: string,
+    maxzoom?: number
+  ): Promise<MapExtractPreflight> {
+    const dryRunOutput = join(tmpdir(), `pmtiles-dry-run-${randomBytes(6).toString('hex')}.pmtiles`)
+    const args = buildPmtilesExtractArgs({
+      sourceUrl: info.url,
+      outputFilepath: dryRunOutput,
+      regionFilepath,
+      maxzoom,
+      dryRun: true,
+    })
+
+    let stdout = ''
+    let stderr = ''
+    try {
+      const result = await execFileAsync(PMTILES_BINARY_PATH, args, {
+        timeout: DRY_RUN_TIMEOUT_MS,
+        maxBuffer: DRY_RUN_MAX_BUFFER,
+      })
+      stdout = result.stdout
+      stderr = result.stderr
+    } catch (err: any) {
+      throw new Error(
+        `pmtiles extract --dry-run failed: ${err.message}. stderr: ${err.stderr ?? ''}`
+      )
+    }
+
+    const parsed = this.parseDryRunOutput(stdout + '\n' + stderr)
+
+    return {
+      tiles: parsed.tiles,
+      bytes: parsed.bytes,
+      source: { url: info.url, date: info.date, key: info.key },
+    }
+  }
+
+  async extractRegion(params: {
+    countries: CountryCode[]
+    maxzoom?: number
+    label?: string
+    estimatedBytes?: number
+  }): Promise<{ filename: string; jobId?: string }> {
+    this.validateMaxzoom(params.maxzoom)
+    const countriesService = CountriesService.getInstance()
+    const countries = await countriesService.resolveCodes(params.countries)
+    const regionFilepath = await countriesService.writeRegionFile(countries)
+    const maxzoom = params.maxzoom ?? EXTRACT_DEFAULT_MAX_ZOOM
+
+    const [baseAssetsExist, info, groups] = await Promise.all([
+      this.ensureBaseAssets(),
+      this.getGlobalMapInfo(),
+      countriesService.listGroups(),
+    ])
+    if (!baseAssetsExist) {
+      throw new Error(
+        'Base map assets are missing and could not be downloaded. Please check your connection and try again.'
+      )
+    }
+
+    const groupMatch = findExactGroupMatch(countries, groups)
+    const slug = this.buildRegionSlug(countries, groupMatch)
+    const dateSlug = info.key.replace('.pmtiles', '')
+    const filename = `${slug}_${dateSlug}_z${maxzoom}.pmtiles`
+    const basePath = resolve(join(this.baseDirPath, 'pmtiles'))
+    const filepath = resolve(join(basePath, filename))
+
+    if (!filepath.startsWith(basePath + sep)) {
+      throw new Error('Invalid filename')
+    }
+
+    let estimatedBytes = params.estimatedBytes ?? 0
+    if (estimatedBytes === 0) {
+      try {
+        const preflight = await this.runDryRun(info, regionFilepath, maxzoom)
+        estimatedBytes = preflight.bytes
+      } catch (err) {
+        logger.warn(`[MapService] extractRegion preflight failed, proceeding without estimate: ${err}`)
+      }
+    }
+
+    const title = params.label ?? this.buildRegionTitle(countries, groupMatch)
+
+    const result = await RunExtractPmtilesJob.dispatch({
+      sourceUrl: info.url,
+      outputFilepath: filepath,
+      regionFilepath,
+      maxzoom,
+      estimatedBytes,
+      filetype: 'map',
+      title,
+      resourceMetadata: {
+        resource_id: slug,
+        version: dateSlug,
+        collection_ref: null,
+      },
+    })
+
+    if (!result.job) {
+      throw new Error('Failed to dispatch extract job')
+    }
+
+    logger.info(
+      `[MapService] Dispatched extract job ${result.job.id} for ${filename} ` +
+        `(countries=[${countries.join(',')}] maxzoom=${maxzoom} est=${estimatedBytes} bytes)`
+    )
+
+    return {
+      filename,
+      jobId: result.job.id,
+    }
+  }
+
+  private buildRegionSlug(countries: CountryCode[], groupMatch: CountryGroup | null): string {
+    if (groupMatch) return groupMatch.id
+    if (countries.length === 1) return countries[0].toLowerCase()
+    const hash = createHash('sha1').update(countries.join(',')).digest('hex').slice(0, 8)
+    return `custom-${hash}`
+  }
+
+  private buildRegionTitle(countries: CountryCode[], groupMatch: CountryGroup | null): string {
+    if (groupMatch) return groupMatch.name
+    if (countries.length === 1) return countries[0]
+    if (countries.length <= 3) return countries.join(', ')
+    return `${countries.slice(0, 2).join(', ')} +${countries.length - 2} more`
+  }
+
+  private validateMaxzoom(maxzoom: number | undefined): void {
+    if (typeof maxzoom !== 'number') return
+    if (
+      !Number.isInteger(maxzoom) ||
+      maxzoom < EXTRACT_MIN_ZOOM ||
+      maxzoom > EXTRACT_MAX_ZOOM
+    ) {
+      throw new Error(
+        `maxzoom must be an integer in [${EXTRACT_MIN_ZOOM}, ${EXTRACT_MAX_ZOOM}]`
+      )
+    }
+  }
+
+  // go-pmtiles output format isn't stable across versions — parse loosely and
+  // fall back to zeros. The extract can still proceed without an estimate.
+  private parseDryRunOutput(output: string): { tiles: number; bytes: number } {
+    let bytes = 0
+    let tiles = 0
+
+    const byteLine = output.match(/archive\s+size\s+of\s+([\d,.]+)\s*(B|KB|MB|GB|TB|bytes?)?/i)
+    if (byteLine) {
+      const raw = parseFloat(byteLine[1].replace(/,/g, ''))
+      const unit = (byteLine[2] ?? 'B').toUpperCase()
+      const multipliers: Record<string, number> = {
+        B: 1,
+        BYTE: 1,
+        BYTES: 1,
+        KB: 1_000,
+        MB: 1_000_000,
+        GB: 1_000_000_000,
+        TB: 1_000_000_000_000,
+      }
+      bytes = Math.round(raw * (multipliers[unit] ?? 1))
+    }
+
+    const tileLine = output.match(/(?:tiles\s+to\s+extract|tiles)[^\d]*([\d,]+)/i)
+    if (tileLine) {
+      tiles = parseInt(tileLine[1].replace(/,/g, ''), 10) || 0
+    }
+
+    return { tiles, bytes }
+  }
+
+  async delete(file: string): Promise<void> {
+    let fileName = file
+    if (!fileName.endsWith('.pmtiles')) {
+      fileName += '.pmtiles'
+    }
+
+    if (fileName === WORLD_BASEMAP_FILENAME) {
+      throw new Error('The world basemap cannot be deleted')
+    }
+
+    const basePath = resolve(join(this.baseDirPath, 'pmtiles'))
+    const fullPath = resolve(join(basePath, fileName))
+
+    // Prevent path traversal — resolved path must stay within the storage directory
+    if (!fullPath.startsWith(basePath + sep)) {
+      throw new Error('Invalid filename')
+    }
 
     const exists = await getFileStatsIfExists(fullPath)
     if (!exists) {
@@ -387,5 +880,80 @@ export class MapService implements IMapService {
     }
 
     await deleteFileIfExists(fullPath)
+
+    // Clean up InstalledResource entry
+    const parsed = CollectionManifestService.parseMapFilename(fileName)
+    if (parsed) {
+      await InstalledResource.query()
+        .where('resource_id', parsed.resource_id)
+        .where('resource_type', 'map')
+        .delete()
+      logger.info(`[MapService] Deleted InstalledResource entry for: ${parsed.resource_id}`)
+    }
+  }
+
+  /**
+   * Gets the appropriate public URL for a map asset depending on environment. The host and protocol that the user
+   * is accessing the maps from must match the host and protocol used in the generated URLs, otherwise maps will fail to load.
+   * If you make changes to this function, you need to ensure it handles all the following cases correctly:
+   * - No host provided (should default to localhost or env URL)
+   * - Host provided as full URL (e.g. "http://example.com:8080")
+   * - Host provided as host:port (e.g. "example.com:8080")
+   * - Host provided as bare hostname (e.g. "example.com")
+   * @param specifiedHost - the host as provided by the user/request, can be null or in various formats (full URL, host:port, bare hostname)
+   * @param childPath - the path to append to the base URL (e.g. "basemaps-assets", "pmtiles")
+   * @param protocol - the protocol to use in the generated URL (e.g. "http", "https"), defaults to "http"
+   * @returns the public URL for the map asset
+   */
+  private getPublicFileBaseUrl(specifiedHost: string | null, childPath: string, protocol: string = 'http'): string {
+    function getHost() {
+      try {
+        const localUrlRaw = env.get('URL')
+        if (!localUrlRaw) return 'localhost'
+
+        const localUrl = new URL(localUrlRaw)
+        return localUrl.host
+      } catch (error) {
+        return 'localhost'
+      }
+    }
+
+    function specifiedHostOrDefault() {
+      if (specifiedHost === null) {
+        return getHost()
+      }
+      // Try as a full URL first (e.g. "http://example.com:8080")
+      try {
+        const specifiedUrl = new URL(specifiedHost)
+        if (specifiedUrl.host) return specifiedUrl.host
+      } catch {}
+      // Try as a bare host or host:port (e.g. "nomad-box:8080", "192.168.1.1:8080", "example.com")
+      try {
+        const specifiedUrl = new URL(`http://${specifiedHost}`)
+        if (specifiedUrl.host) return specifiedUrl.host
+      } catch {}
+      return getHost()
+    }
+
+    const host = specifiedHostOrDefault();
+    const withProtocol = `${protocol}://${host}`
+    const baseUrlPath =
+      process.env.NODE_ENV === 'production' ? childPath : urlJoin(this.mapStoragePath, childPath)
+
+    const baseUrl = new URL(baseUrlPath, withProtocol).toString()
+    return baseUrl
   }
 }
+
+function findExactGroupMatch(
+  countries: CountryCode[],
+  groups: CountryGroup[]
+): CountryGroup | null {
+  return (
+    groups.find(
+      (g) =>
+        g.countries.length === countries.length &&
+        g.countries.every((c, i) => c === countries[i])
+    ) ?? null
+  )
+}

admin/app/services/ollama_service.ts

@@ -0,0 +1,983 @@
+import { inject } from '@adonisjs/core'
+import OpenAI from 'openai'
+import type { ChatCompletionChunk, ChatCompletionMessageParam } from 'openai/resources/chat/completions.js'
+import type { Stream } from 'openai/streaming.js'
+import { NomadOllamaModel } from '../../types/ollama.js'
+import { EMBEDDING_MODEL_NAME, FALLBACK_RECOMMENDED_OLLAMA_MODELS } from '../../constants/ollama.js'
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import logger from '@adonisjs/core/services/logger'
+import axios from 'axios'
+import { DownloadModelJob } from '#jobs/download_model_job'
+import { SERVICE_NAMES } from '../../constants/service_names.js'
+import transmit from '@adonisjs/transmit/services/main'
+import Fuse, { IFuseOptions } from 'fuse.js'
+import { BROADCAST_CHANNELS } from '../../constants/broadcast.js'
+import env from '#start/env'
+import { NOMAD_API_DEFAULT_BASE_URL } from '../../constants/misc.js'
+import KVStore from '#models/kv_store'
+
+const NOMAD_MODELS_API_PATH = '/api/v1/ollama/models'
+const MODELS_CACHE_FILE = path.join(process.cwd(), 'storage', 'ollama-models-cache.json')
+const CACHE_MAX_AGE_MS = 24 * 60 * 60 * 1000 // 24 hours
+
+export type NomadInstalledModel = {
+  name: string
+  size: number
+  digest?: string
+  details?: Record<string, any>
+}
+
+export type NomadChatResponse = {
+  message: { content: string; thinking?: string }
+  done: boolean
+  model: string
+}
+
+export type NomadChatStreamChunk = {
+  message: { content: string; thinking?: string }
+  done: boolean
+}
+
+type ChatInput = {
+  model: string
+  messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>
+  think?: boolean | 'medium'
+  stream?: boolean
+  numCtx?: number
+}
+
+@inject()
+export class OllamaService {
+  private openai: OpenAI | null = null
+  private baseUrl: string | null = null
+  private initPromise: Promise<void> | null = null
+  private isOllamaNative: boolean | null = null
+  private activeDownloads: Map<string, Promise<{ success: boolean; message: string; retryable?: boolean }>> = new Map()
+
+  constructor() {}
+
+  private async _initialize() {
+    if (!this.initPromise) {
+      this.initPromise = (async () => {
+        // Check KVStore for a custom base URL (remote Ollama, LM Studio, llama.cpp, etc.)
+        const customUrl = (await KVStore.getValue('ai.remoteOllamaUrl')) as string | null
+        if (customUrl && customUrl.trim()) {
+          this.baseUrl = customUrl.trim().replace(/\/$/, '')
+        } else {
+          // Fall back to the local Ollama container managed by Docker
+          const dockerService = new (await import('./docker_service.js')).DockerService()
+          const ollamaUrl = await dockerService.getServiceURL(SERVICE_NAMES.OLLAMA)
+          if (!ollamaUrl) {
+            throw new Error('Ollama service is not installed or running.')
+          }
+          this.baseUrl = ollamaUrl.trim().replace(/\/$/, '')
+        }
+
+        this.openai = new OpenAI({
+          apiKey: 'nomad', // Required by SDK; not validated by Ollama/LM Studio/llama.cpp
+          baseURL: `${this.baseUrl}/v1`,
+        })
+      })()
+    }
+    return this.initPromise
+  }
+
+  private async _ensureDependencies() {
+    if (!this.openai) {
+      await this._initialize()
+    }
+  }
+
+  /**
+   * Downloads a model from Ollama with progress tracking. Only works with Ollama backends.
+   * Use dispatchModelDownload() for background job processing where possible.
+   *
+   * @param signal Optional AbortSignal — when triggered, the underlying axios stream is cancelled
+   *               and the method returns a non-retryable failure so callers can mark the job
+   *               unrecoverable in BullMQ and avoid the 40-attempt retry storm.
+   * @param jobId Optional BullMQ job id — included in progress broadcasts so the frontend can
+   *              correlate Transmit events to a cancellable job.
+   */
+  async downloadModel(
+    model: string,
+    progressCallback?: (
+      percent: number,
+      bytes?: { downloadedBytes: number; totalBytes: number }
+    ) => void,
+    signal?: AbortSignal,
+    jobId?: string
+  ): Promise<{ success: boolean; message: string; retryable?: boolean }> {
+    // Deduplicate concurrent downloads of the same model
+    const existing = this.activeDownloads.get(model)
+    if (existing) {
+      logger.info(`[OllamaService] Download already in progress for "${model}", waiting on existing download.`)
+      return existing
+    }
+
+    const downloadPromise = this._doDownloadModel(model, progressCallback, signal, jobId)
+    this.activeDownloads.set(model, downloadPromise)
+    try {
+      return await downloadPromise
+    } finally {
+      this.activeDownloads.delete(model)
+    }
+  }
+
+  private async _doDownloadModel(
+    model: string,
+    progressCallback?: (
+      percent: number,
+      bytes?: { downloadedBytes: number; totalBytes: number }
+    ) => void,
+    signal?: AbortSignal,
+    jobId?: string
+  ): Promise<{ success: boolean; message: string; retryable?: boolean }> {
+    await this._ensureDependencies()
+    if (!this.baseUrl) {
+      return { success: false, message: 'AI service is not initialized.' }
+    }
+
+    try {
+      // See if model is already installed
+      const installedModels = await this.getModels()
+      if (installedModels && installedModels.some((m) => m.name === model)) {
+        logger.info(`[OllamaService] Model "${model}" is already installed.`)
+        return { success: true, message: 'Model is already installed.' }
+      }
+
+      // Model pulling is an Ollama-only operation. Non-Ollama backends (LM Studio, llama.cpp, etc.)
+      // return HTTP 200 for unknown endpoints, so the pull would appear to succeed but do nothing.
+      if (this.isOllamaNative === false) {
+        logger.warn(
+          `[OllamaService] Non-Ollama backend detected — skipping model pull for "${model}". Load the model manually in your AI host.`
+        )
+        return {
+          success: false,
+          message: `Model "${model}" is not available in your AI host. Please load it manually (model pulling is only supported for Ollama backends).`,
+        }
+      }
+
+      // Stream pull via Ollama native API. axios supports `signal` natively for AbortController
+      // integration — when triggered, the request errors with code 'ERR_CANCELED' which we detect
+      // in the catch block below to return a non-retryable cancel result.
+      const pullResponse = await axios.post(
+        `${this.baseUrl}/api/pull`,
+        { model, stream: true },
+        { responseType: 'stream', timeout: 0, signal }
+      )
+
+      // Ollama's pull API reports progress per-digest (each blob). A single model can contain
+      // multiple blobs (weights, tokenizer, template, etc.) and each is reported in turn.
+      // Aggregate across all digests so the UI shows a single monotonically-increasing total,
+      // matching the behavior of the content download progress (Active Downloads section).
+      const digestProgress = new Map<string, { completed: number; total: number }>()
+
+      // Throttle broadcasts to once per BROADCAST_THROTTLE_MS — Ollama can emit hundreds of
+      // progress events per second for fast connections, which would flood the Transmit SSE
+      // channel and cause jittery speed calculations on the frontend.
+      const BROADCAST_THROTTLE_MS = 500
+      let lastBroadcastAt = 0
+
+      await new Promise<void>((resolve, reject) => {
+        let buffer = ''
+        // If the abort fires after headers are received but mid-stream, axios's signal handling
+        // destroys the stream which surfaces as an 'error' event — wire the signal listener so
+        // the promise rejects promptly with a recognizable cancel reason.
+        const onAbort = () => {
+          const err: any = new Error('Download cancelled')
+          err.code = 'ERR_CANCELED'
+          pullResponse.data.destroy(err)
+        }
+        if (signal) {
+          if (signal.aborted) {
+            onAbort()
+            return
+          }
+          signal.addEventListener('abort', onAbort, { once: true })
+        }
+
+        pullResponse.data.on('data', (chunk: Buffer) => {
+          buffer += chunk.toString()
+          const lines = buffer.split('\n')
+          buffer = lines.pop() || ''
+          for (const line of lines) {
+            if (!line.trim()) continue
+            try {
+              const parsed = JSON.parse(line)
+              if (parsed.completed && parsed.total && parsed.digest) {
+                // Update this digest's progress — take the max seen value so transient
+                // out-of-order updates don't make the aggregate jump backwards.
+                const existing = digestProgress.get(parsed.digest)
+                digestProgress.set(parsed.digest, {
+                  completed: Math.max(existing?.completed ?? 0, parsed.completed),
+                  total: Math.max(existing?.total ?? 0, parsed.total),
+                })
+
+                // Compute aggregate across all known blobs
+                let aggCompleted = 0
+                let aggTotal = 0
+                for (const { completed, total } of digestProgress.values()) {
+                  aggCompleted += completed
+                  aggTotal += total
+                }
+
+                const percent = aggTotal > 0
+                  ? parseFloat(((aggCompleted / aggTotal) * 100).toFixed(2))
+                  : 0
+
+                // Throttle broadcasts. Always call the progressCallback though — the worker
+                // uses it to update job state in Redis, which should reflect the latest view.
+                const now = Date.now()
+                if (now - lastBroadcastAt >= BROADCAST_THROTTLE_MS) {
+                  lastBroadcastAt = now
+                  this.broadcastDownloadProgress(model, percent, jobId, {
+                    downloadedBytes: aggCompleted,
+                    totalBytes: aggTotal,
+                  })
+                }
+                if (progressCallback) {
+                  progressCallback(percent, {
+                    downloadedBytes: aggCompleted,
+                    totalBytes: aggTotal,
+                  })
+                }
+              }
+            } catch {
+              // ignore parse errors on partial lines
+            }
+          }
+        })
+        pullResponse.data.on('end', () => {
+          if (signal) signal.removeEventListener('abort', onAbort)
+          resolve()
+        })
+        pullResponse.data.on('error', (err: any) => {
+          if (signal) signal.removeEventListener('abort', onAbort)
+          reject(err)
+        })
+      })
+
+      logger.info(`[OllamaService] Model "${model}" downloaded successfully.`)
+      return { success: true, message: 'Model downloaded successfully.' }
+    } catch (error) {
+      // Detect axios cancel (signal-triggered abort). Don't broadcast an error event for
+      // user-initiated cancels — the cancel handler in DownloadService already broadcasts
+      // a cancelled state. Returning retryable: false prevents BullMQ retries.
+      const isCancelled =
+        axios.isCancel(error) ||
+        (error as any)?.code === 'ERR_CANCELED' ||
+        (error as any)?.name === 'CanceledError'
+      if (isCancelled) {
+        logger.info(`[OllamaService] Model "${model}" download cancelled by user.`)
+        return { success: false, message: 'Download cancelled', retryable: false }
+      }
+
+      const errorMessage = error instanceof Error ? error.message : String(error)
+      logger.error(
+        `[OllamaService] Failed to download model "${model}": ${errorMessage}`
+      )
+
+      // Check for version mismatch (Ollama 412 response)
+      const isVersionMismatch = errorMessage.includes('newer version of Ollama')
+      const userMessage = isVersionMismatch
+        ? 'This model requires a newer version of Ollama. Please update AI Assistant from the Apps page.'
+        : `Failed to download model: ${errorMessage}`
+
+      // Broadcast failure to connected clients so UI can show the error
+      this.broadcastDownloadError(model, userMessage)
+
+      return { success: false, message: userMessage, retryable: !isVersionMismatch }
+    }
+  }
+
+  async dispatchModelDownload(modelName: string): Promise<{ success: boolean; message: string }> {
+    try {
+      logger.info(`[OllamaService] Dispatching model download for ${modelName} via job queue`)
+
+      await DownloadModelJob.dispatch({
+        modelName,
+      })
+
+      return {
+        success: true,
+        message:
+          'Model download has been queued successfully. It will start shortly after Ollama and Open WebUI are ready (if not already).',
+      }
+    } catch (error) {
+      logger.error(
+        `[OllamaService] Failed to dispatch model download for ${modelName}: ${error instanceof Error ? error.message : error}`
+      )
+      return {
+        success: false,
+        message: 'Failed to queue model download. Please try again.',
+      }
+    }
+  }
+
+  public async chat(chatRequest: ChatInput): Promise<NomadChatResponse> {
+    await this._ensureDependencies()
+    if (!this.openai) {
+      throw new Error('AI client is not initialized.')
+    }
+
+    const params: any = {
+      model: chatRequest.model,
+      messages: chatRequest.messages as ChatCompletionMessageParam[],
+      stream: false,
+    }
+    if (chatRequest.think) {
+      params.think = chatRequest.think
+    }
+    if (chatRequest.numCtx) {
+      params.num_ctx = chatRequest.numCtx
+    }
+
+    const response = await this.openai.chat.completions.create(params)
+    const choice = response.choices[0]
+
+    return {
+      message: {
+        content: choice.message.content ?? '',
+        thinking: (choice.message as any).thinking ?? undefined,
+      },
+      done: true,
+      model: response.model,
+    }
+  }
+
+  public async chatStream(chatRequest: ChatInput): Promise<AsyncIterable<NomadChatStreamChunk>> {
+    await this._ensureDependencies()
+    if (!this.openai) {
+      throw new Error('AI client is not initialized.')
+    }
+
+    const params: any = {
+      model: chatRequest.model,
+      messages: chatRequest.messages as ChatCompletionMessageParam[],
+      stream: true,
+    }
+    if (chatRequest.think) {
+      params.think = chatRequest.think
+    }
+    if (chatRequest.numCtx) {
+      params.num_ctx = chatRequest.numCtx
+    }
+
+    const stream = (await this.openai.chat.completions.create(params)) as unknown as Stream<ChatCompletionChunk>
+
+    // Returns how many trailing chars of `text` could be the start of `tag`
+    function partialTagSuffix(tag: string, text: string): number {
+      for (let len = Math.min(tag.length - 1, text.length); len >= 1; len--) {
+        if (text.endsWith(tag.slice(0, len))) return len
+      }
+      return 0
+    }
+
+    async function* normalize(): AsyncGenerator<NomadChatStreamChunk> {
+      // Stateful parser for <think>...</think> tags that may be split across chunks.
+      // Ollama provides thinking natively via delta.thinking; OpenAI-compatible backends
+      // (LM Studio, llama.cpp, etc.) embed them inline in delta.content.
+      let tagBuffer = ''
+      let inThink = false
+
+      for await (const chunk of stream) {
+        const delta = chunk.choices[0]?.delta
+        const nativeThinking: string = (delta as any)?.thinking ?? ''
+        const rawContent: string = delta?.content ?? ''
+
+        // Parse <think> tags out of the content stream
+        tagBuffer += rawContent
+        let parsedContent = ''
+        let parsedThinking = ''
+
+        while (tagBuffer.length > 0) {
+          if (inThink) {
+            const closeIdx = tagBuffer.indexOf('</think>')
+            if (closeIdx !== -1) {
+              parsedThinking += tagBuffer.slice(0, closeIdx)
+              tagBuffer = tagBuffer.slice(closeIdx + 8)
+              inThink = false
+            } else {
+              const hold = partialTagSuffix('</think>', tagBuffer)
+              parsedThinking += tagBuffer.slice(0, tagBuffer.length - hold)
+              tagBuffer = tagBuffer.slice(tagBuffer.length - hold)
+              break
+            }
+          } else {
+            const openIdx = tagBuffer.indexOf('<think>')
+            if (openIdx !== -1) {
+              parsedContent += tagBuffer.slice(0, openIdx)
+              tagBuffer = tagBuffer.slice(openIdx + 7)
+              inThink = true
+            } else {
+              const hold = partialTagSuffix('<think>', tagBuffer)
+              parsedContent += tagBuffer.slice(0, tagBuffer.length - hold)
+              tagBuffer = tagBuffer.slice(tagBuffer.length - hold)
+              break
+            }
+          }
+        }
+
+        yield {
+          message: {
+            content: parsedContent,
+            thinking: nativeThinking + parsedThinking,
+          },
+          done: chunk.choices[0]?.finish_reason !== null && chunk.choices[0]?.finish_reason !== undefined,
+        }
+      }
+    }
+
+    return normalize()
+  }
+
+  public async checkModelHasThinking(modelName: string): Promise<boolean> {
+    await this._ensureDependencies()
+    if (!this.baseUrl) return false
+
+    try {
+      const response = await axios.post(
+        `${this.baseUrl}/api/show`,
+        { model: modelName },
+        { timeout: 5000 }
+      )
+      return Array.isArray(response.data?.capabilities) && response.data.capabilities.includes('thinking')
+    } catch {
+      // Non-Ollama backends don't expose /api/show — assume no thinking support
+      return false
+    }
+  }
+
+  public async deleteModel(modelName: string): Promise<{ success: boolean; message: string }> {
+    await this._ensureDependencies()
+    if (!this.baseUrl) {
+      return { success: false, message: 'AI service is not initialized.' }
+    }
+
+    try {
+      await axios.delete(`${this.baseUrl}/api/delete`, {
+        data: { model: modelName },
+        timeout: 10000,
+      })
+      return { success: true, message: `Model "${modelName}" deleted.` }
+    } catch (error) {
+      logger.error(
+        `[OllamaService] Failed to delete model "${modelName}": ${error instanceof Error ? error.message : error}`
+      )
+      return { success: false, message: 'Failed to delete model. This may not be an Ollama backend.' }
+    }
+  }
+
+  /**
+   * Hard char cap per embed input, applied as a runtime safety net regardless of
+   * which backend path runs. The chunker in RagService caps at MAX_SAFE_TOKENS=1600
+   * (3200 chars at the conservative 2 chars/token estimate), but dense technical
+   * content has been observed to slip past on multi-batch ZIM ingestion (#881).
+   *
+   * 4000 chars ≈ 1000–2000 tokens depending on density, which keeps us comfortably
+   * under nomic-embed-text:v1.5's default 2048-token context even on the OpenAI-compat
+   * fallback path (which can't pass `truncate:true`/`num_ctx` to the model).
+   */
+  public static readonly EMBED_MAX_INPUT_CHARS = 4000
+
+  /**
+   * Aggressive 2048-safe character cap, applied only on a context-length retry. nomic-embed-text:v1.5
+   * defaults to a 2048-token context, and on the OpenAI-compat fallback path (or an older Ollama that
+   * ignores num_ctx for embeddings) we cannot widen it at request time. 2000 chars stays under 2048
+   * tokens even for the densest content (~1 char/token code/markup), so an oversized chunk gets
+   * truncated-and-kept instead of silently dropped from Qdrant — and the embed job stops re-embedding
+   * the whole file 30x on the one bad chunk (#881).
+   */
+  public static readonly EMBED_CONTEXT_SAFE_CHARS = 2000
+
+  /**
+   * True if the error is the model rejecting input that exceeds its context window
+   * ("input length exceeds the context length"). Matches both the native /api/embed axios error
+   * shape and the OpenAI-compat BadRequestError. Drives the truncate-and-retry here and the
+   * non-retryable classification in EmbedFileJob (#881).
+   */
+  public static isContextLengthError(err: unknown): boolean {
+    const parts: string[] = []
+    if (err instanceof Error && err.message) parts.push(err.message)
+    const anyErr = err as any
+    const data = anyErr?.response?.data
+    if (data) parts.push(typeof data === 'string' ? data : JSON.stringify(data))
+    if (anyErr?.error) parts.push(typeof anyErr.error === 'string' ? anyErr.error : JSON.stringify(anyErr.error))
+    const haystack = parts.join(' ').toLowerCase()
+    return (
+      (haystack.includes('context length') && haystack.includes('exceed')) ||
+      haystack.includes('input length exceeds')
+    )
+  }
+
+  /**
+   * Generate embeddings for the given input strings.
+   * Tries the Ollama native /api/embed endpoint first, falls back to /v1/embeddings.
+   *
+   * If the first attempt fails because a chunk exceeds the model's context window, retries once
+   * with an aggressive 2048-safe truncation (EMBED_CONTEXT_SAFE_CHARS) so the chunk is embedded
+   * (start-of-chunk) rather than silently dropped from Qdrant (#881).
+   */
+  public async embed(model: string, input: string[]): Promise<{ embeddings: number[][] }> {
+    await this._ensureDependencies()
+    if (!this.baseUrl || !this.openai) {
+      throw new Error('AI service is not initialized.')
+    }
+
+    const cap = (arr: string[], max: number) => arr.map((s) => (s.length > max ? s.slice(0, max) : s))
+
+    // Generous pre-cap (#881): fine for the native path (num_ctx=8192) but can still exceed a
+    // 2048-context fallback on dense content. The context-length retry below is the hard backstop.
+    const safeInput = cap(input, OllamaService.EMBED_MAX_INPUT_CHARS)
+
+    try {
+      return await this._embedWithFallback(model, safeInput)
+    } catch (err) {
+      if (!OllamaService.isContextLengthError(err)) throw err
+      // One or more chunks exceeded the model's context even after the pre-cap — typically an
+      // older Ollama that ignores num_ctx for embeddings, or the OpenAI-compat fallback path.
+      // Retry once, truncated hard enough to fit a 2048-token context at any density, so the
+      // chunk is embedded (truncated) instead of dropped and the job doesn't storm.
+      const hardCapped = cap(input, OllamaService.EMBED_CONTEXT_SAFE_CHARS)
+      const reduced = hardCapped.reduce((n, s, i) => (s.length < safeInput[i].length ? n + 1 : n), 0)
+      logger.warn(
+        '[OllamaService] embed: context-length overflow; retrying %d/%d inputs hard-capped at %d chars',
+        reduced,
+        input.length,
+        OllamaService.EMBED_CONTEXT_SAFE_CHARS
+      )
+      return await this._embedWithFallback(model, hardCapped)
+    }
+  }
+
+  /**
+   * Single embed attempt: native /api/embed first, then the OpenAI-compat /v1/embeddings fallback.
+   * Both paths request num_ctx/truncate (Ollama's OpenAI-compat shim forwards them). A context-length
+   * error from the native path is re-thrown rather than falling back, because the fallback has a
+   * smaller effective context and would only fail the same way — the caller (embed) retries it
+   * truncated instead.
+   */
+  private async _embedWithFallback(model: string, input: string[]): Promise<{ embeddings: number[][] }> {
+    try {
+      // Pass num_ctx explicitly so we don't depend on the embedding model's modelfile defaults.
+      // Some installs ship nomic-embed-text:v1.5 with num_ctx=2048; 8192 matches its RoPE-extrapolated
+      // max. truncate:true is a server-side net for any chunk that still overshoots.
+      const response = await axios.post(
+        `${this.baseUrl}/api/embed`,
+        {
+          model,
+          input,
+          truncate: true,
+          options: { num_ctx: 8192 },
+        },
+        { timeout: 60000 }
+      )
+      // Some backends (e.g. LM Studio) return HTTP 200 for unknown endpoints with an incompatible
+      // body — validate explicitly before accepting the result.
+      if (!Array.isArray(response.data?.embeddings)) {
+        throw new Error('Invalid /api/embed response — missing embeddings array')
+      }
+      return { embeddings: response.data.embeddings }
+    } catch (err) {
+      // Let context-length errors bubble so embed() can retry with a smaller cap; the fallback
+      // endpoint (smaller effective context, no num_ctx honored on older Ollama) can't help here.
+      if (OllamaService.isContextLengthError(err)) throw err
+      // Log the original error so we know *why* we fell back. Earlier bare catches here masked
+      // recurring failures for months (#369, #670, #881).
+      logger.warn(
+        '[OllamaService] /api/embed failed, falling back to /v1/embeddings: %s',
+        err instanceof Error ? err.message : String(err)
+      )
+      // Fall back to OpenAI-compatible /v1/embeddings. Explicitly request float format — some
+      // backends (e.g. LM Studio) don't reliably implement the base64 the OpenAI SDK defaults to.
+      // truncate/num_ctx are forwarded by Ollama's OpenAI-compat shim; the SDK types omit them,
+      // hence the cast. We only ever talk to a local Ollama here, not real OpenAI.
+      const results = await this.openai!.embeddings.create({
+        model,
+        input,
+        encoding_format: 'float',
+        truncate: true,
+        options: { num_ctx: 8192 },
+      } as any)
+      return { embeddings: results.data.map((e) => e.embedding as number[]) }
+    }
+  }
+
+  /**
+   * Returns true if Ollama is currently running an embedding model with non-zero VRAM
+   * (i.e., GPU-offloaded). Returns false if the model is running CPU-only OR if it's
+   * not currently loaded OR if /api/ps is unreachable.
+   *
+   * Used by EmbedFileJob to pace continuation batches when the embedding model is
+   * CPU-bound — sustained 100% CPU on a multi-batch ZIM ingestion can starve other
+   * services (sshd, etc.) hard enough to require a power-cycle. AMD ROCm installs
+   * hit this today because Ollama's ROCm build doesn't accelerate nomic-bert; on
+   * NVIDIA, nomic-embed-text runs at 100% GPU and pacing is unnecessary.
+   *
+   * Only the Ollama-native endpoint is supported — backends that expose
+   * `/v1/embeddings` (LM Studio, llama.cpp) don't surface placement info.
+   */
+  public async isEmbeddingGpuAccelerated(): Promise<boolean> {
+    await this._ensureDependencies()
+    if (!this.baseUrl) return false
+
+    try {
+      const response = await axios.get(`${this.baseUrl}/api/ps`, { timeout: 5000 })
+      const models: Array<{ name?: string; size_vram?: number }> = response.data?.models ?? []
+      // Match any loaded model whose name signals it's an embedding model.
+      // nomic-embed-text, mxbai-embed-large, snowflake-arctic-embed, etc. all follow this convention.
+      return models.some(
+        (m) => m.name?.toLowerCase().includes('embed') && (m.size_vram ?? 0) > 0
+      )
+    } catch (err: any) {
+      // /api/ps unreachable (Ollama down, non-native backend, etc.) — fail closed: assume CPU,
+      // which means we'll pace. Better to over-pace than risk box-killing CPU saturation.
+      logger.warn(
+        `[OllamaService] Could not check embedding placement via /api/ps: ${err?.message ?? err}`
+      )
+      return false
+    }
+  }
+
+  /**
+   * Enforces the "at most one chat model resident in VRAM" invariant by firing
+   * `keep_alive: 0` against every currently-loaded model except (a) the
+   * embedding model (always exempt) and (b) `targetModel` (the one we want
+   * loaded next — leaving it alone preserves a hot model when the target is
+   * already loaded).
+   *
+   * Best-effort: queries `/api/ps` and POSTs unload hints in parallel. Network
+   * or Ollama errors are swallowed and logged — neither chat nor page-load
+   * should fail just because the unload housekeeping didn't go through.
+   *
+   * Returns the list of model names that were sent the unload hint, so the
+   * caller (and tests) can confirm what actually happened.
+   *
+   * Pass `targetModel: null` to unload every chat model (used for the future
+   * "free up VRAM" path; not exposed yet but the helper supports it).
+   *
+   * Note that `keep_alive: 0` is a post-completion hint, not a force-kill —
+   * Ollama defers eviction until the runner is idle, so in-flight inference
+   * on the same model is never interrupted. See the design doc for the race
+   * analysis behind this.
+   */
+  public async unloadAllChatModelsExcept(targetModel: string | null): Promise<string[]> {
+    await this._ensureDependencies()
+    if (!this.baseUrl) return []
+
+    let loadedModels: string[] = []
+    try {
+      const response = await axios.get(`${this.baseUrl}/api/ps`, { timeout: 5000 })
+      loadedModels = (response.data?.models ?? [])
+        .map((m: { name?: string }) => m.name)
+        .filter((name: unknown): name is string => typeof name === 'string')
+    } catch (err: any) {
+      logger.warn(
+        `[OllamaService] unloadAllChatModelsExcept: /api/ps unreachable, skipping unload sweep: ${err?.message ?? err}`
+      )
+      return []
+    }
+
+    const toUnload = loadedModels.filter(
+      (name) => name !== EMBEDDING_MODEL_NAME && name !== targetModel
+    )
+
+    await Promise.all(
+      toUnload.map(async (modelName) => {
+        try {
+          await axios.post(
+            `${this.baseUrl}/api/generate`,
+            { model: modelName, prompt: '', keep_alive: 0 },
+            { timeout: 10000 }
+          )
+        } catch (err: any) {
+          logger.warn(
+            `[OllamaService] Failed to send unload hint for ${modelName}: ${err?.message ?? err}`
+          )
+        }
+      })
+    )
+
+    if (toUnload.length > 0) {
+      logger.info(
+        `[OllamaService] Sent unload hint for ${toUnload.length} chat model(s): ${toUnload.join(', ')}`
+      )
+    }
+    return toUnload
+  }
+
+  public async getModels(includeEmbeddings = false): Promise<NomadInstalledModel[]> {
+    await this._ensureDependencies()
+    if (!this.baseUrl) {
+      throw new Error('AI service is not initialized.')
+    }
+
+    try {
+      // Prefer the Ollama native endpoint which includes size and metadata
+      const response = await axios.get(`${this.baseUrl}/api/tags`, { timeout: 5000 })
+      // LM Studio returns HTTP 200 for unknown endpoints with an incompatible body — validate explicitly
+      if (!Array.isArray(response.data?.models)) {
+        throw new Error('Not an Ollama-compatible /api/tags response')
+      }
+      this.isOllamaNative = true
+      const models: NomadInstalledModel[] = response.data.models
+      if (includeEmbeddings) return models
+      return models.filter((m) => !m.name.includes('embed'))
+    } catch {
+      // Fall back to the OpenAI-compatible /v1/models endpoint (LM Studio, llama.cpp, etc.)
+      this.isOllamaNative = false
+      logger.info('[OllamaService] /api/tags unavailable, falling back to /v1/models')
+      try {
+        const modelList = await this.openai!.models.list()
+        const models: NomadInstalledModel[] = modelList.data.map((m) => ({ name: m.id, size: 0 }))
+        if (includeEmbeddings) return models
+        return models.filter((m) => !m.name.includes('embed'))
+      } catch (err) {
+        logger.error(
+          `[OllamaService] Failed to list models: ${err instanceof Error ? err.message : err}`
+        )
+        return []
+      }
+    }
+  }
+
+  async getAvailableModels(
+    {
+      sort,
+      recommendedOnly,
+      query,
+      limit,
+      force,
+    }: {
+      sort?: 'pulls' | 'name'
+      recommendedOnly?: boolean
+      query: string | null
+      limit?: number
+      force?: boolean
+    } = {
+      sort: 'pulls',
+      recommendedOnly: false,
+      query: null,
+      limit: 15,
+    }
+  ): Promise<{ models: NomadOllamaModel[]; hasMore: boolean } | null> {
+    try {
+      const models = await this.retrieveAndRefreshModels(sort, force)
+      if (!models) {
+        logger.warn(
+          '[OllamaService] Returning fallback recommended models due to failure in fetching available models'
+        )
+        return {
+          models: FALLBACK_RECOMMENDED_OLLAMA_MODELS,
+          hasMore: false,
+        }
+      }
+
+      if (!recommendedOnly) {
+        const filteredModels = query ? this.fuseSearchModels(models, query) : models
+        return {
+          models: filteredModels.slice(0, limit || 15),
+          hasMore: filteredModels.length > (limit || 15),
+        }
+      }
+
+      const sortedByPulls = sort === 'pulls' ? models : this.sortModels(models, 'pulls')
+      const firstThree = sortedByPulls.slice(0, 3)
+
+      const recommendedModels = firstThree.map((model) => {
+        return {
+          ...model,
+          tags: model.tags && model.tags.length > 0 ? [model.tags[0]] : [],
+        }
+      })
+
+      if (query) {
+        const filteredRecommendedModels = this.fuseSearchModels(recommendedModels, query)
+        return {
+          models: filteredRecommendedModels,
+          hasMore: filteredRecommendedModels.length > (limit || 15),
+        }
+      }
+
+      return {
+        models: recommendedModels,
+        hasMore: recommendedModels.length > (limit || 15),
+      }
+    } catch (error) {
+      logger.error(
+        `[OllamaService] Failed to get available models: ${error instanceof Error ? error.message : error}`
+      )
+      return null
+    }
+  }
+
+  private async retrieveAndRefreshModels(
+    sort?: 'pulls' | 'name',
+    force?: boolean
+  ): Promise<NomadOllamaModel[] | null> {
+    try {
+      if (!force) {
+        const cachedModels = await this.readModelsFromCache()
+        if (cachedModels) {
+          logger.info('[OllamaService] Using cached available models data')
+          return this.sortModels(cachedModels, sort)
+        }
+      } else {
+        logger.info('[OllamaService] Force refresh requested, bypassing cache')
+      }
+
+      logger.info('[OllamaService] Fetching fresh available models from API')
+
+      const baseUrl = env.get('NOMAD_API_URL') || NOMAD_API_DEFAULT_BASE_URL
+      const fullUrl = new URL(NOMAD_MODELS_API_PATH, baseUrl).toString()
+
+      const response = await axios.get(fullUrl)
+      if (!response.data || !Array.isArray(response.data.models)) {
+        logger.warn(
+          `[OllamaService] Invalid response format when fetching available models: ${JSON.stringify(response.data)}`
+        )
+        return null
+      }
+
+      const rawModels = response.data.models as NomadOllamaModel[]
+
+      const noCloud = rawModels
+        .map((model) => ({
+          ...model,
+          tags: model.tags.filter((tag) => !tag.cloud),
+        }))
+        .filter((model) => model.tags.length > 0)
+
+      await this.writeModelsToCache(noCloud)
+      return this.sortModels(noCloud, sort)
+    } catch (error) {
+      logger.error(
+        `[OllamaService] Failed to retrieve models from Nomad API: ${error instanceof Error ? error.message : error}`
+      )
+      return null
+    }
+  }
+
+  private async readModelsFromCache(): Promise<NomadOllamaModel[] | null> {
+    try {
+      const stats = await fs.stat(MODELS_CACHE_FILE)
+      const cacheAge = Date.now() - stats.mtimeMs
+
+      if (cacheAge > CACHE_MAX_AGE_MS) {
+        logger.info('[OllamaService] Cache is stale, will fetch fresh data')
+        return null
+      }
+
+      const cacheData = await fs.readFile(MODELS_CACHE_FILE, 'utf-8')
+      const models = JSON.parse(cacheData) as NomadOllamaModel[]
+
+      if (!Array.isArray(models)) {
+        logger.warn('[OllamaService] Invalid cache format, will fetch fresh data')
+        return null
+      }
+
+      return models
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
+        logger.warn(
+          `[OllamaService] Error reading cache: ${error instanceof Error ? error.message : error}`
+        )
+      }
+      return null
+    }
+  }
+
+  private async writeModelsToCache(models: NomadOllamaModel[]): Promise<void> {
+    try {
+      await fs.mkdir(path.dirname(MODELS_CACHE_FILE), { recursive: true })
+      await fs.writeFile(MODELS_CACHE_FILE, JSON.stringify(models, null, 2), 'utf-8')
+      logger.info('[OllamaService] Successfully cached available models')
+    } catch (error) {
+      logger.warn(
+        `[OllamaService] Failed to write models cache: ${error instanceof Error ? error.message : error}`
+      )
+    }
+  }
+
+  private sortModels(models: NomadOllamaModel[], sort?: 'pulls' | 'name'): NomadOllamaModel[] {
+    if (sort === 'pulls') {
+      models.sort((a, b) => {
+        const parsePulls = (pulls: string) => {
+          const multiplier = pulls.endsWith('K')
+            ? 1_000
+            : pulls.endsWith('M')
+              ? 1_000_000
+              : pulls.endsWith('B')
+                ? 1_000_000_000
+                : 1
+          return parseFloat(pulls) * multiplier
+        }
+        return parsePulls(b.estimated_pulls) - parsePulls(a.estimated_pulls)
+      })
+    } else if (sort === 'name') {
+      models.sort((a, b) => a.name.localeCompare(b.name))
+    }
+
+    models.forEach((model) => {
+      if (model.tags && Array.isArray(model.tags)) {
+        model.tags.sort((a, b) => {
+          const parseSize = (size: string) => {
+            const multiplier = size.endsWith('KB')
+              ? 1 / 1_000
+              : size.endsWith('MB')
+                ? 1 / 1_000_000
+                : size.endsWith('GB')
+                  ? 1
+                  : size.endsWith('TB')
+                    ? 1_000
+                    : 0
+            return parseFloat(size) * multiplier
+          }
+          return parseSize(a.size) - parseSize(b.size)
+        })
+      }
+    })
+
+    return models
+  }
+
+  private broadcastDownloadError(model: string, error: string) {
+    transmit.broadcast(BROADCAST_CHANNELS.OLLAMA_MODEL_DOWNLOAD, {
+      model,
+      percent: -1,
+      error,
+      timestamp: new Date().toISOString(),
+    })
+  }
+
+  private broadcastDownloadProgress(
+    model: string,
+    percent: number,
+    jobId?: string,
+    bytes?: { downloadedBytes: number; totalBytes: number }
+  ) {
+    // Conditional spread on jobId/bytes — Transmit's Broadcastable type rejects fields whose
+    // value is `undefined`, so we omit each key entirely when its value isn't available.
+    transmit.broadcast(BROADCAST_CHANNELS.OLLAMA_MODEL_DOWNLOAD, {
+      model,
+      percent,
+      ...(jobId ? { jobId } : {}),
+      ...(bytes ? { downloadedBytes: bytes.downloadedBytes, totalBytes: bytes.totalBytes } : {}),
+      timestamp: new Date().toISOString(),
+    })
+    logger.info(`[OllamaService] Download progress for model "${model}": ${percent}%`)
+  }
+
+  private fuseSearchModels(models: NomadOllamaModel[], query: string): NomadOllamaModel[] {
+    const options: IFuseOptions<NomadOllamaModel> = {
+      ignoreDiacritics: true,
+      keys: ['name', 'description', 'tags.name'],
+      threshold: 0.3,
+    }
+
+    const fuse = new Fuse(models, options)
+
+    return fuse.search(query).map((result) => result.item)
+  }
+}

admin/app/services/openwebui_service.ts

@@ -1,646 +0,0 @@
-import { inject } from '@adonisjs/core'
-import logger from '@adonisjs/core/services/logger'
-import { DockerService } from './docker_service.js'
-import axios from 'axios'
-import { NomadOllamaModel, OllamaModelListing } from '../../types/ollama.js'
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import { PassThrough } from 'node:stream'
-import { DownloadModelJob } from '#jobs/download_model_job'
-
-const NOMAD_MODELS_API_BASE_URL = 'https://api.projectnomad.us/api/v1/ollama/models'
-const MODELS_CACHE_FILE = path.join(process.cwd(), 'storage', 'ollama-models-cache.json')
-const CACHE_MAX_AGE_MS = 24 * 60 * 60 * 1000 // 24 hours
-
-@inject()
-export class OpenWebUIService {
-  constructor(private dockerService: DockerService) {}
-
-  /** We need to call this in the DownloadModelJob, so it can't be private,
-   * but shouldn't be called directly (dispatch job instead)
-   */
-  async _downloadModel(
-    model: string,
-    onProgress?: (progress: {
-      status: string
-      completed?: number
-      total?: number
-      percent?: number
-    }) => void
-  ): Promise<{ success: boolean; message: string }> {
-    return new Promise((resolve) => {
-      try {
-        const container = this.dockerService.docker.getContainer(DockerService.OLLAMA_SERVICE_NAME)
-        if (!container) {
-          logger.warn('[OpenWebUIService] Ollama container is not running. Cannot download model.')
-          resolve({
-            success: false,
-            message: 'Ollama is not running. Please start Ollama and try again.',
-          })
-          return
-        }
-
-        container.exec(
-          {
-            Cmd: ['ollama', 'pull', model],
-            AttachStdout: true,
-            AttachStderr: true,
-          },
-          (err, exec) => {
-            if (err) {
-              logger.error(
-                `[OpenWebUIService] Failed to execute model download command: ${
-                  err instanceof Error ? err.message : err
-                }`
-              )
-              resolve({ success: false, message: 'Failed to execute download command.' })
-              return
-            }
-
-            if (!exec) {
-              logger.error('[OpenWebUIService] No exec instance returned from exec command')
-              resolve({ success: false, message: 'Failed to create exec instance.' })
-              return
-            }
-
-            exec.start(
-              {
-                hijack: true,
-                stdin: false,
-              },
-              (startErr, stream) => {
-                if (startErr) {
-                  logger.error(
-                    `[OpenWebUIService] Failed to start exec stream: ${
-                      startErr instanceof Error ? startErr.message : startErr
-                    }`
-                  )
-                  resolve({ success: false, message: 'Failed to start download stream.' })
-                  return
-                }
-
-                if (!stream) {
-                  logger.error('[OpenWebUIService] No stream returned when starting exec')
-                  resolve({ success: false, message: 'No stream available.' })
-                  return
-                }
-
-                // Create PassThrough streams to capture output
-                const stdout = new PassThrough()
-                const stderr = new PassThrough()
-
-                // Demultiplex the Docker stream
-                this.dockerService.docker.modem.demuxStream(stream, stdout, stderr)
-
-                // Capture and parse stdout (if any)
-                stdout.on('data', (chunk) => {
-                  const output = chunk.toString()
-                  logger.info(`[OpenWebUIService] Model download (stdout): ${output}`)
-                })
-
-                // Capture stderr - ollama sends progress/status here (not necessarily errors)
-                stderr.on('data', (chunk) => {
-                  const output = chunk.toString()
-
-                  // Check if this is an actual error message
-                  if (
-                    output.toLowerCase().includes('error') ||
-                    output.toLowerCase().includes('failed')
-                  ) {
-                    logger.error(`[OpenWebUIService] Model download error: ${output}`)
-                  } else {
-                    // This is normal progress/status output from ollama
-                    logger.info(`[OpenWebUIService] Model download progress: ${output}`)
-
-                    // Parse JSON progress if available
-                    try {
-                      const lines = output
-                        .split('\n')
-                        .filter(
-                          (line: any) => typeof line.trim() === 'string' && line.trim().length > 0
-                        )
-                      for (const line of lines) {
-                        const parsed = JSON.parse(line)
-                        if (parsed.status) {
-                          const progressData: {
-                            status: string
-                            completed?: number
-                            total?: number
-                            percent?: number
-                          } = {
-                            status: parsed.status,
-                          }
-
-                          // Extract byte progress if available
-                          if (parsed.completed !== undefined && parsed.total !== undefined) {
-                            progressData.completed = parsed.completed
-                            progressData.total = parsed.total
-                            progressData.percent = Math.round(
-                              (parsed.completed / parsed.total) * 100
-                            )
-                          }
-
-                          // Call progress callback
-                          if (onProgress) {
-                            onProgress(progressData)
-                          }
-
-                          // Log structured progress
-                          if (progressData.percent !== undefined) {
-                            logger.info(
-                              `[OpenWebUIService] ${progressData.status}: ${progressData.percent}% (${progressData.completed}/${progressData.total} bytes)`
-                            )
-                          } else {
-                            logger.info(`[OpenWebUIService] ${progressData.status}`)
-                          }
-                        }
-                      }
-                    } catch {
-                      // Not JSON, already logged above
-                    }
-                  }
-                })
-
-                // Handle stream end
-                stream.on('end', () => {
-                  logger.info(
-                    `[OpenWebUIService] Model download process ended for model "${model}"`
-                  )
-                  resolve({
-                    success: true,
-                    message: 'Model download completed successfully.',
-                  })
-                })
-
-                // Handle stream errors
-                stream.on('error', (streamErr) => {
-                  logger.error(
-                    `[OpenWebUIService] Error during model download stream: ${
-                      streamErr instanceof Error ? streamErr.message : streamErr
-                    }`
-                  )
-                  resolve({
-                    success: false,
-                    message: 'Error occurred during model download.',
-                  })
-                })
-              }
-            )
-          }
-        )
-      } catch (error) {
-        logger.error(
-          `[OpenWebUIService] Failed to download model "${model}": ${
-            error instanceof Error ? error.message : error
-          }`
-        )
-        resolve({ success: false, message: 'Failed to download model.' })
-      }
-    })
-  }
-
-  async deleteModel(model: string): Promise<{ success: boolean; message: string }> {
-    return new Promise((resolve) => {
-      try {
-        const container = this.dockerService.docker.getContainer(DockerService.OLLAMA_SERVICE_NAME)
-        if (!container) {
-          logger.warn('[OpenWebUIService] Ollama container is not running. Cannot remove model.')
-          resolve({
-            success: false,
-            message: 'Ollama is not running. Please start Ollama and try again.',
-          })
-          return
-        }
-
-        container.exec(
-          {
-            Cmd: ['ollama', 'rm', model],
-            AttachStdout: true,
-            AttachStderr: true,
-          },
-          (err, exec) => {
-            if (err) {
-              logger.error(
-                `[OpenWebUIService] Failed to execute model remove command: ${
-                  err instanceof Error ? err.message : err
-                }`
-              )
-              resolve({ success: false, message: 'Failed to execute remove command.' })
-              return
-            }
-
-            if (!exec) {
-              logger.error('[OpenWebUIService] No exec instance returned from remove command')
-              resolve({ success: false, message: 'Failed to create exec instance.' })
-              return
-            }
-
-            exec.start(
-              {
-                hijack: true,
-                stdin: false,
-              },
-              (startErr, stream) => {
-                if (startErr) {
-                  logger.error(
-                    `[OpenWebUIService] Failed to start exec stream for remove: ${
-                      startErr instanceof Error ? startErr.message : startErr
-                    }`
-                  )
-                  resolve({ success: false, message: 'Failed to start remove command.' })
-                  return
-                }
-
-                if (!stream) {
-                  logger.error('[OpenWebUIService] No stream returned for remove command')
-                  resolve({ success: false, message: 'No stream available.' })
-                  return
-                }
-
-                const stdout = new PassThrough()
-                const stderr = new PassThrough()
-                let output = ''
-                let errorOutput = ''
-
-                this.dockerService.docker.modem.demuxStream(stream, stdout, stderr)
-
-                stdout.on('data', (chunk) => {
-                  output += chunk.toString()
-                })
-
-                stderr.on('data', (chunk) => {
-                  errorOutput += chunk.toString()
-                })
-
-                stream.on('end', () => {
-                  if (errorOutput) {
-                    logger.error(`[OpenWebUIService] Error removing model: ${errorOutput}`)
-                    resolve({
-                      success: false,
-                      message: errorOutput.trim() || 'Failed to remove model.',
-                    })
-                    return
-                  }
-
-                  logger.info(`[OpenWebUIService] Successfully removed model "${model}"`)
-                  if (output) {
-                    logger.info(`[OpenWebUIService] Remove output: ${output}`)
-                  }
-
-                  resolve({
-                    success: true,
-                    message: 'Model removed successfully.',
-                  })
-                })
-
-                stream.on('error', (streamErr) => {
-                  logger.error(
-                    `[OpenWebUIService] Stream error during model remove: ${
-                      streamErr instanceof Error ? streamErr.message : streamErr
-                    }`
-                  )
-                  resolve({
-                    success: false,
-                    message: 'Error occurred while removing model.',
-                  })
-                })
-              }
-            )
-          }
-        )
-      } catch (error) {
-        logger.error(
-          `[OpenWebUIService] Failed to remove model "${model}": ${
-            error instanceof Error ? error.message : error
-          }`
-        )
-        resolve({ success: false, message: 'Failed to remove model.' })
-      }
-    })
-  }
-
-  async dispatchModelDownload(modelName: string): Promise<{ success: boolean; message: string }> {
-    try {
-      logger.info(`[OpenWebUIService] Dispatching model download for ${modelName} via job queue`)
-
-      await DownloadModelJob.dispatch({
-        modelName,
-      })
-
-      return {
-        success: true,
-        message:
-          'Model download has been queued successfully. It will start shortly after Ollama and Open WebUI are ready (if not already).',
-      }
-    } catch (error) {
-      logger.error(
-        `[OpenWebUIService] Failed to dispatch model download for ${modelName}: ${error instanceof Error ? error.message : error}`
-      )
-      return {
-        success: false,
-        message: 'Failed to queue model download. Please try again.',
-      }
-    }
-  }
-
-  async getAvailableModels(
-    { sort, recommendedOnly }: { sort?: 'pulls' | 'name'; recommendedOnly?: boolean } = {
-      sort: 'pulls',
-      recommendedOnly: false,
-    }
-  ): Promise<NomadOllamaModel[] | null> {
-    try {
-      const models = await this.retrieveAndRefreshModels(sort)
-      if (!models) {
-        return null
-      }
-
-      if (!recommendedOnly) {
-        return models
-      }
-
-      // If recommendedOnly is true, only return the first three models (if sorted by pulls, these will be the top 3)
-      const sortedByPulls = sort === 'pulls' ? models : this.sortModels(models, 'pulls')
-      const firstThree = sortedByPulls.slice(0, 3)
-
-      // Only return the first tag of each of these models (should be the most lightweight variant)
-      const recommendedModels = firstThree.map((model) => {
-        return {
-          ...model,
-          tags: model.tags && model.tags.length > 0 ? [model.tags[0]] : [],
-        }
-      })
-      return recommendedModels
-    } catch (error) {
-      logger.error(
-        `[OpenWebUIService] Failed to get available models: ${error instanceof Error ? error.message : error}`
-      )
-      return null
-    }
-  }
-
-  async getInstalledModels(): Promise<OllamaModelListing[] | null> {
-    return new Promise((resolve) => {
-      try {
-        const container = this.dockerService.docker.getContainer(DockerService.OLLAMA_SERVICE_NAME)
-        if (!container) {
-          logger.warn('[OpenWebUIService] Ollama container is not running. Cannot list models.')
-          resolve(null)
-          return
-        }
-
-        container.exec(
-          {
-            Cmd: ['ollama', 'list'],
-            AttachStdout: true,
-            AttachStderr: true,
-          },
-          (err, exec) => {
-            if (err) {
-              logger.error(
-                `[OpenWebUIService] Failed to execute ollama list command: ${
-                  err instanceof Error ? err.message : err
-                }`
-              )
-              resolve(null)
-              return
-            }
-
-            if (!exec) {
-              logger.error('[OpenWebUIService] No exec instance returned from ollama list')
-              resolve(null)
-              return
-            }
-
-            exec.start(
-              {
-                hijack: true,
-                stdin: false,
-              },
-              (startErr, stream) => {
-                if (startErr) {
-                  logger.error(
-                    `[OpenWebUIService] Failed to start exec stream for ollama list: ${
-                      startErr instanceof Error ? startErr.message : startErr
-                    }`
-                  )
-                  resolve(null)
-                  return
-                }
-
-                if (!stream) {
-                  logger.error('[OpenWebUIService] No stream returned for ollama list')
-                  resolve(null)
-                  return
-                }
-
-                const stdout = new PassThrough()
-                const stderr = new PassThrough()
-                let output = ''
-                let errorOutput = ''
-
-                this.dockerService.docker.modem.demuxStream(stream, stdout, stderr)
-
-                stdout.on('data', (chunk) => {
-                  output += chunk.toString()
-                })
-
-                stderr.on('data', (chunk) => {
-                  errorOutput += chunk.toString()
-                })
-
-                stream.on('end', () => {
-                  if (errorOutput) {
-                    logger.error(
-                      `[OpenWebUIService] Error from ollama list command: ${errorOutput}`
-                    )
-                  }
-
-                  if (!output) {
-                    logger.info('[OpenWebUIService] No models installed')
-                    resolve([])
-                    return
-                  }
-
-                  try {
-                    // Parse the tabular output from ollama list
-                    // Expected format:
-                    // NAME                    ID              SIZE      MODIFIED
-                    // llama2:latest          abc123def456    3.8 GB    2 days ago
-                    const lines = output.split('\n').filter((line) => line.trim())
-
-                    // Skip header line and parse model entries
-                    const models: OllamaModelListing[] = []
-                    for (let i = 1; i < lines.length; i++) {
-                      const line = lines[i].trim()
-                      if (!line) continue
-
-                      // Split by whitespace (2+ spaces to handle columns with spaces)
-                      const parts = line.split(/\s{2,}/)
-
-                      if (parts.length >= 4) {
-                        models.push({
-                          name: parts[0].trim(),
-                          id: parts[1].trim(),
-                          size: parts[2].trim(),
-                          modified: parts[3].trim(),
-                        })
-                      }
-                    }
-
-                    logger.info(`[OpenWebUIService] Found ${models.length} installed models`)
-                    resolve(models)
-                  } catch (parseError) {
-                    logger.error(
-                      `[OpenWebUIService] Failed to parse ollama list output: ${
-                        parseError instanceof Error ? parseError.message : parseError
-                      }`
-                    )
-                    logger.debug(`[OpenWebUIService] Raw output: ${output}`)
-                    resolve(null)
-                  }
-                })
-
-                stream.on('error', (streamErr) => {
-                  logger.error(
-                    `[OpenWebUIService] Stream error during ollama list: ${
-                      streamErr instanceof Error ? streamErr.message : streamErr
-                    }`
-                  )
-                  resolve(null)
-                })
-              }
-            )
-          }
-        )
-      } catch (error) {
-        logger.error(
-          `[OpenWebUIService] Failed to get installed models: ${
-            error instanceof Error ? error.message : error
-          }`
-        )
-        resolve(null)
-      }
-    })
-  }
-
-  private async retrieveAndRefreshModels(
-    sort?: 'pulls' | 'name'
-  ): Promise<NomadOllamaModel[] | null> {
-    try {
-      const cachedModels = await this.readModelsFromCache()
-      if (cachedModels) {
-        logger.info('[OpenWebUIService] Using cached available models data')
-        return this.sortModels(cachedModels, sort)
-      }
-
-      logger.info('[OpenWebUIService] Fetching fresh available models from API')
-      const response = await axios.get(NOMAD_MODELS_API_BASE_URL)
-      if (!response.data || !Array.isArray(response.data.models)) {
-        logger.warn(
-          `[OpenWebUIService] Invalid response format when fetching available models: ${JSON.stringify(response.data)}`
-        )
-        return null
-      }
-
-      const models = response.data.models as NomadOllamaModel[]
-
-      await this.writeModelsToCache(models)
-      return this.sortModels(models, sort)
-    } catch (error) {
-      logger.error(
-        `[OpenWebUIService] Failed to retrieve models from Nomad API: ${
-          error instanceof Error ? error.message : error
-        }`
-      )
-      return null
-    }
-  }
-
-  private async readModelsFromCache(): Promise<NomadOllamaModel[] | null> {
-    try {
-      const stats = await fs.stat(MODELS_CACHE_FILE)
-      const cacheAge = Date.now() - stats.mtimeMs
-
-      if (cacheAge > CACHE_MAX_AGE_MS) {
-        logger.info('[OpenWebUIService] Cache is stale, will fetch fresh data')
-        return null
-      }
-
-      const cacheData = await fs.readFile(MODELS_CACHE_FILE, 'utf-8')
-      const models = JSON.parse(cacheData) as NomadOllamaModel[]
-
-      if (!Array.isArray(models)) {
-        logger.warn('[OpenWebUIService] Invalid cache format, will fetch fresh data')
-        return null
-      }
-
-      return models
-    } catch (error) {
-      // Cache doesn't exist or is invalid
-      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
-        logger.warn(
-          `[OpenWebUIService] Error reading cache: ${error instanceof Error ? error.message : error}`
-        )
-      }
-      return null
-    }
-  }
-
-  private async writeModelsToCache(models: NomadOllamaModel[]): Promise<void> {
-    try {
-      await fs.mkdir(path.dirname(MODELS_CACHE_FILE), { recursive: true })
-      await fs.writeFile(MODELS_CACHE_FILE, JSON.stringify(models, null, 2), 'utf-8')
-      logger.info('[OpenWebUIService] Successfully cached available models')
-    } catch (error) {
-      logger.warn(
-        `[OpenWebUIService] Failed to write models cache: ${error instanceof Error ? error.message : error}`
-      )
-    }
-  }
-
-  private sortModels(models: NomadOllamaModel[], sort?: 'pulls' | 'name'): NomadOllamaModel[] {
-    if (sort === 'pulls') {
-      // Sort by estimated pulls (it should be a string like "1.2K", "500", "4M" etc.)
-      models.sort((a, b) => {
-        const parsePulls = (pulls: string) => {
-          const multiplier = pulls.endsWith('K')
-            ? 1_000
-            : pulls.endsWith('M')
-              ? 1_000_000
-              : pulls.endsWith('B')
-                ? 1_000_000_000
-                : 1
-          return parseFloat(pulls) * multiplier
-        }
-        return parsePulls(b.estimated_pulls) - parsePulls(a.estimated_pulls)
-      })
-    } else if (sort === 'name') {
-      models.sort((a, b) => a.name.localeCompare(b.name))
-    }
-
-    // Always sort model.tags by the size field in descending order
-    // Size is a string like '75GB', '8.5GB', '2GB' etc. Smaller models first
-    models.forEach((model) => {
-      if (model.tags && Array.isArray(model.tags)) {
-        model.tags.sort((a, b) => {
-          const parseSize = (size: string) => {
-            const multiplier = size.endsWith('KB')
-              ? 1 / 1_000
-              : size.endsWith('MB')
-                ? 1 / 1_000_000
-                : size.endsWith('GB')
-                  ? 1
-                  : size.endsWith('TB')
-                    ? 1_000
-                    : 0 // Unknown size format
-            return parseFloat(size) * multiplier
-          }
-          return parseSize(a.size) - parseSize(b.size)
-        })
-      }
-    })
-
-    return models
-  }
-}

admin/app/services/queue_service.ts

@@ -1,9 +1,25 @@
 import { Queue } from 'bullmq'
 import queueConfig from '#config/queue'
 
+// Process-wide singleton. Instantiating a fresh QueueService per dispatch /
+// status lookup leaks connections, and under sustained job churn (e.g.
+// multi-batch ZIM ingestion enqueueing a continuation every few seconds) it
+// saturates Redis's maxclients within hours. All queues additionally reuse the
+// single shared ioredis instance exported from #config/queue (#885).
 export class QueueService {
   private queues: Map<string, Queue> = new Map()
 
+  private static _instance: QueueService | null = null
+
+  private constructor() {}
+
+  static getInstance(): QueueService {
+    if (!QueueService._instance) {
+      QueueService._instance = new QueueService()
+    }
+    return QueueService._instance
+  }
+
   getQueue(name: string): Queue {
     if (!this.queues.has(name)) {
       const queue = new Queue(name, {
@@ -18,5 +34,6 @@ export class QueueService {
     for (const queue of this.queues.values()) {
       await queue.close()
     }
+    this.queues.clear()
   }
 }

admin/app/services/system_service.ts

@@ -1,15 +1,27 @@
 import Service from '#models/service'
+import InstalledResource from '#models/installed_resource'
 import { inject } from '@adonisjs/core'
 import { DockerService } from '#services/docker_service'
 import { ServiceSlim } from '../../types/services.js'
 import logger from '@adonisjs/core/services/logger'
 import si from 'systeminformation'
-import { NomadDiskInfo, NomadDiskInfoRaw, SystemInformationResponse } from '../../types/system.js'
-import { readFileSync } from 'fs'
-import path, { join } from 'path'
+import {
+  GpuHealthStatus,
+  NomadDiskInfo,
+  NomadDiskInfoRaw,
+  SystemInformationResponse,
+} from '../../types/system.js'
+import { SERVICE_NAMES } from '../../constants/service_names.js'
+import { readFileSync } from 'node:fs'
+import { readFile } from 'node:fs/promises'
+import path, { join } from 'node:path'
 import { getAllFilesystems, getFile } from '../utils/fs.js'
 import axios from 'axios'
 import env from '#start/env'
+import KVStore from '#models/kv_store'
+import { KV_STORE_SCHEMA, KVStoreKey } from '../../types/kv_store.js'
+import { isNewerVersion } from '../utils/version.js'
+import { invalidateAssistantNameCache } from '../../config/inertia.js'
 
 @inject()
 export class SystemService {
@@ -18,30 +30,60 @@ export class SystemService {
 
   constructor(private dockerService: DockerService) {}
 
+  async checkServiceInstalled(serviceName: string): Promise<boolean> {
+    const services = await this.getServices({ installedOnly: true })
+    return services.some((service) => service.service_name === serviceName)
+  }
+
   async getInternetStatus(): Promise<boolean> {
-    const DEFAULT_TEST_URL = 'https://1.1.1.1/cdn-cgi/trace'
+    // Primary endpoint stays Cloudflare's privacy-respecting utility endpoint.
+    // The fallbacks are hosts the application already contacts elsewhere
+    // (GitHub API for update checks, the Project N.O.M.A.D. API for release-note
+    // subscriptions), so no new third-party services are introduced. They exist
+    // to avoid false "offline" reports on networks that block 1.1.1.1.
+    const DEFAULT_TEST_URLS = [
+      'https://1.1.1.1/cdn-cgi/trace',
+      'https://api.github.com',
+      'https://api.projectnomad.us',
+    ]
     const MAX_ATTEMPTS = 3
 
-    let testUrl = DEFAULT_TEST_URL
-    let customTestUrl = env.get('INTERNET_STATUS_TEST_URL')?.trim()
+    let testUrls = DEFAULT_TEST_URLS
 
-    // check that customTestUrl is a valid URL, if provided
+    // Resolve the test endpoint in priority order: the INTERNET_STATUS_TEST_URL
+    // env var always wins (legacy override for operators who intentionally point
+    // connectivity checks at a specific endpoint), then the UI-configurable value
+    // stored in KVStore, and finally the built-in defaults.
+    const envTestUrl = env.get('INTERNET_STATUS_TEST_URL')?.trim()
+    const kvTestUrl = (await KVStore.getValue('system.internetStatusTestUrl'))?.trim()
+    const customTestUrl = envTestUrl || kvTestUrl
+
+    // If a custom test URL is provided and valid, use it exclusively.
     if (customTestUrl && customTestUrl !== '') {
       try {
         new URL(customTestUrl)
-        testUrl = customTestUrl
+        testUrls = [customTestUrl]
       } catch (error) {
         logger.warn(
-          `Invalid INTERNET_STATUS_TEST_URL: ${customTestUrl}. Falling back to default URL.`
+          `Invalid internet status test URL: ${customTestUrl}. Falling back to default URLs.`
         )
       }
     }
 
     for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
       try {
-        const res = await axios.get(testUrl, { timeout: 5000 })
-        return res.status === 200
+        // Probe all test endpoints in parallel and resolve as soon as the first one
+        // responds. Any HTTP response (including non-2xx) means we reached the
+        // internet, so accept all status codes rather than requiring a strict 200.
+        await Promise.any(
+          testUrls.map((testUrl) => {
+            logger.debug(`[SystemService] Checking internet connectivity via: ${testUrl}`)
+            return axios.get(testUrl, { timeout: 5000, validateStatus: () => true })
+          })
+        )
+        return true
       } catch (error) {
+        // Promise.any only rejects (with an AggregateError) when every endpoint failed.
         logger.warn(
           `Internet status check attempt ${attempt}/${MAX_ATTEMPTS} failed: ${error instanceof Error ? error.message : error}`
         )
@@ -57,10 +99,228 @@ export class SystemService {
     return false
   }
 
+  /**
+   * Probe Ollama startup logs for the canonical "inference compute" line that records
+   * which compute backend was selected. This catches silent CPU fallback (e.g. when
+   * /dev/kfd is mounted but ROCm initialization fails, or NVML dies after an update)
+   * which the older nvidia-smi exec probe could not detect.
+   *
+   * Returns the parsed library, GPU model name, and VRAM in MiB, or null when:
+   *   - the Ollama container is not running
+   *   - the line has not been emitted (Ollama still starting up)
+   *   - logs show CPU-only operation (no GPU detected)
+   */
+  async getOllamaInferenceComputeFromLogs(): Promise<{
+    library: 'CUDA' | 'ROCm'
+    name: string
+    vramMiB: number
+  } | null> {
+    try {
+      const containers = await this.dockerService.docker.listContainers({ all: false })
+      const ollamaContainer = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`))
+      if (!ollamaContainer) return null
+
+      const container = this.dockerService.docker.getContainer(ollamaContainer.Id)
+
+      // Read logs only from the first 5 minutes after container start. The
+      // "inference compute" line is written once during Ollama's GPU discovery
+      // phase, within seconds of startup. Using tail:N here is fragile: under
+      // active embedding workloads we've seen >1000 lines/min, which pushes the
+      // line past any reasonable tail in minutes. Pinning to the startup window
+      // is bounded (~5 min of logs regardless of container uptime) and never
+      // ages out.
+      //
+      // Fall back to the previous tail:500 strategy if StartedAt is missing or
+      // unparseable — we can't construct a since/until window without it, but
+      // tail:500 is still useful when the container just started and the line
+      // is still recent.
+      const inspect = await container.inspect()
+      const startedAtRaw = inspect?.State?.StartedAt
+      const startedAtMs = startedAtRaw ? new Date(startedAtRaw).getTime() : NaN
+      const hasValidStartedAt = Number.isFinite(startedAtMs) && startedAtMs > 0
+
+      const logsOpts: { stdout: true; stderr: true; follow: false; since?: number; until?: number; tail?: number } = {
+        stdout: true,
+        stderr: true,
+        follow: false,
+      }
+      if (hasValidStartedAt) {
+        const startedAtSec = Math.floor(startedAtMs / 1000)
+        logsOpts.since = startedAtSec
+        logsOpts.until = startedAtSec + 300 // 5-minute window
+      } else {
+        logger.warn(
+          `[SystemService] nomad_ollama State.StartedAt missing or invalid (${startedAtRaw ?? 'undefined'}); falling back to tail:500 for inference-compute probe`
+        )
+        logsOpts.tail = 500
+      }
+      const buf = (await container.logs(logsOpts)) as unknown as Buffer
+      const logs = buf.toString('utf8')
+
+      const lines = logs.split('\n').filter((l) => l.includes('msg="inference compute"'))
+      if (lines.length === 0) return null
+
+      const lastLine = lines[lines.length - 1]
+      const libraryMatch = lastLine.match(/library=(CUDA|ROCm)/)
+      if (!libraryMatch) return null
+
+      const descMatch = lastLine.match(/description="([^"]+)"/)
+      const totalMatch = lastLine.match(/total="([0-9.]+)\s*GiB"/)
+
+      return {
+        library: libraryMatch[1] as 'CUDA' | 'ROCm',
+        name:
+          descMatch?.[1] ||
+          (libraryMatch[1] === 'CUDA' ? 'NVIDIA GPU' : 'AMD GPU'),
+        vramMiB: totalMatch ? Math.round(Number.parseFloat(totalMatch[1]) * 1024) : 0,
+      }
+    } catch (error) {
+      logger.warn(
+        `[SystemService] Failed to probe Ollama logs for inference compute line: ${error instanceof Error ? error.message : error}`
+      )
+      return null
+    }
+  }
+
+  async getNvidiaSmiInfo(): Promise<
+    | Array<{ vendor: string; model: string; vram: number }>
+    | { error: string }
+    | 'OLLAMA_NOT_FOUND'
+    | 'BAD_RESPONSE'
+    | 'UNKNOWN_ERROR'
+  > {
+    try {
+      const containers = await this.dockerService.docker.listContainers({ all: false })
+      const ollamaContainer = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`))
+      if (!ollamaContainer) {
+        logger.info(
+          'Ollama container not found for nvidia-smi info retrieval. This is expected if Ollama is not installed.'
+        )
+        return 'OLLAMA_NOT_FOUND'
+      }
+
+      // Execute nvidia-smi inside the Ollama container to get GPU info
+      const container = this.dockerService.docker.getContainer(ollamaContainer.Id)
+      const exec = await container.exec({
+        Cmd: ['nvidia-smi', '--query-gpu=name,memory.total', '--format=csv,noheader,nounits'],
+        AttachStdout: true,
+        AttachStderr: true,
+        Tty: true,
+      })
+
+      // Read the output stream with a timeout to prevent hanging if nvidia-smi fails
+      const stream = await exec.start({ Tty: true })
+      const output = await new Promise<string>((resolve) => {
+        let data = ''
+        const timeout = setTimeout(() => resolve(data), 5000)
+        stream.on('data', (chunk: Buffer) => {
+          data += chunk.toString()
+        })
+        stream.on('end', () => {
+          clearTimeout(timeout)
+          resolve(data)
+        })
+      })
+
+      // Remove any non-printable characters and trim the output
+      const cleaned = Array.from(output)
+        .filter((character) => character.charCodeAt(0) > 8)
+        .join('')
+        .trim()
+      if (
+        cleaned &&
+        !cleaned.toLowerCase().includes('error') &&
+        !cleaned.toLowerCase().includes('not found')
+      ) {
+        // Split by newlines to handle multiple GPUs installed
+        const lines = cleaned.split('\n').filter((line) => line.trim())
+
+        // Map each line out to a useful structure for us
+        const gpus = lines.map((line) => {
+          const parts = line.split(',').map((s) => s.trim())
+          return {
+            vendor: 'NVIDIA',
+            model: parts[0] || 'NVIDIA GPU',
+            vram: parts[1] ? Number.parseInt(parts[1], 10) : 0,
+          }
+        })
+
+        return gpus.length > 0 ? gpus : 'BAD_RESPONSE'
+      }
+
+      // If we got output but looks like an error, consider it a bad response from nvidia-smi
+      return 'BAD_RESPONSE'
+    } catch (error) {
+      logger.error('Error getting nvidia-smi info:', error)
+      if (error instanceof Error && error.message) {
+        return { error: error.message }
+      }
+      return 'UNKNOWN_ERROR'
+    }
+  }
+
+  async getExternalOllamaGpuInfo(): Promise<Array<{
+    vendor: string
+    model: string
+    vram: number
+  }> | null> {
+    try {
+      // If a remote Ollama URL is configured, use it directly without requiring a local container
+      const remoteOllamaUrl = await KVStore.getValue('ai.remoteOllamaUrl')
+      if (!remoteOllamaUrl) {
+        const containers = await this.dockerService.docker.listContainers({ all: false })
+        const ollamaContainer = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`))
+        if (!ollamaContainer) {
+          return null
+        }
+
+        const actualImage = (ollamaContainer.Image || '').toLowerCase()
+        if (actualImage.includes('ollama/ollama') || actualImage.startsWith('ollama:')) {
+          return null
+        }
+      }
+
+      const ollamaUrl = remoteOllamaUrl || (await this.dockerService.getServiceURL(SERVICE_NAMES.OLLAMA))
+      if (!ollamaUrl) {
+        return null
+      }
+
+      await axios.get(new URL('/api/tags', ollamaUrl).toString(), { timeout: 3000 })
+
+      let vramMb = 0
+      try {
+        const psResponse = await axios.get(new URL('/api/ps', ollamaUrl).toString(), {
+          timeout: 3000,
+        })
+        const loadedModels = Array.isArray(psResponse.data?.models) ? psResponse.data.models : []
+        const largestAllocation = loadedModels.reduce(
+          (max: number, model: { size_vram?: number | string }) =>
+            Math.max(max, Number(model.size_vram) || 0),
+          0
+        )
+        vramMb = largestAllocation > 0 ? Math.round(largestAllocation / (1024 * 1024)) : 0
+      } catch {}
+
+      return [
+        {
+          vendor: 'NVIDIA',
+          model: 'NVIDIA GPU (external Ollama)',
+          vram: vramMb,
+        },
+      ]
+    } catch (error) {
+      logger.info(
+        `[SystemService] External Ollama GPU probe failed: ${error instanceof Error ? error.message : error}`
+      )
+      return null
+    }
+  }
+
   async getServices({ installedOnly = true }: { installedOnly?: boolean }): Promise<ServiceSlim[]> {
-    await this._syncContainersWithDatabase() // Sync up before fetching to ensure we have the latest status
+    const statuses = await this._syncContainersWithDatabase() // Sync and reuse the fetched status list
 
     const query = Service.query()
+      .orderBy('display_order', 'asc')
       .orderBy('friendly_name', 'asc')
       .select(
         'id',
@@ -68,11 +328,26 @@ export class SystemService {
         'installed',
         'installation_status',
         'ui_location',
+        'custom_url',
         'friendly_name',
         'description',
-        'icon'
+        'icon',
+        'powered_by',
+        'display_order',
+        'container_image',
+        'available_update_version',
+        'auto_update_enabled',
+        'is_custom',
+        'is_user_modified',
+        'is_deprecated',
+        'category'
       )
       .where('is_dependency_service', false)
+      // Deprecated/sunset apps stay visible only while still installed, so the user can manage and
+      // uninstall them — they never reappear in the install catalog once removed.
+      .where((q) => {
+        q.where('is_deprecated', false).orWhere('installed', true)
+      })
     if (installedOnly) {
       query.where('installed', true)
     }
@@ -82,8 +357,6 @@ export class SystemService {
       return []
     }
 
-    const statuses = await this.dockerService.getServicesStatus()
-
     const toReturn: ServiceSlim[] = []
 
     for (const service of services) {
@@ -98,6 +371,16 @@ export class SystemService {
         installation_status: service.installation_status,
         status: status ? status.status : 'unknown',
         ui_location: service.ui_location || '',
+        custom_url: service.custom_url,
+        powered_by: service.powered_by,
+        display_order: service.display_order,
+        container_image: service.container_image,
+        available_update_version: service.available_update_version,
+        auto_update_enabled: service.auto_update_enabled,
+        is_custom: service.is_custom,
+        is_user_modified: service.is_user_modified,
+        is_deprecated: service.is_deprecated,
+        category: service.category,
       })
     }
 
@@ -131,13 +414,14 @@ export class SystemService {
 
   async getSystemInfo(): Promise<SystemInformationResponse | undefined> {
     try {
-      const [cpu, mem, os, currentLoad, fsSize, uptime] = await Promise.all([
+      const [cpu, mem, os, currentLoad, fsSize, uptime, graphics] = await Promise.all([
         si.cpu(),
         si.mem(),
         si.osInfo(),
         si.currentLoad(),
         si.fsSize(),
         si.time(),
+        si.graphics(),
       ])
 
       let diskInfo: NomadDiskInfoRaw | undefined
@@ -160,6 +444,192 @@ export class SystemService {
         logger.error('Error reading disk info file:', error)
       }
 
+      // GPU health tracking — detect when host has a GPU runtime but Ollama can't access it.
+      // Primary probe: parse Ollama's "inference compute" startup log line for both NVIDIA
+      // and AMD. Secondary probe (NVIDIA only): nvidia-smi exec, retained as a fallback for
+      // hardware enrichment when log parsing has not yet captured a startup line.
+      let gpuHealth: GpuHealthStatus = {
+        status: 'no_gpu',
+        hasNvidiaRuntime: false,
+        hasRocmRuntime: false,
+        ollamaGpuAccessible: false,
+      }
+
+      // Query Docker API for host-level info (hostname, OS, GPU runtime)
+      // si.osInfo() returns the container's info inside Docker, not the host's
+      try {
+        const dockerInfo = await this.dockerService.docker.info()
+
+        if (dockerInfo.Name) {
+          os.hostname = dockerInfo.Name
+        }
+        if (dockerInfo.OperatingSystem) {
+          os.distro = dockerInfo.OperatingSystem
+        }
+        if (dockerInfo.KernelVersion) {
+          os.kernel = dockerInfo.KernelVersion
+        }
+
+        // si.graphics() in the admin container uses lspci (pciutils ships in
+        // the image for AMD detection). lspci has no real VRAM info for
+        // discrete GPUs, so systeminformation parses the first PCI memory
+        // Region (BAR0, typically 1-32 MiB) as `vram`. nvidia-smi / ROCm
+        // tooling enrichment also can't run since neither is in the admin
+        // image. No real dGPU has under 256 MiB, so any discrete-GPU controller
+        // below that threshold needs the probes below to give us real data.
+        // Applies to both NVIDIA and AMD; Intel iGPUs are exempt because their
+        // shared-system-memory VRAM reading via lspci can legitimately be small.
+        const DGPU_BOGUS_VRAM_THRESHOLD_MIB = 256
+        const isDiscreteGpuVendor = (vendor: string) =>
+          /nvidia|advanced micro devices|amd|ati/i.test(vendor)
+        const isBogusDgpuVram = (c: { vendor?: string; vram?: number | null }) =>
+          isDiscreteGpuVendor(c.vendor || '') &&
+          typeof c.vram === 'number' &&
+          c.vram < DGPU_BOGUS_VRAM_THRESHOLD_MIB
+
+        // Clear the bogus value up front. If a probe replaces the entry below
+        // we get the real VRAM; if no probe succeeds (Ollama not installed,
+        // passthrough_failed) the UI falls back to "N/A" instead of showing
+        // "1 MB" / "32 MB". The lspci model/vendor strings stay since they're
+        // still useful for identifying the card.
+        const hasLspciBogusDgpuVram = (graphics.controllers || []).some(isBogusDgpuVram)
+        if (hasLspciBogusDgpuVram) {
+          for (const c of graphics.controllers) {
+            if (isBogusDgpuVram(c)) c.vram = null
+          }
+        }
+
+        // Run the probes when controllers are empty (common inside Docker) or
+        // when lspci gave us bogus discrete-GPU BAR0 values that need replacing.
+        if (
+          !graphics.controllers ||
+          graphics.controllers.length === 0 ||
+          hasLspciBogusDgpuVram
+        ) {
+          const runtimes = dockerInfo.Runtimes || {}
+          gpuHealth.hasNvidiaRuntime = 'nvidia' in runtimes
+
+          // AMD doesn't register a Docker runtime. Detection sources, in priority order:
+          //   1. KV 'gpu.type' (set by DockerService._detectGPUType after first Ollama install)
+          //   2. Marker file at /app/storage/.nomad-gpu-type (written by install_nomad.sh)
+          // The marker file matters because the System page should reflect AMD presence
+          // even before AI Assistant has been installed for the first time.
+          let savedGpuType: string | null | undefined = await KVStore.getValue('gpu.type') as string | undefined
+          if (!savedGpuType) {
+            try {
+              savedGpuType = (await readFile('/app/storage/.nomad-gpu-type', 'utf8')).trim()
+            } catch {}
+          }
+          const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration')
+          const amdAccelerationEnabled = String(amdEnabledRaw) !== 'false'
+          gpuHealth.hasRocmRuntime = savedGpuType === 'amd' && amdAccelerationEnabled
+
+          if (gpuHealth.hasNvidiaRuntime || gpuHealth.hasRocmRuntime) {
+            gpuHealth.gpuVendor = gpuHealth.hasNvidiaRuntime ? 'nvidia' : 'amd'
+
+            // Primary probe: Ollama log parsing — works for both vendors and catches silent fallback
+            const logInfo = await this.getOllamaInferenceComputeFromLogs()
+            if (logInfo) {
+              graphics.controllers = [
+                {
+                  model: logInfo.name,
+                  vendor: logInfo.library === 'CUDA' ? 'NVIDIA' : 'AMD',
+                  bus: '',
+                  vram: logInfo.vramMiB,
+                  vramDynamic: false,
+                },
+              ]
+              gpuHealth.status = 'ok'
+              gpuHealth.ollamaGpuAccessible = true
+            } else if (gpuHealth.hasNvidiaRuntime) {
+              // NVIDIA secondary path: nvidia-smi exec preserves prior behavior when
+              // the log parser hasn't seen a startup line yet (e.g. log rotation,
+              // very fresh container). Distinguishes "no Ollama container" from
+              // "container exists but GPU broken".
+              const nvidiaInfo = await this.getNvidiaSmiInfo()
+              if (Array.isArray(nvidiaInfo)) {
+                graphics.controllers = nvidiaInfo.map((gpu) => ({
+                  model: gpu.model,
+                  vendor: gpu.vendor,
+                  bus: '',
+                  vram: gpu.vram,
+                  vramDynamic: false,
+                }))
+                gpuHealth.status = 'ok'
+                gpuHealth.ollamaGpuAccessible = true
+              } else if (nvidiaInfo === 'OLLAMA_NOT_FOUND') {
+                const externalOllamaGpu = await this.getExternalOllamaGpuInfo()
+                if (externalOllamaGpu) {
+                  graphics.controllers = externalOllamaGpu.map((gpu) => ({
+                    model: gpu.model,
+                    vendor: gpu.vendor,
+                    bus: '',
+                    vram: gpu.vram,
+                    vramDynamic: false,
+                  }))
+                  gpuHealth.status = 'ok'
+                  gpuHealth.ollamaGpuAccessible = true
+                } else {
+                  gpuHealth.status = 'ollama_not_installed'
+                }
+              } else {
+                const externalOllamaGpu = await this.getExternalOllamaGpuInfo()
+                if (externalOllamaGpu) {
+                  graphics.controllers = externalOllamaGpu.map((gpu) => ({
+                    model: gpu.model,
+                    vendor: gpu.vendor,
+                    bus: '',
+                    vram: gpu.vram,
+                    vramDynamic: false,
+                  }))
+                  gpuHealth.status = 'ok'
+                  gpuHealth.ollamaGpuAccessible = true
+                } else {
+                  gpuHealth.status = 'passthrough_failed'
+                  logger.warn(
+                    `NVIDIA runtime detected but GPU passthrough failed: ${typeof nvidiaInfo === 'string' ? nvidiaInfo : JSON.stringify(nvidiaInfo)}`
+                  )
+                }
+              }
+            } else {
+              // AMD path: no nvidia-smi equivalent worth running — log parser is authoritative.
+              // Distinguish "Ollama not running" from "Ollama running but no GPU log line".
+              const containers = await this.dockerService.docker.listContainers({ all: false })
+              const ollamaRunning = containers.some((c) =>
+                c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`)
+              )
+              if (!ollamaRunning) {
+                const externalOllamaGpu = await this.getExternalOllamaGpuInfo()
+                if (externalOllamaGpu) {
+                  graphics.controllers = externalOllamaGpu.map((gpu) => ({
+                    model: gpu.model,
+                    vendor: gpu.vendor,
+                    bus: '',
+                    vram: gpu.vram,
+                    vramDynamic: false,
+                  }))
+                  gpuHealth.status = 'ok'
+                  gpuHealth.ollamaGpuAccessible = true
+                } else {
+                  gpuHealth.status = 'ollama_not_installed'
+                }
+              } else {
+                gpuHealth.status = 'passthrough_failed'
+                logger.warn(
+                  'AMD GPU detected but Ollama logs show no ROCm initialization — passthrough or HSA override may have failed'
+                )
+              }
+            }
+          }
+        } else {
+          // si.graphics() returned controllers (host install, not Docker) — GPU is working
+          gpuHealth.status = 'ok'
+          gpuHealth.ollamaGpuAccessible = true
+        }
+      } catch {
+        // Docker info query failed, skip host-level enrichment
+      }
+
       return {
         cpu,
         mem,
@@ -168,6 +638,8 @@ export class SystemService {
         currentLoad,
         fsSize,
         uptime,
+        graphics,
+        gpuHealth,
       }
     } catch (error) {
       logger.error('Error getting system info:', error)
@@ -175,7 +647,7 @@ export class SystemService {
     }
   }
 
-  async checkLatestVersion(): Promise<{
+  async checkLatestVersion(force?: boolean): Promise<{
     success: boolean
     updateAvailable: boolean
     currentVersion: string
@@ -183,25 +655,50 @@ export class SystemService {
     message?: string
   }> {
     try {
-      const response = await axios.get(
-        'https://api.github.com/repos/Crosstalk-Solutions/project-nomad/releases/latest',
-        {
-          headers: { Accept: 'application/vnd.github+json' },
-          timeout: 5000,
-        }
-      )
+      const currentVersion = SystemService.getAppVersion()
+      const cachedUpdateAvailable = await KVStore.getValue('system.updateAvailable')
+      const cachedLatestVersion = await KVStore.getValue('system.latestVersion')
 
-      if (!response || !response.data?.tag_name) {
-        throw new Error('Invalid response from GitHub API')
+      // Use cached values if not forcing a fresh check.
+      // the CheckUpdateJob will update these values every 12 hours
+      if (!force) {
+        return {
+          success: true,
+          updateAvailable: cachedUpdateAvailable ?? false,
+          currentVersion,
+          latestVersion: cachedLatestVersion || '',
+        }
       }
 
-      const latestVersion = response.data.tag_name.replace(/^v/, '') // Remove leading 'v' if present
-      const currentVersion = SystemService.getAppVersion()
+      const earlyAccess = (await KVStore.getValue('system.earlyAccess')) ?? false
+
+      let latestVersion: string
+      if (earlyAccess) {
+        const response = await axios.get(
+          'https://api.github.com/repos/Crosstalk-Solutions/project-nomad/releases',
+          { headers: { Accept: 'application/vnd.github+json' }, timeout: 5000 }
+        )
+        if (!response?.data?.length) throw new Error('No releases found')
+        latestVersion = response.data[0].tag_name.replace(/^v/, '').trim()
+      } else {
+        const response = await axios.get(
+          'https://api.github.com/repos/Crosstalk-Solutions/project-nomad/releases/latest',
+          { headers: { Accept: 'application/vnd.github+json' }, timeout: 5000 }
+        )
+        if (!response?.data?.tag_name) throw new Error('Invalid response from GitHub API')
+        latestVersion = response.data.tag_name.replace(/^v/, '').trim()
+      }
 
       logger.info(`Current version: ${currentVersion}, Latest version: ${latestVersion}`)
 
-      // NOTE: this will always return true in dev environment! See getAppVersion()
-      const updateAvailable = latestVersion !== currentVersion
+      const updateAvailable =
+        process.env.NODE_ENV === 'development'
+          ? false
+          : isNewerVersion(latestVersion, currentVersion.trim(), earlyAccess)
+
+      // Cache the results in KVStore for frontend checks
+      await KVStore.setValue('system.updateAvailable', updateAvailable)
+      await KVStore.setValue('system.latestVersion', latestVersion)
 
       return {
         success: true,
@@ -221,13 +718,192 @@ export class SystemService {
     }
   }
 
+  async subscribeToReleaseNotes(email: string): Promise<{ success: boolean; message: string }> {
+    try {
+      const response = await axios.post(
+        'https://api.projectnomad.us/api/v1/lists/release-notes/subscribe',
+        { email },
+        { timeout: 5000 }
+      )
+
+      if (response.status === 200) {
+        return {
+          success: true,
+          message: 'Successfully subscribed to release notes',
+        }
+      }
+
+      return {
+        success: false,
+        message: `Failed to subscribe: ${response.statusText}`,
+      }
+    } catch (error) {
+      logger.error('Error subscribing to release notes:', error)
+      return {
+        success: false,
+        message: `Failed to subscribe: ${error instanceof Error ? error.message : error}`,
+      }
+    }
+  }
+
+  async getDebugInfo(): Promise<string> {
+    const appVersion = SystemService.getAppVersion()
+    const environment = process.env.NODE_ENV || 'unknown'
+
+    const [systemInfo, services, internetStatus, versionCheck] = await Promise.all([
+      this.getSystemInfo(),
+      this.getServices({ installedOnly: false }),
+      this.getInternetStatus().catch(() => null),
+      this.checkLatestVersion().catch(() => null),
+    ])
+
+    const lines: string[] = [
+      'Project NOMAD Debug Info',
+      '========================',
+      `App Version: ${appVersion}`,
+      `Environment: ${environment}`,
+    ]
+
+    if (systemInfo) {
+      const { cpu, mem, os, disk, fsSize, uptime, graphics } = systemInfo
+
+      lines.push('')
+      lines.push('System:')
+      if (os.distro) lines.push(`  OS: ${os.distro}`)
+      if (os.hostname) lines.push(`  Hostname: ${os.hostname}`)
+      if (os.kernel) lines.push(`  Kernel: ${os.kernel}`)
+      if (os.arch) lines.push(`  Architecture: ${os.arch}`)
+      if (uptime?.uptime) lines.push(`  Uptime: ${this._formatUptime(uptime.uptime)}`)
+
+      lines.push('')
+      lines.push('Hardware:')
+      if (cpu.brand) {
+        lines.push(`  CPU: ${cpu.brand} (${cpu.cores} cores)`)
+      }
+      if (mem.total) {
+        const total = this._formatBytes(mem.total)
+        const used = this._formatBytes(mem.total - (mem.available || 0))
+        const available = this._formatBytes(mem.available || 0)
+        lines.push(`  RAM: ${total} total, ${used} used, ${available} available`)
+      }
+      if (graphics.controllers && graphics.controllers.length > 0) {
+        for (const gpu of graphics.controllers) {
+          const vram = gpu.vram ? ` (${gpu.vram} MB VRAM)` : ''
+          lines.push(`  GPU: ${gpu.model}${vram}`)
+        }
+      } else {
+        lines.push('  GPU: None detected')
+      }
+
+      // Disk info — try disk array first, fall back to fsSize
+      const diskEntries = disk.filter((d) => d.totalSize > 0)
+      if (diskEntries.length > 0) {
+        for (const d of diskEntries) {
+          const size = this._formatBytes(d.totalSize)
+          const type = d.tran?.toUpperCase() || (d.rota ? 'HDD' : 'SSD')
+          lines.push(`  Disk: ${size}, ${Math.round(d.percentUsed)}% used, ${type}`)
+        }
+      } else if (fsSize.length > 0) {
+        const realFs = fsSize.filter((f) => f.fs.startsWith('/dev/'))
+        const seen = new Set<number>()
+        for (const f of realFs) {
+          if (seen.has(f.size)) continue
+          seen.add(f.size)
+          lines.push(`  Disk: ${this._formatBytes(f.size)}, ${Math.round(f.use)}% used`)
+        }
+      }
+    }
+
+    const installed = services.filter((s) => s.installed)
+    lines.push('')
+    if (installed.length > 0) {
+      lines.push('Installed Services:')
+      for (const svc of installed) {
+        lines.push(`  ${svc.friendly_name} (${svc.service_name}): ${svc.status}`)
+      }
+    } else {
+      lines.push('Installed Services: None')
+    }
+
+    if (internetStatus !== null) {
+      lines.push('')
+      lines.push(`Internet Status: ${internetStatus ? 'Online' : 'Offline'}`)
+    }
+
+    if (versionCheck?.success) {
+      const updateMsg = versionCheck.updateAvailable
+        ? `Yes (${versionCheck.latestVersion} available)`
+        : `No (${versionCheck.currentVersion} is latest)`
+      lines.push(`Update Available: ${updateMsg}`)
+    }
+
+    return lines.join('\n')
+  }
+
+  private _formatUptime(seconds: number): string {
+    const days = Math.floor(seconds / 86400)
+    const hours = Math.floor((seconds % 86400) / 3600)
+    const minutes = Math.floor((seconds % 3600) / 60)
+    if (days > 0) return `${days}d ${hours}h ${minutes}m`
+    if (hours > 0) return `${hours}h ${minutes}m`
+    return `${minutes}m`
+  }
+
+  private _formatBytes(bytes: number, decimals = 1): string {
+    if (bytes === 0) return '0 Bytes'
+    const k = 1024
+    const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB']
+    const i = Math.floor(Math.log(bytes) / Math.log(k))
+    return Number.parseFloat((bytes / Math.pow(k, i)).toFixed(decimals)) + ' ' + sizes[i]
+  }
+
+  async updateSetting(key: KVStoreKey, value: any): Promise<void> {
+    if (
+      (value === '' || value === undefined || value === null) &&
+      KV_STORE_SCHEMA[key] === 'string'
+    ) {
+      await KVStore.clearValue(key)
+    } else {
+      await KVStore.setValue(key, value)
+    }
+    if (key === 'ai.assistantCustomName') {
+      invalidateAssistantNameCache()
+    }
+    // Re-enabling auto-update after a backoff-driven auto-disable clears the
+    // failure state so it gets a fresh start instead of immediately re-tripping.
+    if (key === 'autoUpdate.enabled' && (value === true || value === 'true')) {
+      await KVStore.setValue('autoUpdate.consecutiveFailures', '0')
+      await KVStore.clearValue('autoUpdate.autoDisabledReason')
+    }
+    // Re-enabling the global app auto-update master switch clears every app's
+    // per-app failure backoff so previously self-disabled apps get a fresh start.
+    if (key === 'appAutoUpdate.enabled' && (value === true || value === 'true')) {
+      await Service.query().update({
+        auto_update_consecutive_failures: 0,
+        auto_update_disabled_reason: null,
+      })
+    }
+    // Re-enabling content auto-update clears the feature-level backoff and every
+    // resource's per-resource backoff so previously self-disabled content gets a
+    // fresh start.
+    if (key === 'contentAutoUpdate.enabled' && (value === true || value === 'true')) {
+      await KVStore.setValue('contentAutoUpdate.consecutiveFailures', '0')
+      await KVStore.clearValue('contentAutoUpdate.autoDisabledReason')
+      await InstalledResource.query().update({
+        auto_update_consecutive_failures: 0,
+        auto_update_disabled_reason: null,
+      })
+    }
+  }
+
   /**
    * Checks the current state of Docker containers against the database records and updates the database accordingly.
    * It will mark services as not installed if their corresponding containers do not exist, regardless of their running state.
    * Handles cases where a container might have been manually removed, ensuring the database reflects the actual existence of containers.
    * Containers that exist but are stopped, paused, or restarting will still be considered installed.
+   * Returns the fetched service status list so callers can reuse it without a second Docker API call.
    */
-  private async _syncContainersWithDatabase() {
+  private async _syncContainersWithDatabase(): Promise<{ service_name: string; status: string }[]> {
     try {
       const allServices = await Service.all()
       const serviceStatusList = await this.dockerService.getServicesStatus()
@@ -236,10 +912,15 @@ export class SystemService {
         const containerExists = serviceStatusList.find(
           (s) => s.service_name === service.service_name
         )
-        
+
         if (service.installed) {
           // If marked as installed but container doesn't exist, mark as not installed
           if (!containerExists) {
+            // Exception: remote Ollama is configured without a local container — don't reset it
+            if (service.service_name === SERVICE_NAMES.OLLAMA) {
+              const remoteUrl = await KVStore.getValue('ai.remoteOllamaUrl')
+              if (remoteUrl) continue
+            }
             logger.warn(
               `Service ${service.service_name} is marked as installed but container does not exist. Marking as not installed.`
             )
@@ -259,8 +940,11 @@ export class SystemService {
           }
         }
       }
+
+      return serviceStatusList
     } catch (error) {
       logger.error('Error syncing containers with database:', error)
+      return []
     }
   }
 
@@ -271,10 +955,21 @@ export class SystemService {
       return []
     }
 
+    // Deduplicate: same device path mounted in multiple places (Docker bind-mounts)
+    // Keep the entry with the largest size — that's the real partition
+    const deduped = new Map<string, NomadDiskInfoRaw['fsSize'][0]>()
+    for (const entry of fsSize) {
+      const existing = deduped.get(entry.fs)
+      if (!existing || entry.size > existing.size) {
+        deduped.set(entry.fs, entry)
+      }
+    }
+    const dedupedFsSize = Array.from(deduped.values())
+
     return diskLayout.blockdevices
       .filter((disk) => disk.type === 'disk') // Only physical disks
       .map((disk) => {
-        const filesystems = getAllFilesystems(disk, fsSize)
+        const filesystems = getAllFilesystems(disk, dedupedFsSize)
 
         // Across all partitions
         const totalUsed = filesystems.reduce((sum, p) => sum + (p.used || 0), 0)
@@ -301,4 +996,83 @@ export class SystemService {
         }
       })
   }
+
+  /**
+   * Check whether the host has enough free memory and disk to comfortably run an app.
+   * Returns an array of human-readable warning strings; an empty array means no concerns.
+   * These are advisory only — the caller decides whether to block or warn.
+   */
+  async checkResourceWarnings(minMemoryMB: number, minDiskMB: number): Promise<string[]> {
+    const warnings: string[] = []
+
+    try {
+      const mem = await si.mem()
+      const availableMB = Math.floor(mem.available / 1024 / 1024)
+      if (availableMB < minMemoryMB) {
+        warnings.push(
+          `Low memory: ${availableMB} MB available, this app recommends at least ${minMemoryMB} MB free.`
+        )
+      }
+    } catch (err: any) {
+      logger.warn(`[SystemService] checkResourceWarnings mem check failed: ${err.message}`)
+    }
+
+    try {
+      const storagePath = env.get('NOMAD_STORAGE_PATH', '/opt/project-nomad/storage')
+      const fsSizes = await si.fsSize()
+      // Find the filesystem whose mount point is the longest prefix of storagePath
+      const fs = fsSizes
+        .filter((f) => storagePath.startsWith(f.mount))
+        .sort((a, b) => b.mount.length - a.mount.length)[0]
+
+      if (fs) {
+        const availableDiskMB = Math.floor((fs.size - fs.used) / 1024 / 1024)
+        if (availableDiskMB < minDiskMB) {
+          warnings.push(
+            `Low disk space: ${availableDiskMB} MB available on ${fs.mount}, this app recommends at least ${minDiskMB} MB free.`
+          )
+        }
+      }
+    } catch (err: any) {
+      logger.warn(`[SystemService] checkResourceWarnings disk check failed: ${err.message}`)
+    }
+
+    return warnings
+  }
+
+  /**
+   * Return the next suggested host port for a custom app in the 8600+ range.
+   * Looks at existing custom service records and all Docker container port bindings.
+   */
+  async getNextSuggestedCustomPort(): Promise<number> {
+    const CUSTOM_PORT_START = 8600
+    const occupied = new Set<number>()
+
+    try {
+      // Ports used by existing custom services in the DB
+      const customServices = await Service.query().where('is_custom', true)
+      for (const svc of customServices) {
+        const config = svc.container_config ? JSON.parse(svc.container_config) : null
+        const bindings = config?.HostConfig?.PortBindings ?? {}
+        for (const binding of Object.values(bindings) as any[]) {
+          const port = parseInt(binding?.[0]?.HostPort, 10)
+          if (!isNaN(port)) occupied.add(port)
+        }
+      }
+
+      // Ports used by any running Docker container in the 8600+ range
+      const containers = await this.dockerService.docker.listContainers({ all: true })
+      for (const c of containers) {
+        for (const p of c.Ports) {
+          if (p.PublicPort && p.PublicPort >= CUSTOM_PORT_START) occupied.add(p.PublicPort)
+        }
+      }
+    } catch (err: any) {
+      logger.warn(`[SystemService] getNextSuggestedCustomPort probe failed: ${err.message}`)
+    }
+
+    let candidate = CUSTOM_PORT_START
+    while (occupied.has(candidate)) candidate += 10
+    return candidate
+  }
 }

admin/app/services/system_update_service.ts

@@ -2,6 +2,7 @@ import logger from '@adonisjs/core/services/logger'
 import { readFileSync, existsSync } from 'fs'
 import { writeFile } from 'fs/promises'
 import { join } from 'path'
+import KVStore from '#models/kv_store'
 
 interface UpdateStatus {
   stage: 'idle' | 'starting' | 'pulling' | 'pulled' | 'recreating' | 'complete' | 'error'
@@ -17,11 +18,20 @@ export class SystemUpdateService {
   private static LOG_FILE = join(SystemUpdateService.SHARED_DIR, 'update-log')
 
   /**
-   * Requests a system update by creating a request file that the sidecar will detect
+   * Requests a system update by creating a request file that the sidecar will detect.
+   *
+   * @param options.targetTag - Explicit Docker image tag to install (e.g. "v1.33.2").
+   *   When omitted, falls back to the cached `system.latestVersion` (manual-update
+   *   behavior). Auto-update passes an eligibility-vetted tag here, which may differ
+   *   from `system.latestVersion` when the newest release is a major bump.
+   * @param options.requester - Identifier recorded in the request file for auditing.
    */
-  async requestUpdate(): Promise<{ success: boolean; message: string }> {
+  async requestUpdate(options?: {
+    targetTag?: string
+    requester?: string
+  }): Promise<{ success: boolean; message: string }> {
     try {
-      const currentStatus = this.getUpdateStatus()      
+      const currentStatus = this.getUpdateStatus()
       if (currentStatus && !['idle', 'complete', 'error'].includes(currentStatus.stage)) {
         return {
           success: false,
@@ -29,23 +39,32 @@ export class SystemUpdateService {
         }
       }
 
+      // Determine the Docker image tag to install. Prefer an explicit caller-supplied
+      // tag; otherwise use the cached latest version.
+      let targetTag = options?.targetTag
+      if (!targetTag) {
+        const latestVersion = await KVStore.getValue('system.latestVersion')
+        targetTag = latestVersion ? `v${latestVersion}` : 'latest'
+      }
+
       const requestData = {
         requested_at: new Date().toISOString(),
-        requester: 'admin-api',
+        requester: options?.requester ?? 'admin-api',
+        target_tag: targetTag,
       }
 
       await writeFile(SystemUpdateService.REQUEST_FILE, JSON.stringify(requestData, null, 2))
-      logger.info('[SystemUpdateService]: System update requested - sidecar will process shortly')
+      logger.info(`[SystemUpdateService]: System update requested (target tag: ${requestData.target_tag}) - sidecar will process shortly`)
 
       return {
         success: true,
         message: 'System update initiated. The admin container will restart during the process.',
       }
     } catch (error) {
-      logger.error('[SystemUpdateService]: Failed to request system update:', error)
+      logger.error({ err: error }, '[SystemUpdateService] Failed to request system update')
       return {
         success: false,
-        message: `Failed to request update: ${error.message}`,
+        message: 'Failed to request system update. Check server logs for details.',
       }
     }
   }

admin/app/services/zim_extraction_service.ts

@@ -0,0 +1,337 @@
+import { Archive, Entry } from '@openzim/libzim'
+import * as cheerio from 'cheerio'
+import { HTML_SELECTORS_TO_REMOVE, NON_CONTENT_HEADING_PATTERNS } from '../../constants/zim_extraction.js'
+import logger from '@adonisjs/core/services/logger'
+import { ExtractZIMChunkingStrategy, ExtractZIMContentOptions, ZIMContentChunk, ZIMArchiveMetadata } from '../../types/zim.js'
+import { randomUUID } from 'node:crypto'
+import { access } from 'node:fs/promises'
+import { isValidZimFile } from '../utils/fs.js'
+
+export class ZIMExtractionService {
+
+    private extractArchiveMetadata(archive: Archive): ZIMArchiveMetadata {
+        try {
+            return {
+                title: archive.getMetadata('Title') || archive.getMetadata('Name') || 'Unknown',
+                creator: archive.getMetadata('Creator') || 'Unknown',
+                publisher: archive.getMetadata('Publisher') || 'Unknown',
+                date: archive.getMetadata('Date') || 'Unknown',
+                language: archive.getMetadata('Language') || 'Unknown',
+                description: archive.getMetadata('Description') || '',
+            }
+        } catch (error) {
+            logger.warn('[ZIMExtractionService]: Could not extract all metadata, using defaults', error)
+            return {
+                title: 'Unknown',
+                creator: 'Unknown',
+                publisher: 'Unknown',
+                date: 'Unknown',
+                language: 'Unknown',
+                description: '',
+            }
+        }
+    }
+
+    /**
+     * Breaks out a ZIM file's entries into their structured content form
+     * to facilitate better indexing and retrieval.
+     * Returns enhanced chunks with full article context and metadata.
+     * 
+     * @param filePath - Path to the ZIM file
+     * @param opts - Options including maxArticles, strategy, onProgress, startOffset, and batchSize
+     */
+    async extractZIMContent(
+        filePath: string,
+        opts: ExtractZIMContentOptions = {}
+    ): Promise<{ chunks: ZIMContentChunk[]; totalArticles: number }> {
+        try {
+            logger.info(`[ZIMExtractionService]: Processing ZIM file at path: ${filePath}`)
+            
+            // defensive - check if file still exists before opening
+            // could have been deleted by another process or batch
+            try {
+                await access(filePath)
+            } catch (error) {
+                logger.error(`[ZIMExtractionService]: ZIM file not accessible: ${filePath}`)
+                throw new Error(`ZIM file not found or not accessible: ${filePath}`)
+            }
+
+            // Validate ZIM magic number before opening with native library.
+            // A corrupted file causes a native C++ abort that cannot be caught by JS.
+            if (!(await isValidZimFile(filePath))) {
+                throw new Error(`ZIM file is invalid or corrupted: ${filePath}`)
+            }
+
+            const archive = new Archive(filePath)
+
+            // Extract archive-level metadata once
+            const archiveMetadata = this.extractArchiveMetadata(archive)
+            logger.info(`[ZIMExtractionService]: Archive metadata - Title: ${archiveMetadata.title}, Language: ${archiveMetadata.language}`)
+
+            let articlesProcessed = 0
+            let articlesSkipped = 0
+            const processedPaths = new Set<string>()
+            const toReturn: ZIMContentChunk[] = []
+
+            // Support batch processing to avoid lock timeouts on large ZIM files
+            const startOffset = opts.startOffset || 0
+            const batchSize = opts.batchSize || (opts.maxArticles || Infinity)
+
+            for (const entry of archive.iterByPath()) {
+                // Skip articles until we reach the start offset
+                if (articlesSkipped < startOffset) {
+                    if (this.isArticleEntry(entry) && !processedPaths.has(entry.path)) {
+                        articlesSkipped++
+                    }
+                    continue
+                }
+
+                if (articlesProcessed >= batchSize) {
+                    break
+                }
+
+                if (!this.isArticleEntry(entry)) {
+                    logger.debug(`[ZIMExtractionService]: Skipping non-article entry at path: ${entry.path}`)
+                    continue
+                }
+
+                if (processedPaths.has(entry.path)) {
+                    logger.debug(`[ZIMExtractionService]: Skipping duplicate entry at path: ${entry.path}`)
+                    continue
+                }
+                processedPaths.add(entry.path)
+
+                const item = entry.item
+                const blob = item.data
+                const html = this.getCleanedHTMLString(blob.data)
+
+                const strategy = opts.strategy || this.chooseChunkingStrategy(html);
+                logger.debug(`[ZIMExtractionService]: Chosen chunking strategy for path ${entry.path}: ${strategy}`)
+
+                // Generate a unique document ID. All chunks from same article will share it
+                const documentId = randomUUID()
+                const articleTitle = entry.title || entry.path
+
+                let chunks: ZIMContentChunk[]
+
+                if (strategy === 'structured') {
+                    const structured = this.extractStructuredContent(html)
+                    chunks = structured.sections.map(s => ({
+                        text: s.text,
+                        articleTitle,
+                        articlePath: entry.path,
+                        sectionTitle: s.heading,
+                        fullTitle: `${articleTitle} - ${s.heading}`,
+                        hierarchy: `${articleTitle} > ${s.heading}`,
+                        sectionLevel: s.level,
+                        documentId,
+                        archiveMetadata,
+                        strategy,
+                    }))
+                } else {
+                    // Simple strategy - entire article as one chunk
+                    const text = this.extractTextFromHTML(html) || ''
+                    chunks = [{
+                        text,
+                        articleTitle,
+                        articlePath: entry.path,
+                        sectionTitle: articleTitle, // Same as article for simple strategy
+                        fullTitle: articleTitle,
+                        hierarchy: articleTitle,
+                        documentId,
+                        archiveMetadata,
+                        strategy,
+                    }]
+                }
+
+                logger.debug(`Extracted ${chunks.length} chunks from article at path: ${entry.path} using strategy: ${strategy}`)
+
+                const nonEmptyChunks = chunks.filter(c => c.text.trim().length > 0)
+                logger.debug(`After filtering empty chunks, ${nonEmptyChunks.length} chunks remain for article at path: ${entry.path}`)
+                toReturn.push(...nonEmptyChunks)
+                articlesProcessed++
+
+                if (opts.onProgress) {
+                    opts.onProgress(articlesProcessed, archive.articleCount)
+                }
+            }
+
+            logger.info(`[ZIMExtractionService]: Completed processing ZIM file. Total articles processed: ${articlesProcessed}`)
+            logger.debug("Final structured content sample:", toReturn.slice(0, 3).map(c => ({
+                articleTitle: c.articleTitle,
+                sectionTitle: c.sectionTitle,
+                hierarchy: c.hierarchy,
+                textPreview: c.text.substring(0, 100)
+            })))
+            logger.debug("Total structured sections extracted:", toReturn.length)
+            return { chunks: toReturn, totalArticles: archive.articleCount }
+        } catch (error) {
+            logger.error('Error processing ZIM file:', error)
+            throw error
+        }
+    }
+
+    private chooseChunkingStrategy(html: string, options = {
+        forceStrategy: null as ExtractZIMChunkingStrategy | null,
+    }): ExtractZIMChunkingStrategy {
+        const {
+            forceStrategy = null,
+        } = options;
+
+        if (forceStrategy) return forceStrategy;
+
+        // Use a simple analysis to determin if the HTML has any meaningful structure
+        // that we can leverage for better chunking. If not, we'll just chunk it as one big piece of text.
+        return this.hasStructuredHeadings(html) ? 'structured' : 'simple';
+    }
+
+    private getCleanedHTMLString(buff: Buffer<ArrayBufferLike>): string {
+        const rawString = buff.toString('utf-8');
+        const $ = cheerio.load(rawString);
+
+        HTML_SELECTORS_TO_REMOVE.forEach((selector) => {
+            $(selector).remove()
+        });
+
+        return $.html();
+    }
+
+    private extractTextFromHTML(html: string): string | null {
+        try {
+            const $ = cheerio.load(html)
+
+            // Search body first, then root if body is absent
+            const text = $('body').length ? $('body').text() : $.root().text()
+
+            return text.replace(/\s+/g, ' ').replace(/\n\s*\n/g, '\n').trim()
+        } catch (error) {
+            logger.error('Error extracting text from HTML:', error)
+            return null
+        }
+    }
+
+    private extractStructuredContent(html: string) {
+        const $ = cheerio.load(html);
+
+        const title = $('h1').first().text().trim() || $('title').text().trim();
+
+        // Extract sections with their headings and heading levels
+        const sections: Array<{ heading: string; text: string; level: number }> = [];
+        let currentSection = { heading: 'Introduction', content: [] as string[], level: 2 };
+
+        // Walk the full DOM rather than only direct children of <body>. Modern ZIMs (Devdocs,
+        // Wikipedia, FreeCodeCamp, etc.) wrap article content in a container div, which under
+        // .children() would be a single non-heading/non-paragraph element and yield zero sections.
+        $('body').find('h2, h3, h4, p, ul, ol, dl, table').each((_, element) => {
+            const $el = $(element);
+            const tagName = element.tagName?.toLowerCase();
+
+            if (['h2', 'h3', 'h4'].includes(tagName)) {
+                // Save current section if it has content
+                if (currentSection.content.length > 0) {
+                    sections.push({
+                        heading: currentSection.heading,
+                        text: currentSection.content.join(' ').replace(/\s+/g, ' ').trim(),
+                        level: currentSection.level,
+                    });
+                }
+                // Start new section
+                const level = parseInt(tagName.substring(1)); // Extract number from h2, h3, h4
+                currentSection = {
+                    heading: $el.text().replace(/\[edit\]/gi, '').trim(),
+                    content: [],
+                    level,
+                };
+            } else if (['p', 'ul', 'ol', 'dl', 'table'].includes(tagName)) {
+                const text = $el.text().trim();
+                if (text.length > 0) {
+                    currentSection.content.push(text);
+                }
+            }
+        });
+
+        // Push the last section if it has content
+        if (currentSection.content.length > 0) {
+            sections.push({
+                heading: currentSection.heading,
+                text: currentSection.content.join(' ').replace(/\s+/g, ' ').trim(),
+                level: currentSection.level,
+            });
+        }
+
+        // Fallback: if the selector walk produced no sections but the body has meaningful
+        // text (unusual structure, minimal markup), emit one section with the full body text
+        // so the article still contributes to the knowledge base.
+        if (sections.length === 0) {
+            const bodyText = $('body').text().replace(/\s+/g, ' ').trim();
+            if (bodyText.length > 0) {
+                sections.push({
+                    heading: title || 'Content',
+                    text: bodyText,
+                    level: 2,
+                });
+            }
+        }
+
+        return {
+            title,
+            sections,
+            fullText: sections.map(s => `${s.heading}\n${s.text}`).join('\n\n'),
+        };
+    }
+
+    private hasStructuredHeadings(html: string): boolean {
+        const $ = cheerio.load(html);
+
+        const headings = $('h2, h3').toArray();
+
+        // Consider it structured if it has at least 2 headings to break content into meaningful sections
+        if (headings.length < 2) return false;
+
+        // Check that headings have substantial content between them
+        let sectionsWithContent = 0;
+
+        for (const heading of headings) {
+            const $heading = $(heading);
+            const headingText = $heading.text().trim();
+
+            // Skip empty or very short headings, likely not meaningful
+            if (headingText.length < 3) continue;
+
+            // Skip common non-content headings
+            if (NON_CONTENT_HEADING_PATTERNS.some(pattern => pattern.test(headingText))) {
+                continue;
+            }
+
+            // Content until next heading
+            let contentLength = 0;
+            let $next = $heading.next();
+
+            while ($next.length && !$next.is('h1, h2, h3, h4')) {
+                contentLength += $next.text().trim().length;
+                $next = $next.next();
+            }
+
+            // Consider it a real section if it has at least 100 chars of content
+            if (contentLength >= 100) {
+                sectionsWithContent++;
+            }
+        }
+
+        // Require at least 2 sections with substantial content
+        return sectionsWithContent >= 2;
+    }
+
+    private isArticleEntry(entry: Entry): boolean {
+        try {
+            if (entry.isRedirect) return false;
+
+            const item = entry.item;
+            const mimeType = item.mimetype;
+
+            return mimeType === 'text/html' || mimeType === 'application/xhtml+xml';
+        } catch {
+            return false;
+        }
+    }
+}

admin/app/utils/content_auto_update_backoff.ts

@@ -0,0 +1,51 @@
+import logger from '@adonisjs/core/services/logger'
+import type InstalledResource from '#models/installed_resource'
+
+/**
+ * Per-resource failure backoff for content (ZIM/map) auto-updates, shared by the
+ * three places that observe an auto-update's real lifecycle:
+ *
+ *   - {@link ContentAutoUpdateService.attempt} — a dispatch that fails to even
+ *     enqueue (counts as a failure; no job runs so no terminal event follows).
+ *   - `RunDownloadJob.onComplete` — a download that actually finished (success).
+ *   - the worker `failed` handler in `commands/queue/work.ts` — a download that
+ *     exhausted its retries (terminal failure).
+ *
+ * Kept in a dependency-light util (not on ContentAutoUpdateService) on purpose:
+ * RunDownloadJob is imported by CollectionUpdateService, which is imported by
+ * ContentAutoUpdateService, so importing the service back into the job would
+ * close an import cycle. Only the InstalledResource model and the logger are
+ * touched here.
+ */
+
+/** Genuine consecutive auto-update failures before a resource self-disables. */
+export const MAX_CONSECUTIVE_FAILURES = 3
+
+/** Clear a resource's failure backoff after a successful auto-update. */
+export async function recordResourceUpdateSuccess(resource: InstalledResource): Promise<void> {
+  if (resource.auto_update_consecutive_failures === 0 && !resource.auto_update_disabled_reason) {
+    return
+  }
+  resource.auto_update_consecutive_failures = 0
+  resource.auto_update_disabled_reason = null
+  await resource.save()
+}
+
+/** Record an auto-update failure and self-disable the resource at the threshold. */
+export async function recordResourceUpdateFailure(
+  resource: InstalledResource,
+  reason: string
+): Promise<void> {
+  const failures = (resource.auto_update_consecutive_failures || 0) + 1
+  resource.auto_update_consecutive_failures = failures
+  if (failures >= MAX_CONSECUTIVE_FAILURES) {
+    resource.auto_update_disabled_reason = `Auto-update disabled after ${failures} consecutive failures. Last error: ${reason}`
+    logger.error(
+      `[ContentAutoUpdate] ${resource.resource_id} auto-disabled after ${failures} failures`
+    )
+  }
+  await resource.save()
+  logger.error(
+    `[ContentAutoUpdate] ${resource.resource_id} failure ${failures}/${MAX_CONSECUTIVE_FAILURES}: ${reason}`
+  )
+}

admin/app/utils/content_reindex_decision.ts

@@ -0,0 +1,58 @@
+/**
+ * Decision for reconciling the AI knowledge base (Qdrant) after a curated
+ * content file (a ZIM) is replaced by a newer downloaded version.
+ *
+ * This is the pure, I/O-free core of `RagService.reconcileReplacedContentFile`.
+ * Keeping the branching here (mirrors `decideScanAction` in
+ * `kb_ingest_decision.ts`) makes the contract exhaustively testable without a
+ * database or a live Qdrant.
+ *
+ * The logic deliberately MIRRORS the replaced file's prior indexed state rather
+ * than applying the global `rag.defaultIngestPolicy`: on a content *update* the
+ * user has already made an indexing choice for this content, so we honor it in
+ * both directions (re-index a previously-indexed file even under Manual; leave a
+ * previously-unindexed file alone even under Always). Fresh installs still go
+ * through the normal policy path — those return `not_a_replacement` here.
+ *
+ * Outcomes (evaluated top-down, short-circuiting):
+ * - `not_a_replacement` — no prior file, or the new file has the same path
+ *   (same-version re-download). Caller defers to normal ingest policy.
+ * - `qdrant_not_installed` (step 2) — no knowledge base exists; nothing to do.
+ * - `old_not_indexed` (step 4) — the replaced file was never embedded
+ *   (no state row, or state ≠ `indexed`); leave the new file un-indexed.
+ * - `qdrant_not_running` (step 5) — the replaced file WAS indexed but Qdrant is
+ *   currently offline. We do nothing: we can't remove the stale points, and a
+ *   queued embed job could be dropped before Qdrant returns. Acting half-way is
+ *   wasteful, so we defer entirely (accepted tradeoff: stale points linger).
+ * - `reindex` (step 3) — the replaced file was indexed and Qdrant is running:
+ *   delete ONLY the old file's points, drop its state row, and queue the new
+ *   file for embedding.
+ *
+ * Note the install-before-indexed ordering: step 2 short-circuits before any KB
+ * state lookup, matching the spec.
+ */
+export type ReindexOutcome =
+  | 'not_a_replacement'
+  | 'qdrant_not_installed'
+  | 'old_not_indexed'
+  | 'qdrant_not_running'
+  | 'reindex'
+
+export interface ContentReindexInput {
+  /** The replaced file existed AND its path differs from the new file's path. */
+  isReplacement: boolean
+  /** `nomad_qdrant` service exists (installed), regardless of running state. */
+  qdrantInstalled: boolean
+  /** The replaced file's `KbIngestState.state === 'indexed'`. */
+  oldFileWasIndexed: boolean
+  /** Qdrant answered a live health check (currently reachable). */
+  qdrantRunning: boolean
+}
+
+export function decideContentReindex(input: ContentReindexInput): ReindexOutcome {
+  if (!input.isReplacement) return 'not_a_replacement'
+  if (!input.qdrantInstalled) return 'qdrant_not_installed'
+  if (!input.oldFileWasIndexed) return 'old_not_indexed'
+  if (!input.qdrantRunning) return 'qdrant_not_running'
+  return 'reindex'
+}

admin/app/utils/downloads.ts

@@ -6,6 +6,7 @@ import axios from 'axios'
 import { Transform } from 'stream'
 import { deleteFileIfExists, ensureDirectoryExists, getFileStatsIfExists } from './fs.js'
 import { createWriteStream } from 'fs'
+import { rename } from 'fs/promises'
 import path from 'path'
 
 /**
@@ -27,13 +28,16 @@ export async function doResumableDownload({
   const dirname = path.dirname(filepath)
   await ensureDirectoryExists(dirname)
 
-  // Check if partial file exists for resume
+  // Stage download to a .tmp file so consumers (e.g. Kiwix) never see a partial file
+  const tempPath = filepath + '.tmp'
+
+  // Check if partial .tmp file exists for resume
   let startByte = 0
   let appendMode = false
 
-  const existingStats = await getFileStatsIfExists(filepath)
+  const existingStats = await getFileStatsIfExists(tempPath)
   if (existingStats && !forceNew) {
-    startByte = existingStats.size
+    startByte = Number(existingStats.size)
     appendMode = true
   }
 
@@ -43,8 +47,15 @@ export async function doResumableDownload({
     timeout,
   })
 
-  const contentType = headResponse.headers['content-type'] || ''
-  const totalBytes = parseInt(headResponse.headers['content-length'] || '0')
+  // Some upstream hosts (notably download.kiwix.org for .zim files) don't set a
+  // Content-Type header at all. Per RFC 7231 §3.1.1.5, "if no Content-Type is
+  // provided" the recipient may treat it as application/octet-stream — which is
+  // already in every binary-content allowlist we use (ZIM, PMTILES, base assets).
+  // Without this default, the validator below throws `MIME type  is not allowed`
+  // and breaks all downloads from kiwix's primary host (#848).
+  const contentType =
+    headResponse.headers['content-type']?.toString() || 'application/octet-stream'
+  const totalBytes = parseInt(headResponse.headers['content-length']?.toString() || '0', 10)
   const supportsRangeRequests = headResponse.headers['accept-ranges'] === 'bytes'
 
   // If allowedMimeTypes is provided, check content type
@@ -55,14 +66,24 @@ export async function doResumableDownload({
     }
   }
 
-  // If file is already complete and not forcing overwrite just return filepath
-  if (startByte === totalBytes && totalBytes > 0 && !forceNew) {
+  // If final file already exists at correct size, return early (idempotent)
+  const finalFileStats = await getFileStatsIfExists(filepath)
+  if (finalFileStats && Number(finalFileStats.size) === totalBytes && totalBytes > 0 && !forceNew) {
     return filepath
   }
 
-  // If server doesn't support range requests and we have a partial file, delete it
+  // If .tmp file is already at correct size (complete but never renamed), just rename it
+  if (startByte === totalBytes && totalBytes > 0 && !forceNew) {
+    await rename(tempPath, filepath)
+    if (onComplete) {
+      await onComplete(url, filepath)
+    }
+    return filepath
+  }
+
+  // If server doesn't support range requests and we have a partial .tmp file, delete it
   if (!supportsRangeRequests && startByte > 0) {
-    await deleteFileIfExists(filepath)
+    await deleteFileIfExists(tempPath)
     startByte = 0
     appendMode = false
   }
@@ -72,26 +93,57 @@ export async function doResumableDownload({
     headers.Range = `bytes=${startByte}-`
   }
 
-  const response = await axios.get(url, {
-    responseType: 'stream',
-    headers,
-    signal,
-    timeout,
-  })
+  const fetchStream = (hdrs: Record<string, string>) =>
+    axios.get(url, { responseType: 'stream', headers: hdrs, signal, timeout })
+
+  let response = await fetchStream(headers)
 
   if (response.status !== 200 && response.status !== 206) {
     throw new Error(`Failed to download: HTTP ${response.status}`)
   }
 
+  // If we requested a range but the server returned 200 (ignored the Range header),
+  // appending would corrupt the .tmp file — delete it and restart from byte 0.
+  if (headers.Range && response.status === 200) {
+    response.data.destroy()
+    await deleteFileIfExists(tempPath)
+    startByte = 0
+    appendMode = false
+    delete headers.Range
+    response = await fetchStream(headers)
+    if (response.status !== 200 && response.status !== 206) {
+      throw new Error(`Failed to download: HTTP ${response.status}`)
+    }
+  }
+
   return new Promise((resolve, reject) => {
     let downloadedBytes = startByte
     let lastProgressTime = Date.now()
     let lastDownloadedBytes = startByte
 
+    // Stall detection: if no data arrives for 5 minutes, abort the download
+    const STALL_TIMEOUT_MS = 5 * 60 * 1000
+    let stallTimer: ReturnType<typeof setTimeout> | null = null
+
+    const clearStallTimer = () => {
+      if (stallTimer) {
+        clearTimeout(stallTimer)
+        stallTimer = null
+      }
+    }
+
+    const resetStallTimer = () => {
+      clearStallTimer()
+      stallTimer = setTimeout(() => {
+        cleanup(new Error('Download stalled - no data received for 5 minutes'))
+      }, STALL_TIMEOUT_MS)
+    }
+
     // Progress tracking stream to monitor data flow
     const progressStream = new Transform({
       transform(chunk: Buffer, _: any, callback: Function) {
         downloadedBytes += chunk.length
+        resetStallTimer()
 
         // Update progress tracking
         const now = Date.now()
@@ -112,12 +164,12 @@ export async function doResumableDownload({
       },
     })
 
-    const writeStream = createWriteStream(filepath, {
+    const writeStream = createWriteStream(tempPath, {
       flags: appendMode ? 'a' : 'w',
     })
 
-    // Handle errors and cleanup
     const cleanup = (error?: Error) => {
+      clearStallTimer()
       progressStream.destroy()
       response.data.destroy()
       writeStream.destroy()
@@ -129,13 +181,27 @@ export async function doResumableDownload({
     response.data.on('error', cleanup)
     progressStream.on('error', cleanup)
     writeStream.on('error', cleanup)
-    writeStream.on('error', cleanup)
 
     signal?.addEventListener('abort', () => {
       cleanup(new Error('Download aborted'))
     })
 
     writeStream.on('finish', async () => {
+      clearStallTimer()
+      try {
+        // Atomically move the completed .tmp file to the final path
+        await rename(tempPath, filepath)
+      } catch (renameError) {
+        // A parallel job may have completed the same file first — treat as success
+        // if the destination already exists at the expected size.
+        const existing = await getFileStatsIfExists(filepath)
+        if (existing && Number(existing.size) === totalBytes && totalBytes > 0) {
+          // fall through to resolve
+        } else {
+          reject(renameError)
+          return
+        }
+      }
       if (onProgress) {
         onProgress({
           downloadedBytes,
@@ -151,7 +217,8 @@ export async function doResumableDownload({
       resolve(filepath)
     })
 
-    // Pipe: response -> progressStream -> writeStream
+    // Start stall timer and pipe: response -> progressStream -> writeStream
+    resetStallTimer()
     response.data.pipe(progressStream).pipe(writeStream)
   })
 }
@@ -185,7 +252,7 @@ export async function doResumableDownloadWithRetry({
       })
 
       return result // return on success
-    } catch (error) {
+    } catch (error: any) {
       attempt++
       lastError = error as Error
 

admin/app/utils/fs.ts

@@ -1,10 +1,27 @@
-import { mkdir, readdir, readFile, stat, unlink } from 'fs/promises'
-import { join } from 'path'
+import { mkdir, open, readdir, readFile, stat, unlink } from 'fs/promises'
+import path, { join } from 'path'
 import { FileEntry } from '../../types/files.js'
 import { createReadStream } from 'fs'
 import { LSBlockDevice, NomadDiskInfoRaw } from '../../types/system.js'
 
 export const ZIM_STORAGE_PATH = '/storage/zim'
+export const KIWIX_LIBRARY_XML_PATH = '/storage/zim/kiwix-library.xml'
+export const BOOKS_STORAGE_PATH = '/storage/books'
+// Shared media root (Jellyfin reads it as /media; File Browser shows it as "media"). Per-type
+// subfolders are pre-created on Jellyfin install — see _runPreinstallActions__Jellyfin.
+export const MEDIA_STORAGE_PATH = '/storage/media'
+export const JELLYFIN_MEDIA_SUBFOLDERS = ['Movies', 'TV Shows', 'Music', 'Photos']
+// Empty Calibre library bundled into the admin image (see install/calibre-empty-library/).
+// Seeded into storage/books on Calibre-Web install so it doesn't dead-end at db config.
+export const CALIBRE_EMPTY_LIBRARY_ASSET_PATH = 'assets/calibre/metadata.db'
+// Vaultwarden's /data volume. A self-signed TLS cert is generated here on install so the
+// web vault has the secure context (HTTPS) it requires — see _runPreinstallActions__Vaultwarden.
+export const VAULTWARDEN_STORAGE_PATH = '/storage/vaultwarden'
+// MeshCore Web's working dir. On install a self-signed cert (certs/) and an SSL nginx config
+// (nginx-ssl.conf) are generated here, then bind-mounted into the container so the static client is
+// served over HTTPS — required for its Web Bluetooth/Serial connections. See
+// _runPreinstallActions__MeshCoreWeb.
+export const MESHCORE_WEB_STORAGE_PATH = '/storage/meshcore-web'
 
 export async function listDirectoryContents(path: string): Promise<FileEntry[]> {
   const entries = await readdir(path, { withFileTypes: true })
@@ -49,7 +66,7 @@ export async function listDirectoryContentsRecursive(path: string): Promise<File
 export async function ensureDirectoryExists(path: string): Promise<void> {
   try {
     await stat(path)
-  } catch (error) {
+  } catch (error: any) {
     if (error.code === 'ENOENT') {
       await mkdir(path, { recursive: true })
     }
@@ -73,7 +90,7 @@ export async function getFile(
       return createReadStream(path)
     }
     return await readFile(path)
-  } catch (error) {
+  } catch (error: any) {
     if (error.code === 'ENOENT') {
       return null
     }
@@ -90,7 +107,7 @@ export async function getFileStatsIfExists(
       size: stats.size,
       modifiedTime: stats.mtime,
     }
-  } catch (error) {
+  } catch (error: any) {
     if (error.code === 'ENOENT') {
       return null
     }
@@ -98,10 +115,32 @@ export async function getFileStatsIfExists(
   }
 }
 
+/**
+ * Validates that a file has the ZIM magic number (0x44D495A).
+ * Must be called before passing a file to @openzim/libzim Archive,
+ * because a corrupted ZIM causes a native C++ abort that cannot be
+ * caught by JS try/catch.
+ */
+export async function isValidZimFile(filePath: string): Promise<boolean> {
+  let fh
+  try {
+    fh = await open(filePath, 'r')
+    const buf = Buffer.alloc(4)
+    const { bytesRead } = await fh.read(buf, 0, 4, 0)
+    if (bytesRead < 4) return false
+    // ZIM magic number: 72 17 32 04 (little-endian 0x044D4953)
+    return buf[0] === 0x5a && buf[1] === 0x49 && buf[2] === 0x4d && buf[3] === 0x04
+  } catch {
+    return false
+  } finally {
+    await fh?.close()
+  }
+}
+
 export async function deleteFileIfExists(path: string): Promise<void> {
   try {
     await unlink(path)
-  } catch (error) {
+  } catch (error: any) {
     if (error.code !== 'ENOENT') {
       throw error
     }
@@ -138,16 +177,41 @@ export function matchesDevice(fsPath: string, deviceName: string): boolean {
   // Remove /dev/ and /dev/mapper/ prefixes
   const normalized = fsPath.replace('/dev/mapper/', '').replace('/dev/', '')
 
-  // Direct match
+  // Direct match (covers /dev/sda1 ↔ sda1, /dev/nvme0n1p1 ↔ nvme0n1p1)
   if (normalized === deviceName) {
     return true
   }
 
-  // LVM volumes use dashes instead of slashes
-  // e.g., ubuntu--vg-ubuntu--lv matches the device name
-  if (fsPath.includes(deviceName)) {
+  // LVM/device-mapper: e.g., /dev/mapper/ubuntu--vg-ubuntu--lv contains "ubuntu--lv"
+  if (fsPath.startsWith('/dev/mapper/') && fsPath.includes(deviceName)) {
     return true
   }
 
   return false
 }
+
+export function determineFileType(filename: string): 'image' | 'pdf' | 'text' | 'epub' | 'zim' | 'unknown' {
+  const ext = path.extname(filename).toLowerCase()
+  if (['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp'].includes(ext)) {
+    return 'image'
+  } else if (ext === '.pdf') {
+    return 'pdf'
+  } else if (['.txt', '.md', '.docx', '.rtf'].includes(ext)) {
+    return 'text'
+  } else if (ext === '.epub') {
+    return 'epub'
+  } else if (ext === '.zim') {
+    return 'zim'
+  } else {
+    return 'unknown'
+  }
+}
+
+/**
+ * Sanitize a filename by removing potentially dangerous characters.
+ * @param filename The original filename
+ * @returns The sanitized filename
+ */
+export function sanitizeFilename(filename: string): string {
+  return filename.replace(/[^a-zA-Z0-9._-]/g, '_')
+}

admin/app/utils/image_disk_preflight.ts

@@ -0,0 +1,83 @@
+import logger from '@adonisjs/core/services/logger'
+import type { ContainerRegistryService } from '#services/container_registry_service'
+import type { SystemService } from '#services/system_service'
+
+/**
+ * Shared pre-flight primitives for update flows (core app + installed apps).
+ * Kept framework-light (plain functions + injected service instances) so both
+ * {@link AutoUpdateService} and {@link AppAutoUpdateService} reuse one implementation.
+ */
+
+export type BlockerSeverity = 'skip' | 'failure'
+
+export interface Blocker {
+  reason: string
+  severity: BlockerSeverity
+}
+
+export interface PreflightResult {
+  ok: boolean
+  blockers: Blocker[]
+}
+
+/** Require free space >= imageSize * factor to cover decompressed layers + headroom. */
+export const DISK_SAFETY_FACTOR = 2
+/** Conservative fallback when the registry image size can't be determined. */
+export const MIN_FREE_BYTES = 5 * 1024 * 1024 * 1024 // 5 GiB
+
+/** Free bytes on the root filesystem (best-effort, falls back to max available). */
+export async function getFreeBytes(systemService: SystemService): Promise<number | null> {
+  const info = await systemService.getSystemInfo()
+  if (!info?.fsSize?.length) return null
+  const root = info.fsSize.find((f) => f.mount === '/')
+  if (root) return root.available
+  return Math.max(...info.fsSize.map((f) => f.available))
+}
+
+function gib(bytes: number): string {
+  return `${(bytes / 1024 / 1024 / 1024).toFixed(1)} GiB`
+}
+
+/**
+ * Returns a `failure` disk blocker if free space is insufficient for the given
+ * image reference, otherwise null. Mirrors the core update's behavior: estimate
+ * the image's compressed download size from the registry manifest, require
+ * `size * DISK_SAFETY_FACTOR` (or {@link MIN_FREE_BYTES} when size is unknown),
+ * and never block on transient lookup errors (returns null on failure).
+ *
+ * @param image Full image reference INCLUDING tag (e.g. "ollama/ollama:0.23.2").
+ */
+export async function checkImageDiskSpace(params: {
+  image: string
+  hostArch: string
+  containerRegistryService: ContainerRegistryService
+  systemService: SystemService
+}): Promise<Blocker | null> {
+  const { image, hostArch, containerRegistryService, systemService } = params
+  try {
+    const parsed = containerRegistryService.parseImageReference(image)
+    const imageSize = await containerRegistryService.getImageDownloadSize(
+      parsed,
+      parsed.tag,
+      hostArch
+    )
+    const required = imageSize !== null ? imageSize * DISK_SAFETY_FACTOR : MIN_FREE_BYTES
+
+    const free = await getFreeBytes(systemService)
+    if (free === null) {
+      logger.warn('[ImageDiskPreflight] Could not determine free disk space; skipping disk check')
+      return null
+    }
+
+    if (free < required) {
+      return {
+        reason: `Insufficient disk space: ${gib(free)} free, ${gib(required)} required`,
+        severity: 'failure',
+      }
+    }
+    return null
+  } catch (error) {
+    logger.warn(`[ImageDiskPreflight] Disk space check failed: ${error.message}`)
+    return null
+  }
+}

admin/app/utils/kb_ingest_decision.ts

@@ -0,0 +1,70 @@
+import type { KbIngestStateValue } from '../../types/kb_ingest_state.js'
+
+/**
+ * Decision returned by `decideScanAction` describing what scanAndSyncStorage
+ * should do for one file given its current state row (if any), whether Qdrant
+ * already has chunks for it, and the global ingest policy.
+ *
+ * - `skip` — file is in a settled state (already indexed, deliberately not
+ *   indexed, or in a manual-recovery state); no auto-dispatch.
+ * - `dispatch` — file needs to be (re-)embedded; an EmbedFileJob should be
+ *   dispatched. `createStateRow` indicates whether a new state row needs to
+ *   be created before dispatch (i.e. first time the scanner has seen it).
+ * - `backfill_indexed` — Qdrant has chunks but no state row exists yet
+ *   (pre-RFC install, or new admin instance pointed at an existing Qdrant
+ *   volume). Create a row in `indexed` state without re-embedding.
+ * - `create_pending` — Manual mode: record that we've seen the file but
+ *   don't dispatch. Frontend surfaces a per-card "Index" affordance.
+ */
+export type ScanAction =
+  | { kind: 'skip' }
+  | { kind: 'dispatch'; createStateRow: boolean }
+  | { kind: 'backfill_indexed' }
+  | { kind: 'create_pending' }
+
+export interface KbIngestStateRow {
+  state: KbIngestStateValue
+}
+
+/**
+ * Global auto-index policy stored at KV `rag.defaultIngestPolicy`. Unset is
+ * treated as `Always` so existing installs keep their current behavior until
+ * the user opts into Manual mode through the KB panel.
+ */
+export type IngestPolicy = 'Always' | 'Manual'
+
+/**
+ * Decide what scanAndSyncStorage should do for a single embeddable file.
+ *
+ * Replaces the earlier `!sourcesInQdrant.has(filePath)` binary check, which
+ * couldn't tell a fully-indexed file from a stalled mid-batch ingestion, and
+ * couldn't honor a user's "browse only" choice. The state row is now the
+ * authoritative answer; Qdrant chunk presence is corroborating evidence.
+ */
+export function decideScanAction(
+  stateRow: KbIngestStateRow | null,
+  hasChunksInQdrant: boolean,
+  policy: IngestPolicy = 'Always'
+): ScanAction {
+  if (!stateRow) {
+    if (hasChunksInQdrant) return { kind: 'backfill_indexed' }
+    return policy === 'Always'
+      ? { kind: 'dispatch', createStateRow: true }
+      : { kind: 'create_pending' }
+  }
+
+  switch (stateRow.state) {
+    case 'indexed':
+      return hasChunksInQdrant ? { kind: 'skip' } : { kind: 'dispatch', createStateRow: false }
+    case 'pending_decision':
+      // Manual mode: file is waiting for the user to opt in via per-card Index.
+      // Always mode: treat as "user-equivalent of auto-index" and dispatch.
+      return policy === 'Always'
+        ? { kind: 'dispatch', createStateRow: false }
+        : { kind: 'skip' }
+    case 'browse_only':
+    case 'failed':
+    case 'stalled':
+      return { kind: 'skip' }
+  }
+}

admin/app/utils/kb_job_health.ts

@@ -0,0 +1,50 @@
+/**
+ * Visual status assigned to an in-flight (or stuck) embedding job, used to
+ * pick the colored status pill in the KB Processing Queue. See RFC #883 §5.
+ *
+ * - `waiting`  — queued, no batch has started yet
+ * - `healthy`  — last batch < 2 minutes ago
+ * - `slow`     — last batch 2-5 minutes ago (CPU-paced multi-batch ingestion
+ *                falls into this band; not necessarily a problem)
+ * - `stalled`  — last batch > 5 minutes ago (likely a real problem)
+ * - `failed`   — job recorded a failed status
+ */
+export type JobHealthStatus = 'waiting' | 'healthy' | 'slow' | 'stalled' | 'failed'
+
+export interface JobHealthInput {
+  /** BullMQ job.data.status — set by EmbedFileJob.handle on transitions. */
+  status: string
+  /** 0-100. 0 means no work observed yet on this job-row. */
+  progress: number
+  /** ms epoch of the last completed batch. Multi-batch ZIMs update this on
+   * every continuation; single-batch jobs leave it unset until completion. */
+  lastBatchAt?: number
+  /** ms epoch of the first batch start. Used as a fallback "last activity"
+   * signal for jobs that haven't yet completed their first batch. */
+  startedAt?: number
+  /** Current ms epoch. Injected for testability. */
+  now: number
+}
+
+const SLOW_THRESHOLD_MS = 2 * 60 * 1000
+const STALLED_THRESHOLD_MS = 5 * 60 * 1000
+
+export function computeJobHealth(input: JobHealthInput): JobHealthStatus {
+  if (input.status === 'failed') return 'failed'
+
+  // No progress recorded and no activity timestamps — job is still queued.
+  if (
+    input.progress === 0 &&
+    input.lastBatchAt === undefined &&
+    input.startedAt === undefined
+  ) {
+    return 'waiting'
+  }
+
+  const lastActivity = input.lastBatchAt ?? input.startedAt ?? input.now
+  const stalenessMs = input.now - lastActivity
+
+  if (stalenessMs > STALLED_THRESHOLD_MS) return 'stalled'
+  if (stalenessMs > SLOW_THRESHOLD_MS) return 'slow'
+  return 'healthy'
+}

admin/app/utils/kb_ratio_lookup.ts

@@ -0,0 +1,109 @@
+export interface RatioRow {
+  pattern: string
+  chunks_per_mb: number
+}
+
+/**
+ * Bytes of on-disk storage one embedded chunk consumes inside Qdrant.
+ *
+ * Rough composition for our pipeline:
+ *   - vector: 768 dims × float32 = 3,072 B
+ *   - chunk text payload: ~3,000 B (target 1,500 tokens × 2 chars/token)
+ *   - source/metadata payload + Qdrant indexes: ~2,000 B
+ *
+ * Used for surfacing pre-ingest disk-cost estimates; the actual figure
+ * varies with collection params and will be replaced by self-calibration
+ * (RFC #883 Phase 4) once we have real measurements.
+ */
+export const BYTES_PER_CHUNK_ON_DISK = 8_000
+
+export interface BatchEstimateInput {
+  filename: string
+  sizeBytes: number
+}
+
+export interface BatchEstimate {
+  totalChunks: number
+  totalBytes: number
+  hasUnknown: boolean
+}
+
+/**
+ * Aggregate an embedding-disk-cost estimate across a batch of files (curated
+ * tier add, multi-upload, sync preview, etc). `hasUnknown` is true when at
+ * least one file did not match any registry row — the totals only include
+ * matched files, so callers should annotate "estimate excludes unknown files"
+ * when surfacing the figure.
+ */
+export function estimateBatch(
+  files: BatchEstimateInput[],
+  rows: RatioRow[]
+): BatchEstimate {
+  let totalChunks = 0
+  let hasUnknown = false
+  for (const f of files) {
+    const chunks = estimateChunkCount(f.filename, f.sizeBytes, rows)
+    if (chunks === null) {
+      hasUnknown = true
+    } else {
+      totalChunks += chunks
+    }
+  }
+  return {
+    totalChunks,
+    totalBytes: totalChunks * BYTES_PER_CHUNK_ON_DISK,
+    hasUnknown,
+  }
+}
+
+/**
+ * Pick the chunks_per_mb estimate for a filename by longest-prefix match.
+ *
+ * Patterns are filename prefixes (`devdocs_`, `wikipedia_en_simple_`, ...).
+ * The longest matching prefix wins, so a specific entry (`wikipedia_en_simple_`)
+ * overrides the broader fallback (`wikipedia_en_`). An empty-string pattern in
+ * the registry serves as a catch-all that matches every input.
+ *
+ * Returns `null` if no row matches and no empty-string fallback is present —
+ * caller decides whether to surface "unknown" or use its own default.
+ *
+ * `ignoreCatchAll` excludes the empty-string catch-all row from matching, so a
+ * filename that only the fallback would have matched returns `null` instead.
+ * Callers that need a *specific* expectation (e.g. the partial_stall warning,
+ * which must not fire on atypical ZIMs the registry can't actually characterize)
+ * pass this; rough aggregate estimates (disk cost) leave it off. See #913.
+ */
+export function findChunksPerMb(
+  filename: string,
+  rows: RatioRow[],
+  opts: { ignoreCatchAll?: boolean } = {}
+): number | null {
+  let best: RatioRow | null = null
+  for (const row of rows) {
+    if (opts.ignoreCatchAll && row.pattern === '') continue
+    if (!filename.startsWith(row.pattern)) continue
+    if (best === null || row.pattern.length > best.pattern.length) {
+      best = row
+    }
+  }
+  return best === null ? null : best.chunks_per_mb
+}
+
+/**
+ * Estimate the number of embedding chunks a ZIM-style file will produce given
+ * its size on disk in bytes. Returns `null` when the registry has nothing to
+ * match against. Caller is responsible for converting the estimate into either
+ * a disk-footprint estimate (chunks × bytes-per-chunk in Qdrant) or a time
+ * estimate (chunks ÷ chunks-per-minute-on-this-hardware).
+ */
+export function estimateChunkCount(
+  filename: string,
+  fileSizeBytes: number,
+  rows: RatioRow[],
+  opts: { ignoreCatchAll?: boolean } = {}
+): number | null {
+  const ratio = findChunksPerMb(filename, rows, opts)
+  if (ratio === null) return null
+  const megabytes = fileSizeBytes / (1024 * 1024)
+  return Math.round(ratio * megabytes)
+}

admin/app/utils/kb_warning_decision.ts

@@ -0,0 +1,70 @@
+/**
+ * Conditional warnings surfaced on Stored Files rows in the KB panel.
+ * See RFC #883 §6 — these warnings appear ONLY when their triggering condition
+ * is met, never on healthy files, to keep the panel silent in the common case.
+ *
+ * - `zero_chunks`   — a non-trivial file produced 0 embedding chunks. Common
+ *                     cause: video-only or image-only ZIMs that the pipeline
+ *                     completes "successfully" with no extractable text.
+ *                     AI Assistant cannot reference this content.
+ * - `partial_stall` — the file has embedded chunks but well below the count
+ *                     expected from the ratio registry. Likely a mid-batch
+ *                     stall (which the binary "any chunks ⇒ embedded" check
+ *                     used to mask). Surfaces a Retry affordance.
+ */
+import type { FileWarning } from '../../types/rag.js'
+
+export type { FileWarning }
+
+/** Files smaller than this are too small to flag as suspicious zero-chunk
+ *  cases — a 5 KB upload that produces 0 chunks is much more likely to be a
+ *  legitimate edge case (placeholder file) than the gigabyte-scale video ZIM
+ *  problem this warning targets. */
+export const ZERO_CHUNKS_MIN_SIZE_BYTES = 100 * 1024 * 1024 // 100 MB
+
+/** Fraction of expected chunks below which we consider a file partially
+ *  stalled. 0.5 (50%) matches the threshold described in RFC #883 §6 Warning B. */
+export const PARTIAL_STALL_RATIO_THRESHOLD = 0.5
+
+export interface WarningInputs {
+  /** Source file size on disk in bytes. */
+  fileSizeBytes: number
+  /** Distinct chunks present in Qdrant for this source. */
+  chunksInQdrant: number
+  /** Best estimate of chunks the file should produce, from the ratio
+   *  registry. `null` when no registry pattern matches and no fallback is
+   *  configured — Warning B is suppressed in that case (we'd rather be silent
+   *  than wrong). */
+  expectedChunks: number | null
+}
+
+export function decideWarnings(inputs: WarningInputs): FileWarning[] {
+  const warnings: FileWarning[] = []
+
+  // Warning A: file is large but produced nothing. Almost always a video-only
+  // or image-only ZIM; AI Assistant literally cannot reference this content.
+  if (
+    inputs.chunksInQdrant === 0 &&
+    inputs.fileSizeBytes > ZERO_CHUNKS_MIN_SIZE_BYTES
+  ) {
+    warnings.push({ kind: 'zero_chunks', fileSizeBytes: inputs.fileSizeBytes })
+  }
+
+  // Warning B: chunks present but far below expectation. Suppresses when we
+  // have no expectation (registry miss) since the comparison would be
+  // meaningless and we'd rather under-warn than mislead.
+  if (
+    inputs.expectedChunks !== null &&
+    inputs.expectedChunks > 0 &&
+    inputs.chunksInQdrant > 0 &&
+    inputs.chunksInQdrant < inputs.expectedChunks * PARTIAL_STALL_RATIO_THRESHOLD
+  ) {
+    warnings.push({
+      kind: 'partial_stall',
+      chunksEmbedded: inputs.chunksInQdrant,
+      chunksExpected: inputs.expectedChunks,
+    })
+  }
+
+  return warnings
+}

admin/app/utils/misc.ts

@@ -3,3 +3,23 @@ export function formatSpeed(bytesPerSecond: number): string {
   if (bytesPerSecond < 1024 * 1024) return `${(bytesPerSecond / 1024).toFixed(1)} KB/s`
   return `${(bytesPerSecond / (1024 * 1024)).toFixed(1)} MB/s`
 }
+
+export function toTitleCase(str: string): string {
+  return str
+    .toLowerCase()
+    .split(' ')
+    .map((word) => word.charAt(0).toUpperCase() + word.slice(1))
+    .join(' ')
+}
+
+export function parseBoolean(value: any): boolean {
+  if (typeof value === 'boolean') return value
+  if (typeof value === 'string') {
+    const lower = value.toLowerCase()
+    return lower === 'true' || lower === '1'
+  }
+  if (typeof value === 'number') {
+    return value === 1
+  }
+  return false
+}

admin/app/utils/superseded_resource.ts

@@ -0,0 +1,72 @@
+import { resolve, sep } from 'node:path'
+
+/**
+ * Decides whether a curated resource's PREVIOUSLY-installed file should be
+ * deleted now that a newer version has been downloaded (issue #634 — old map
+ * and ZIM versions accumulated on disk indefinitely because only Wikipedia had
+ * version cleanup).
+ *
+ * This is intentionally a pure function so every safety rail is unit-testable
+ * without touching the DB or filesystem. The caller looks up the prior
+ * `InstalledResource` row, records the new version, then asks this whether the
+ * old file is safe to remove.
+ *
+ * Safety rails (a "delete" decision requires ALL of these):
+ *  - There was a prior install for this exact resource_id (`existing` non-null).
+ *    Untracked / sideloaded files have no row and are therefore never touched.
+ *  - The old file path actually differs from the new one (a genuine version
+ *    swap, not a re-download of the same file).
+ *  - The new file is confirmed present on disk — we never remove the old copy
+ *    before the replacement is verified.
+ *  - The new version is strictly newer than the recorded one, so a re-install
+ *    or downgrade can't wipe a newer file.
+ *  - The old path resolves to within the resource's storage directory, so a
+ *    malformed DB value can't direct a delete outside the content store.
+ */
+
+export interface SupersededInputs {
+  /** Prior InstalledResource row for this resource_id, or null on first install. */
+  existing: { file_path: string; version: string } | null
+  /** Absolute path of the newly downloaded file. */
+  newFilePath: string
+  /** Version of the newly downloaded file (e.g. "2026-05"). */
+  newVersion: string
+  /** Whether the new file is confirmed present on disk. */
+  newFileExists: boolean
+  /** Absolute storage directory the old file must live under to be eligible. */
+  storageBaseDir: string
+}
+
+export type SupersededReason =
+  | 'first_install'
+  | 'same_file'
+  | 'new_file_missing'
+  | 'not_newer'
+  | 'outside_storage'
+  | 'superseded'
+
+export interface SupersededDecision {
+  delete: boolean
+  /** Resolved old path to delete — set only when `delete` is true. */
+  path?: string
+  reason: SupersededReason
+}
+
+export function decideSupersededDeletion(inputs: SupersededInputs): SupersededDecision {
+  const { existing, newFilePath, newVersion, newFileExists, storageBaseDir } = inputs
+
+  if (!existing) return { delete: false, reason: 'first_install' }
+  if (existing.file_path === newFilePath) return { delete: false, reason: 'same_file' }
+  if (!newFileExists) return { delete: false, reason: 'new_file_missing' }
+  // Versions are zero-padded date strings (YYYY-MM / YYYY-MM-DD), so a lexical
+  // compare orders them correctly. Require strictly newer.
+  if (!(newVersion > existing.version)) return { delete: false, reason: 'not_newer' }
+
+  const resolvedOld = resolve(existing.file_path)
+  const base = resolve(storageBaseDir)
+  if (resolvedOld !== base && !resolvedOld.startsWith(base + sep)) {
+    return { delete: false, reason: 'outside_storage' }
+  }
+
+  return { delete: true, path: resolvedOld, reason: 'superseded' }
+}

admin/app/utils/update_window.ts

@@ -0,0 +1,35 @@
+import { DateTime } from 'luxon'
+
+/**
+ * Shared update-window helpers used by both the core auto-update
+ * ({@link AutoUpdateService}) and the per-app auto-update ({@link AppAutoUpdateService}).
+ *
+ * The window is interpreted in the container's local time (set via the TZ env var).
+ * Windows that wrap past midnight (start > end, e.g. 22:00-02:00) are supported.
+ */
+
+/** Parse an "HH:MM" 24-hour string into minutes-since-midnight, or null if malformed. */
+export function parseWindowMinutes(hhmm: string): number | null {
+  const match = /^([01]\d|2[0-3]):([0-5]\d)$/.exec(hhmm)
+  if (!match) return null
+  return Number(match[1]) * 60 + Number(match[2])
+}
+
+/** Whether `now` falls inside the [windowStart, windowEnd) window (handles midnight wrap). */
+export function isWithinWindow(
+  windowStart: string,
+  windowEnd: string,
+  now: DateTime = DateTime.now()
+): boolean {
+  const start = parseWindowMinutes(windowStart)
+  const end = parseWindowMinutes(windowEnd)
+  if (start === null || end === null) return false
+
+  const current = now.hour * 60 + now.minute
+  if (start === end) return false // zero-length window
+  if (start < end) {
+    return current >= start && current < end
+  }
+  // Wraps midnight
+  return current >= start || current < end
+}

admin/app/utils/version.ts

@@ -0,0 +1,49 @@
+/**
+ * Compare two semantic version strings to determine if the first is newer than the second.
+ * @param version1 - The version to check (e.g., "1.25.0")
+ * @param version2 - The current version (e.g., "1.24.0")
+ * @returns true if version1 is newer than version2
+ */
+export function isNewerVersion(version1: string, version2: string, includePreReleases = false): boolean {
+  const normalize = (v: string) => v.replace(/^v/, '')
+  const [base1, pre1] = normalize(version1).split('-')
+  const [base2, pre2] = normalize(version2).split('-')
+
+  // If pre-releases are not included and version1 is a pre-release, don't consider it newer
+  if (!includePreReleases && pre1) {
+    return false
+  }
+
+  const v1Parts = base1.split('.').map((p) => parseInt(p, 10) || 0)
+  const v2Parts = base2.split('.').map((p) => parseInt(p, 10) || 0)
+
+  const maxLen = Math.max(v1Parts.length, v2Parts.length)
+  for (let i = 0; i < maxLen; i++) {
+    const a = v1Parts[i] || 0
+    const b = v2Parts[i] || 0
+    if (a > b) return true
+    if (a < b) return false
+  }
+
+  // Base versions equal — GA > RC, RC.n+1 > RC.n
+  if (!pre1 && pre2) return true // v1 is GA, v2 is RC → v1 is newer
+  if (pre1 && !pre2) return false // v1 is RC, v2 is GA → v2 is newer
+  if (!pre1 && !pre2) return false // both GA, equal
+
+  // Both prerelease: compare numeric suffix (e.g. "rc.2" vs "rc.1")
+  const pre1Num = parseInt(pre1.split('.')[1], 10) || 0
+  const pre2Num = parseInt(pre2.split('.')[1], 10) || 0
+  return pre1Num > pre2Num
+}
+
+/**
+ * Parse the major version number from a tag string.
+ * Strips the 'v' prefix if present.
+ * @param tag - Version tag (e.g., "v3.8.1", "10.19.4")
+ * @returns The major version number
+ */
+export function parseMajorVersion(tag: string): number {
+  const normalized = tag.replace(/^v/, '')
+  const major = parseInt(normalized.split('.')[0], 10)
+  return isNaN(major) ? 0 : major
+}

admin/app/utils/zim_filename.ts

@@ -0,0 +1,26 @@
+/**
+ * Strip the trailing `_YYYY-MM(-DD).zim` date suffix from a Kiwix-style ZIM
+ * filename so different release dates of the same variant share a stem
+ * (e.g., `wikipedia_en_all_nopic`) while distinct corpora keep distinct stems
+ * (`wikipedia_en_simple_all_nopic`, `wikipedia_en_medicine_nopic`, etc.).
+ */
+export function zimFilenameStem(name: string): string {
+  return name.replace(/_\d{4}-\d{2}(?:-\d{2})?\.zim$/i, '')
+}
+
+/**
+ * Of the existing files, return only those that are prior-version replacements
+ * of `currentFilename` — same Wikipedia variant stem, different release. Used
+ * by the post-download cleanup to avoid deleting unrelated Wikipedia corpora
+ * the user has installed independently (issue #884).
+ */
+export function findReplacedWikipediaFiles(
+  currentFilename: string,
+  existingNames: string[]
+): string[] {
+  const currentStem = zimFilenameStem(currentFilename)
+  return existingNames.filter(
+    (n) =>
+      n.startsWith('wikipedia_en_') && n !== currentFilename && zimFilenameStem(n) === currentStem
+  )
+}

admin/app/validators/benchmark.ts

@@ -0,0 +1,13 @@
+import vine from '@vinejs/vine'
+
+export const runBenchmarkValidator = vine.compile(
+  vine.object({
+    benchmark_type: vine.enum(['full', 'system', 'ai']).optional(),
+  })
+)
+
+export const submitBenchmarkValidator = vine.compile(
+  vine.object({
+    benchmark_id: vine.string().optional(),
+  })
+)