473 changed files with 6238 additions and 73891 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -2,7 +2,4 @@
 .env.*
 .git
 node_modules
-*.log
+*.log
 admin/storage
 admin/node_modules
 admin/build
--- a/.github/ISSUE_TEMPLATE/bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/bug_report.yml
@ -1,194 +0,0 @@
 name: Bug Report
 description: Report a bug or issue with Project NOMAD
 title: "[Bug]: "
 labels: ["bug", "needs-triage"]
 body:
  - type: markdown
    attributes:
      value: |
        Thanks for taking the time to report a bug! Please fill out the information below to help us diagnose and fix the issue.
        **Before submitting:**
        - Search existing issues to avoid duplicates
        - Ensure you're running the latest version of NOMAD
        - Redact any personal or sensitive information from logs/configs
        - Please don't submit issues related to running NOMAD on Unraid or another NAS - we don't have plans to support these kinds of platforms at this time
  - type: dropdown
    id: issue-category
    attributes:
      label: Issue Category
      description: What area is this issue related to?
      options:
        - Installation/Setup
        - AI Assistant (Ollama)
        - Knowledge Base/RAG (Document Upload)
        - Docker/Container Issues
        - GPU Configuration
        - Content Downloads (ZIM, Maps, Collections)
        - Service Management (Start/Stop/Update)
        - System Performance/Resources
        - UI/Frontend Issue
        - Other
    validations:
      required: true
  - type: textarea
    id: description
    attributes:
      label: Bug Description
      description: Provide a clear and concise description of what the bug is
      placeholder: What happened? What did you expect to happen?
    validations:
      required: true
  - type: textarea
    id: reproduction
    attributes:
      label: Steps to Reproduce
      description: How can we reproduce this issue?
      placeholder: |
        1. Go to '...'
        2. Click on '...'
        3. See error
    validations:
      required: true
  - type: textarea
    id: expected-behavior
    attributes:
      label: Expected Behavior
      description: What did you expect to happen?
      placeholder: Describe the expected outcome
    validations:
      required: true
  - type: textarea
    id: actual-behavior
    attributes:
      label: Actual Behavior
      description: What actually happened?
      placeholder: Describe what actually occurred, including any error messages
    validations:
      required: true
  - type: input
    id: nomad-version
    attributes:
      label: NOMAD Version
      description: What version of NOMAD are you running? (Check Settings > Update or run `docker ps` and check nomad_admin image tag)
      placeholder: "e.g., 1.29.0"
    validations:
      required: true
  - type: dropdown
    id: os
    attributes:
      label: Operating System
      description: What OS are you running NOMAD on?
      options:
        - Ubuntu 26.04 (Resolute Raccoon)
        - Ubuntu 24.04
        - Ubuntu 22.04
        - Ubuntu 20.04
        - Debian 13 (Trixie)
        - Debian 12 (Bookworm)
        - Debian 11 (Bullseye)
        - Other Debian-based
        - Other (not yet officially supported)
    validations:
      required: true
  - type: input
    id: docker-version
    attributes:
      label: Docker Version
      description: What version of Docker are you running? (`docker --version`)
      placeholder: "e.g., Docker version 24.0.7"
  - type: dropdown
    id: gpu-present
    attributes:
      label: Do you have a dedicated GPU?
      options:
        - "Yes"
        - "No"
        - "Not sure"
    validations:
      required: true
  - type: input
    id: gpu-model
    attributes:
      label: GPU Model (if applicable)
      description: What GPU model do you have? (Check Settings > System or run `nvidia-smi` if NVIDIA GPU)
      placeholder: "e.g., NVIDIA GeForce RTX 3060"
  - type: textarea
    id: system-specs
    attributes:
      label: System Specifications
      description: Provide relevant system specs (CPU, RAM, available disk space)
      placeholder: |
        CPU: 
        RAM: 
        Available Disk Space: 
        GPU (if any): 
  - type: textarea
    id: service-status
    attributes:
      label: Service Status (if relevant)
      description: If this is a service-related issue, what's the status of relevant services? (Check Settings > Apps or run `docker ps`)
      placeholder: |
        Paste output from `docker ps` or describe service states from the UI
  - type: textarea
    id: logs
    attributes:
      label: Relevant Logs
      description: |
        Include any relevant logs or error messages. **Please redact any personal/sensitive information.**
        Useful commands for collecting logs:
        - NOMAD management app: `docker logs nomad_admin`
        - Ollama: `docker logs nomad_ollama`
        - Qdrant: `docker logs nomad_qdrant`
        - Specific service: `docker logs nomad_<service-name>`
      placeholder: Paste relevant log output here
      render: shell
  - type: textarea
    id: browser-console
    attributes:
      label: Browser Console Errors (if UI issue)
      description: If this is a UI issue, include any errors from your browser's developer console (F12)
      placeholder: Paste browser console errors here
      render: javascript
  - type: textarea
    id: screenshots
    attributes:
      label: Screenshots
      description: If applicable, add screenshots to help explain your problem (drag and drop images here)
  - type: textarea
    id: additional-context
    attributes:
      label: Additional Context
      description: Add any other context about the problem here (network setup, custom configurations, recent changes, etc.)
  - type: checkboxes
    id: terms
    attributes:
      label: Pre-submission Checklist
      description: Please confirm the following before submitting
      options:
        - label: I have searched for existing issues that might be related to this bug
          required: true
        - label: I am running the latest version of Project NOMAD (or have noted my version above)
          required: true
        - label: I have redacted any personal or sensitive information from logs and screenshots
          required: true
        - label: This issue is NOT related to running NOMAD on an unsupported/non-Debian-based OS
          required: false
--- a/.github/ISSUE_TEMPLATE/config.yml
+++ b/.github/ISSUE_TEMPLATE/config.yml
@ -1,17 +0,0 @@
 blank_issues_enabled: false
 contact_links:
  - name: 💬 Discord Community
    url: https://discord.com/invite/crosstalksolutions
    about: Join our Discord community for general questions, support, and discussions
  - name: 📖 Documentation
    url: https://projectnomad.us
    about: Check the official documentation and guides
  - name: 🏆 Community Leaderboard
    url: https://benchmark.projectnomad.us
    about: View the NOMAD benchmark leaderboard
  - name: 🤝 Contributing Guide
    url: https://github.com/Crosstalk-Solutions/project-nomad/blob/main/CONTRIBUTING.md
    about: Learn how to contribute to Project NOMAD
  - name: 📅 Roadmap
    url: https://roadmap.projectnomad.us
    about: See our public roadmap, vote on features, and suggest new ones
--- a/.github/ISSUE_TEMPLATE/feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/feature_request.yml
@ -1,150 +0,0 @@
 name: Feature Request
 description: Suggest a new feature or enhancement for Project NOMAD
 title: "[Feature]: "
 labels: ["enhancement", "needs-discussion"]
 body:
  - type: markdown
    attributes:
      value: |
        Thanks for your interest in improving Project NOMAD! Before you submit a feature request, consider checking our [roadmap](https://roadmap.projectnomad.us) to see if it's already planned or in progress. You're welcome to suggest new ideas there if you don't plan on opening PRs yourself.
        **Please note:** Feature requests are not guaranteed to be implemented. All requests are evaluated based on alignment with the project's goals, feasibility, and community demand.
        **Before submitting:**
        - Search existing feature requests and our [roadmap](https://roadmap.projectnomad.us) to avoid duplicates
        - Consider if this aligns with NOMAD's mission: offline-first knowledge and education
        - Consider the technical feasibility of the feature: NOMAD is designed to be containerized and run on a wide range of hardware, so features that require heavy resources (aside from GPU-intensive tasks) or complex host configurations may be less likely to be implemented
        - Consider the scope of the feature: Small, focused enhancements that can be implemented incrementally are more likely to be implemented than large, broad features that would require significant development effort or have an unclear path forward
        - If you're able to contribute code, testing, or documentation, that significantly increases the chances of your feature being implemented
  - type: dropdown
    id: feature-category
    attributes:
      label: Feature Category
      description: What area does this feature relate to?
      options:
        - New Service/Tool Integration
        - AI Assistant Enhancement
        - Knowledge Base/RAG Improvement
        - Content Management (ZIM, Maps, Collections)
        - UI/UX Improvement
        - System Management
        - Performance Optimization
        - Documentation
        - Security
        - Other
    validations:
      required: true
  - type: textarea
    id: problem
    attributes:
      label: Problem Statement
      description: What problem does this feature solve? Is your feature request related to a pain point?
      placeholder: I find it frustrating when... / It would be helpful if... / Users struggle with...
    validations:
      required: true
  - type: textarea
    id: solution
    attributes:
      label: Proposed Solution
      description: Describe the feature or enhancement you'd like to see
      placeholder: Add a feature that... / Change the behavior to... / Integrate with...
    validations:
      required: true
  - type: textarea
    id: alternatives
    attributes:
      label: Alternative Solutions
      description: Have you considered any alternative solutions or workarounds?
      placeholder: I've tried... / Another approach could be... / A workaround is...
  - type: textarea
    id: use-case
    attributes:
      label: Use Case
      description: Describe a specific scenario where this feature would be valuable
      placeholder: |
        As a [type of user], when I [do something], I want to [accomplish something] so that [benefit].
        Example: Because I have a dedicated GPU, I want to be able to see in the UI if GPU support is enabled so that I can optimize performance and troubleshoot issues more easily.
  - type: dropdown
    id: user-type
    attributes:
      label: Who would benefit from this feature?
      description: What type of users would find this most valuable?
      multiple: true
      options:
        - Individual/Home Users
        - Families
        - Teachers/Educators
        - Students
        - Survivalists/Preppers
        - Developers/Contributors
        - Organizations
        - All Users
    validations:
      required: true
  - type: dropdown
    id: priority
    attributes:
      label: How important is this feature to you?
      options:
        - Critical - Blocking my use of NOMAD
        - High - Would significantly improve my experience
        - Medium - Would be nice to have
        - Low - Minor convenience
    validations:
      required: true
  - type: textarea
    id: implementation-ideas
    attributes:
      label: Implementation Ideas (Optional)
      description: If you have technical suggestions for how this could be implemented, share them here
      placeholder: This could potentially use... / It might integrate with... / A possible approach is...
  - type: textarea
    id: examples
    attributes:
      label: Examples or References
      description: Are there similar features in other applications? Include links, screenshots, or descriptions
      placeholder: Similar to how [app name] does... / See this example at [URL]
  - type: dropdown
    id: willing-to-contribute
    attributes:
      label: Would you be willing to help implement this?
      description: Contributing increases the likelihood of implementation
      options:
        - "Yes - I can write the code"
        - "Yes - I can help test"
        - "Yes - I can help with documentation"
        - "Maybe - with guidance"
        - "No - I don't have the skills/time"
    validations:
      required: true
  - type: textarea
    id: additional-context
    attributes:
      label: Additional Context
      description: Add any other context, mockups, diagrams, or information about the feature request
  - type: checkboxes
    id: checklist
    attributes:
      label: Pre-submission Checklist
      description: Please confirm the following before submitting
      options:
        - label: I have searched for existing feature requests that might be similar
          required: true
        - label: This feature aligns with NOMAD's mission of offline-first knowledge and education
          required: true
        - label: I understand that feature requests are not guaranteed to be implemented
          required: true
--- a/.github/dependabot.yaml
+++ b/.github/dependabot.yaml
@ -1,7 +0,0 @@
 version: 2
 updates:
  - package-ecosystem: "npm"
    directory: "/admin"
    schedule:
      interval: "weekly"
    target-branch: "rc"
--- a/.github/scripts/finalize-release-notes.sh
+++ b/.github/scripts/finalize-release-notes.sh
@ -1,133 +0,0 @@
 #!/usr/bin/env bash
 #
 # finalize-release-notes.sh
 #
 # Stamps the "## Unreleased" section in a release-notes file with a version
 # and date, and extracts the section content for use in GitHub releases / email.
 # Also includes all commits since the last release for complete transparency.
 #
 # Usage:  finalize-release-notes.sh <version> <file-path>
 #
 # Exit codes:
 #   0 - Success: section stamped and extracted
 #   1 - No "## Unreleased" section found (skip gracefully)
 #   2 - Unreleased section exists but is empty (skip gracefully)
 set -euo pipefail
 VERSION="${1:?Usage: finalize-release-notes.sh <version> <file-path>}"
 FILE="${2:?Usage: finalize-release-notes.sh <version> <file-path>}"
 if [[ ! -f "$FILE" ]]; then
  echo "Error: File not found: $FILE" >&2
  exit 1
 fi
 # Find the line number of the ## Unreleased header (case-insensitive)
 HEADER_LINE=$(grep -inm1 '^## unreleased' "$FILE" | cut -d: -f1)
 if [[ -z "$HEADER_LINE" ]]; then
  echo "No '## Unreleased' section found. Skipping."
  exit 1
 fi
 TOTAL_LINES=$(wc -l < "$FILE")
 # Find the next section header (## Version ...) or --- separator after the Unreleased header
 NEXT_SECTION_LINE=""
 if [[ $HEADER_LINE -lt $TOTAL_LINES ]]; then
  NEXT_SECTION_LINE=$(tail -n +"$((HEADER_LINE + 1))" "$FILE" \
    | grep -nm1 '^## \|^---$' \
    | cut -d: -f1)
 fi
 if [[ -n "$NEXT_SECTION_LINE" ]]; then
  # NEXT_SECTION_LINE is relative to HEADER_LINE+1, convert to absolute
  END_LINE=$((HEADER_LINE + NEXT_SECTION_LINE - 1))
 else
  # Section runs to end of file
  END_LINE=$TOTAL_LINES
 fi
 # Extract content between header and next section (exclusive of both boundaries)
 CONTENT_START=$((HEADER_LINE + 1))
 CONTENT_END=$END_LINE
 # Extract the section body (between header line and the next boundary)
 SECTION_BODY=$(sed -n "${CONTENT_START},${CONTENT_END}p" "$FILE" | sed '/^$/N;/^\n$/d')
 # Check for actual content: strip blank lines and lines that are only markdown headers (###...)
 TRIMMED=$(echo "$SECTION_BODY" | sed '/^[[:space:]]*$/d')
 HAS_CONTENT=$(echo "$SECTION_BODY" | sed '/^[[:space:]]*$/d' | grep -v '^###' || true)
 if [[ -z "$TRIMMED" || -z "$HAS_CONTENT" ]]; then
  echo "Unreleased section is empty. Skipping."
  exit 2
 fi
 # Format the date as "Month Day, Year"
 DATE_STAMP=$(date +'%B %-d, %Y')
 NEW_HEADER="## Version ${VERSION} - ${DATE_STAMP}"
 # Build the replacement: swap the header line, keep everything else intact
 {
  # Lines before the Unreleased header
  if [[ $HEADER_LINE -gt 1 ]]; then
    head -n "$((HEADER_LINE - 1))" "$FILE"
  fi
  # New versioned header
  echo "$NEW_HEADER"
  # Content between header and next section
  sed -n "${CONTENT_START},${CONTENT_END}p" "$FILE"
  # Rest of the file after the section
  if [[ $END_LINE -lt $TOTAL_LINES ]]; then
    tail -n +"$((END_LINE + 1))" "$FILE"
  fi
 } > "${FILE}.tmp"
 mv "${FILE}.tmp" "$FILE"
 # Get commits since the last release
 LAST_TAG=$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null || echo "")
 COMMIT_LIST=""
 if [[ -n "$LAST_TAG" ]]; then
  echo "Fetching commits since ${LAST_TAG}..."
  # Get commits between last tag and HEAD, excluding merge commits and skip ci commits
  COMMIT_LIST=$(git log "${LAST_TAG}..HEAD" \
    --no-merges \
    --pretty=format:"- %s ([%h](https://github.com/${GITHUB_REPOSITORY}/commit/%H))" \
    --grep="\[skip ci\]" --invert-grep \
    || echo "")
 else
  echo "No previous tag found, fetching all commits..."
  COMMIT_LIST=$(git log \
    --no-merges \
    --pretty=format:"- %s ([%h](https://github.com/${GITHUB_REPOSITORY}/commit/%H))" \
    --grep="\[skip ci\]" --invert-grep \
    || echo "")
 fi
 # Write the extracted section content (for GitHub release body / future email)
 {
  echo "$NEW_HEADER"
  echo ""
  if [[ -n "$TRIMMED" ]]; then
    echo "$TRIMMED"
    echo ""
  fi
  # Add commit history if available
  if [[ -n "$COMMIT_LIST" ]]; then
    echo "---"
    echo ""
    echo "### 📝 All Changes"
    echo ""
    echo "$COMMIT_LIST"
  fi
 } > "${FILE}.section"
 echo "Finalized release notes for v${VERSION}"
 echo "  Updated: ${FILE}"
 echo "  Extracted: ${FILE}.section"
 exit 0
--- a/.github/workflows/build-admin-on-pr.yml
+++ b/.github/workflows/build-admin-on-pr.yml
@ -1,25 +0,0 @@
 name: Build Admin
 on: pull_request
 jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v6
      - name: Set up Node.js
        uses: actions/setup-node@v6
        with:
          node-version: '24'
          cache: 'npm'
      - name: Install dependencies
        run: npm ci
        working-directory: ./admin
      - name: Run build
        run: npm run build
        working-directory: ./admin
--- a/.github/workflows/build-disk-collector.yml
+++ b/.github/workflows/build-disk-collector.yml
@ -1,51 +0,0 @@
 name: Build Disk Collector Image
 on:
  workflow_dispatch:
    inputs:
      version:
        description: 'Semantic version to label the Docker image under (no "v" prefix, e.g. "1.2.3")'
        required: true
        type: string
      tag_latest:
        description: 'Also tag this image as :latest?'
        required: false
        type: boolean
        default: false
 jobs:
  check_authorization:
    name: Check authorization to publish new Docker image
    runs-on: ubuntu-latest
    outputs:
      isAuthorized: ${{ steps.check-auth.outputs.is_authorized }}
    steps:
      - name: check-auth
        id: check-auth
        run: echo "is_authorized=${{ contains(secrets.DEPLOYMENT_AUTHORIZED_USERS, github.triggering_actor) }}" >> $GITHUB_OUTPUT
  build:
    name: Build disk-collector image
    needs: check_authorization
    if: needs.check_authorization.outputs.isAuthorized == 'true'
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write
    steps:
      - name: Checkout code
        uses: actions/checkout@v6
      - name: Log in to GitHub Container Registry
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Build and push
        uses: docker/build-push-action@v7
        with:
          context: install/sidecar-disk-collector
          push: true
          tags: |
            ghcr.io/crosstalk-solutions/project-nomad-disk-collector:${{ inputs.version }}
            ghcr.io/crosstalk-solutions/project-nomad-disk-collector:v${{ inputs.version }}
            ${{ inputs.tag_latest && 'ghcr.io/crosstalk-solutions/project-nomad-disk-collector:latest' || '' }}
--- a/.github/workflows/build-primary-image.yml
+++ b/.github/workflows/build-primary-image.yml
@ -1,55 +0,0 @@
 name: Build Primary Docker Image
 on:
  workflow_dispatch:
    inputs:
      version:
        description: 'Semantic version to label the Docker image under (no "v" prefix, e.g. "1.2.3")'
        required: true
        type: string
      tag_latest:
        description: 'Also tag this image as :latest? (Keep false for RC and beta releases)'
        required: false
        type: boolean
        default: false
 jobs:
  check_authorization:
    name: Check authorization to publish new Docker image
    runs-on: ubuntu-latest
    outputs:
      isAuthorized: ${{ steps.check-auth.outputs.is_authorized }}
    steps:
      - name: check-auth
        id: check-auth
        run: echo "is_authorized=${{ contains(secrets.DEPLOYMENT_AUTHORIZED_USERS, github.triggering_actor) }}" >> $GITHUB_OUTPUT      
  build:
    name: Build Docker image
    needs: check_authorization
    if: needs.check_authorization.outputs.isAuthorized == 'true'
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write
    steps:
      - name: Checkout code
        uses: actions/checkout@v6
      - name: Log in to GitHub Container Registry
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Build and push
        uses: docker/build-push-action@v7
        with:
          push: true
          tags: |
            ghcr.io/crosstalk-solutions/project-nomad:${{ inputs.version }}
            ghcr.io/crosstalk-solutions/project-nomad:v${{ inputs.version }}
            ${{ inputs.tag_latest && 'ghcr.io/crosstalk-solutions/project-nomad:latest' || '' }}
          build-args: |
            VERSION=${{ inputs.version }}
            BUILD_DATE=${{ github.event.workflow_run.created_at }}
            VCS_REF=${{ github.sha }}
            CREATOR_PACKS_APP_KEY=${{ secrets.CREATOR_PACKS_APP_KEY }}
--- a/.github/workflows/build-sidecar-updater.yml
+++ b/.github/workflows/build-sidecar-updater.yml
@ -1,17 +1,12 @@
-name: Build Sidecar Updater Image
+name: Build Docker Image
 on:
  workflow_dispatch:
    inputs:
      version:
-        description: 'Semantic version to label the Docker image under (no "v" prefix, e.g. "1.2.3")'
+        description: 'Semantic version to label the Docker image under'
        required: true
        type: string
      tag_latest:
        description: 'Also tag this image as :latest?'
        required: false
        type: boolean
        default: false
 jobs:
  check_authorization:
@ -22,9 +17,9 @@ jobs:
    steps:
      - name: check-auth
        id: check-auth
-        run: echo "is_authorized=${{ contains(secrets.DEPLOYMENT_AUTHORIZED_USERS, github.triggering_actor) }}" >> $GITHUB_OUTPUT
+        run: echo "is_authorized=${{ contains(secrets.DEPLOYMENT_AUTHORIZED_USERS, github.triggering_actor) }}" >> $GITHUB_OUTPUT      
  build:
-    name: Build sidecar-updater image
+    name: Build Docker image
    needs: check_authorization
    if: needs.check_authorization.outputs.isAuthorized == 'true'
    runs-on: ubuntu-latest
@ -33,7 +28,7 @@ jobs:
      packages: write
    steps:
      - name: Checkout code
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
      - name: Log in to GitHub Container Registry
        uses: docker/login-action@v2
        with:
@ -43,9 +38,7 @@ jobs:
      - name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: install/sidecar-updater
          push: true
          tags: |
-            ghcr.io/crosstalk-solutions/project-nomad-sidecar-updater:${{ inputs.version }}
+            ghcr.io/crosstalk-solutions/project-nomad:${{ inputs.version }}
-            ghcr.io/crosstalk-solutions/project-nomad-sidecar-updater:v${{ inputs.version }}
+            ghcr.io/crosstalk-solutions/project-nomad:latest
            ${{ inputs.tag_latest && 'ghcr.io/crosstalk-solutions/project-nomad-sidecar-updater:latest' || '' }}
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@ -22,72 +22,16 @@ jobs:
      newVersion: ${{ steps.semver.outputs.new_release_version }}
    steps:
      - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          persist-credentials: false
      - name: Sync tags
        run: git fetch --tags --force
      - name: semantic-release
-        uses: cycjimmy/semantic-release-action@v6
+        uses: cycjimmy/semantic-release-action@v3
        id: semver
        env:
          GITHUB_TOKEN: ${{ secrets.COSMISTACKBOT_ACCESS_TOKEN }}
          GIT_AUTHOR_NAME: cosmistack-bot
          GIT_AUTHOR_EMAIL: dev@cosmistack.com
          GIT_COMMITTER_NAME: cosmistack-bot
-          GIT_COMMITTER_EMAIL: dev@cosmistack.com
+          GIT_COMMITTER_EMAIL: dev@cosmistack.com
      - name: Finalize release notes
        # Skip for pre-releases (versions containing a hyphen, e.g. 1.27.0-rc.1)
        if: |
          steps.semver.outputs.new_release_published == 'true' &&
          !contains(steps.semver.outputs.new_release_version, '-')
        id: finalize-notes
        env:
          GITHUB_REPOSITORY: ${{ github.repository }}
        run: |
          git pull origin main
          chmod +x .github/scripts/finalize-release-notes.sh
          EXIT_CODE=0
          .github/scripts/finalize-release-notes.sh \
            "${{ steps.semver.outputs.new_release_version }}" \
            admin/docs/release-notes.md || EXIT_CODE=$?
          if [[ "$EXIT_CODE" -eq 0 ]]; then
            echo "has_notes=true" >> $GITHUB_OUTPUT
          else
            echo "has_notes=false" >> $GITHUB_OUTPUT
          fi
      - name: Commit finalized release notes
        if: |
          steps.semver.outputs.new_release_published == 'true' &&
          steps.finalize-notes.outputs.has_notes == 'true' &&
          !contains(steps.semver.outputs.new_release_version, '-')
        run: |
          git config user.name "cosmistack-bot"
          git config user.email "dev@cosmistack.com"
          git remote set-url origin https://x-access-token:${{ secrets.COSMISTACKBOT_ACCESS_TOKEN }}@github.com/${{ github.repository }}.git
          git add admin/docs/release-notes.md
          git commit -m "docs(release): finalize v${{ steps.semver.outputs.new_release_version }} release notes [skip ci]"
          git push origin main
      - name: Update GitHub release body
        if: |
          steps.semver.outputs.new_release_published == 'true' &&
          steps.finalize-notes.outputs.has_notes == 'true' &&
          !contains(steps.semver.outputs.new_release_version, '-')
        env:
          GH_TOKEN: ${{ secrets.COSMISTACKBOT_ACCESS_TOKEN }}
        run: |
          gh release edit "v${{ steps.semver.outputs.new_release_version }}" \
            --notes-file admin/docs/release-notes.md.section
      # Future: Send release notes email
      # - name: Send release notes email
      #   if: steps.semver.outputs.new_release_published == 'true' && steps.finalize-notes.outputs.has_notes == 'true'
      #   run: |
      #     curl -X POST "https://api.projectnomad.us/api/v1/newsletter/release" \
      #       -H "Authorization: Bearer ${{ secrets.NOMAD_API_KEY }}" \
      #       -H "Content-Type: application/json" \
      #       -d "{\"version\": \"${{ steps.semver.outputs.new_release_version }}\", \"body\": $(cat admin/docs/release-notes.md.section | jq -Rs .)}"
--- a/.github/workflows/validate-collection-urls.yml
+++ b/.github/workflows/validate-collection-urls.yml
@ -1,82 +0,0 @@
 name: Validate Collection URLs
 on:
  push:
    paths:
      - 'collections/**.json'
  pull_request:
    paths:
      - 'collections/**.json'
 jobs:
  validate-urls:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - name: Extract and validate URLs
        run: |
          FAILED=0
          CHECKED=0
          FAILED_URLS=""
          # Recursively extract all non-null string URLs from every JSON file in collections/
          URLS=$(jq -r '.. | .url? | select(type == "string")' collections/*.json | sort -u)
          while IFS= read -r url; do
            [ -z "$url" ] && continue
            CHECKED=$((CHECKED + 1))
            printf "Checking: %s ... " "$url"
            # HEAD transfers no body at all, so nothing is downloaded even when a
            # server ignores Range headers. curl can still exit non-zero (DNS,
            # TLS, timeout), so capture the status without letting `set -e`
            # (bash -e) kill the whole step mid-loop.
            METHOD="HEAD"
            CURL_EXIT=0
            HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \
              --head \
              --max-time 30 \
              --retry 2 \
              --retry-delay 2 \
              --location \
              "$url") || CURL_EXIT=$?
            # Some servers refuse HEAD outright. Fall back to a single-byte
            # ranged GET for those. --max-filesize caps the damage if the server
            # ignores the Range header, but has to stay comfortably above a
            # redirect body: --location applies the limit to the 3xx body too,
            # and a cap below that aborts on the redirect itself.
            case "$HTTP_CODE" in
              403|405|501)
                METHOD="GET"
                CURL_EXIT=0
                HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \
                  --range 0-0 \
                  --max-filesize 8192 \
                  --max-time 30 \
                  --retry 2 \
                  --retry-delay 2 \
                  --location \
                  "$url") || CURL_EXIT=$?
                ;;
            esac
            if [ "$HTTP_CODE" = "200" ] || [ "$HTTP_CODE" = "206" ]; then
              echo "OK ($HTTP_CODE via $METHOD)"
            else
              echo "FAILED (HTTP $HTTP_CODE via $METHOD, curl exit $CURL_EXIT)"
              FAILED=$((FAILED + 1))
              FAILED_URLS="$FAILED_URLS\n  - $url (HTTP $HTTP_CODE via $METHOD, curl exit $CURL_EXIT)"
            fi
          done <<< "$URLS"
          echo ""
          echo "Checked $CHECKED URLs, $FAILED failed."
          if [ "$FAILED" -gt 0 ]; then
            echo ""
            echo "Broken URLs:"
            printf "%b\n" "$FAILED_URLS"
            exit 1
          fi
--- a/.releaserc.json
+++ b/.releaserc.json
@ -1,8 +1,5 @@
 {
-  "branches": [
+  "branches": ["master"],
    "main",
    { "name": "rc", "prerelease": "rc" }
  ],
  "plugins": [
    "@semantic-release/commit-analyzer",
    "@semantic-release/release-notes-generator",
--- a/CODE_OF_CONDUCT.md
+++ b/CODE_OF_CONDUCT.md
@ -1,128 +0,0 @@
 # Contributor Covenant Code of Conduct
 ## Our Pledge
 We as members, contributors, and leaders pledge to make participation in our
 community a harassment-free experience for everyone, regardless of age, body
 size, visible or invisible disability, ethnicity, sex characteristics, gender
 identity and expression, level of experience, education, socio-economic status,
 nationality, personal appearance, race, religion, or sexual identity
 and orientation.
 We pledge to act and interact in ways that contribute to an open, welcoming,
 diverse, inclusive, and healthy community.
 ## Our Standards
 Examples of behavior that contributes to a positive environment for our
 community include:
 * Demonstrating empathy and kindness toward other people
 * Being respectful of differing opinions, viewpoints, and experiences
 * Giving and gracefully accepting constructive feedback
 * Accepting responsibility and apologizing to those affected by our mistakes,
  and learning from the experience
 * Focusing on what is best not just for us as individuals, but for the
  overall community
 Examples of unacceptable behavior include:
 * The use of sexualized language or imagery, and sexual attention or
  advances of any kind
 * Trolling, insulting or derogatory comments, and personal or political attacks
 * Public or private harassment
 * Publishing others' private information, such as a physical or email
  address, without their explicit permission
 * Other conduct which could reasonably be considered inappropriate in a
  professional setting
 ## Enforcement Responsibilities
 Community leaders are responsible for clarifying and enforcing our standards of
 acceptable behavior and will take appropriate and fair corrective action in
 response to any behavior that they deem inappropriate, threatening, offensive,
 or harmful.
 Community leaders have the right and responsibility to remove, edit, or reject
 comments, commits, code, wiki edits, issues, and other contributions that are
 not aligned to this Code of Conduct, and will communicate reasons for moderation
 decisions when appropriate.
 ## Scope
 This Code of Conduct applies within all community spaces, and also applies when
 an individual is officially representing the community in public spaces.
 Examples of representing our community include using an official e-mail address,
 posting via an official social media account, or acting as an appointed
 representative at an online or offline event.
 ## Enforcement
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
 reported to the community leaders responsible for enforcement at
 .
 All complaints will be reviewed and investigated promptly and fairly.
 All community leaders are obligated to respect the privacy and security of the
 reporter of any incident.
 ## Enforcement Guidelines
 Community leaders will follow these Community Impact Guidelines in determining
 the consequences for any action they deem in violation of this Code of Conduct:
 ### 1. Correction
 **Community Impact**: Use of inappropriate language or other behavior deemed
 unprofessional or unwelcome in the community.
 **Consequence**: A private, written warning from community leaders, providing
 clarity around the nature of the violation and an explanation of why the
 behavior was inappropriate. A public apology may be requested.
 ### 2. Warning
 **Community Impact**: A violation through a single incident or series
 of actions.
 **Consequence**: A warning with consequences for continued behavior. No
 interaction with the people involved, including unsolicited interaction with
 those enforcing the Code of Conduct, for a specified period of time. This
 includes avoiding interactions in community spaces as well as external channels
 like social media. Violating these terms may lead to a temporary or
 permanent ban.
 ### 3. Temporary Ban
 **Community Impact**: A serious violation of community standards, including
 sustained inappropriate behavior.
 **Consequence**: A temporary ban from any sort of interaction or public
 communication with the community for a specified period of time. No public or
 private interaction with the people involved, including unsolicited interaction
 with those enforcing the Code of Conduct, is allowed during this period.
 Violating these terms may lead to a permanent ban.
 ### 4. Permanent Ban
 **Community Impact**: Demonstrating a pattern of violation of community
 standards, including sustained inappropriate behavior,  harassment of an
 individual, or aggression toward or disparagement of classes of individuals.
 **Consequence**: A permanent ban from any sort of public interaction within
 the community.
 ## Attribution
 This Code of Conduct is adapted from the [Contributor Covenant][homepage],
 version 2.0, available at
 https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
 Community Impact Guidelines were inspired by [Mozilla's code of conduct
 enforcement ladder](https://github.com/mozilla/diversity).
 [homepage]: https://www.contributor-covenant.org
 For answers to common questions about this code of conduct, see the FAQ at
 https://www.contributor-covenant.org/faq. Translations are available at
 https://www.contributor-covenant.org/translations.
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -1,204 +0,0 @@
 # Contributing to Project NOMAD
 Thank you for your interest in contributing to Project NOMAD! Community contributions are what keep this project growing and improving. Please read this guide fully before getting started — it will save you (and the maintainers) a lot of time.
 > **Note:** Acceptance of contributions is not guaranteed. All pull requests are evaluated based on quality, relevance, and alignment with the project's goals. The maintainers of Project NOMAD ("NOMAD") reserve the right to accept, deny, or modify any pull request at their sole discretion.
 ---
 ## Table of Contents
 - [Code of Conduct](#code-of-conduct)
 - [Before You Start](#before-you-start)
 - [Getting Started](#getting-started)
 - [Development Workflow](#development-workflow)
 - [UI Consistency](#ui-consistency)
 - [Commit Messages](#commit-messages)
 - [Release Notes](#release-notes)
 - [Versioning](#versioning)
 - [Submitting a Pull Request](#submitting-a-pull-request)
 - [Feedback & Community](#feedback--community)
 ---
 ## Code of Conduct
 Please read and review our full [Code of Conduct](https://github.com/Crosstalk-Solutions/project-nomad/blob/main/CODE_OF_CONDUCT.md) before contributing. In short: please be respectful and considerate in all interactions with maintainers and other contributors.
 We are committed to providing a welcoming environment for everyone. Disrespectful or abusive behavior will not be tolerated. 
 ---
 ## Before You Start
 **Open an issue first.** Before writing any code for a non-trivial change, you must [open an issue](../../issues/new) to discuss your proposed change. This helps avoid duplicate work and ensures your contribution aligns with the project's direction. **Pull requests submitted without a corresponding issue may be closed at the maintainers' discretion.**
 **Trivial fixes are exempt** and may be submitted directly as a PR. Examples:
 - Typo and grammar corrections
 - Documentation clarifications
 - Small one-line bug fixes with an obvious cause
 If you're not sure whether your change qualifies as trivial, open an issue first.
 When opening an issue:
 - Use a clear, descriptive title
 - Describe the problem you're solving or the feature you want to add
 - If it's a bug, include steps to reproduce it and as much detail about your environment as possible
 - Ensure you redact any personal or sensitive information in any logs, configs, etc.
 ---
 ## Getting Started with Contributing
 **Please note**: this is the Getting Started guide for developing and contributing to NOMAD, NOT [installing NOMAD](https://github.com/Crosstalk-Solutions/project-nomad/blob/main/README.md) for regular use! 
 ### Prerequisites
 - A Debian-based OS (Ubuntu 26.04 LTS recommended)
 - `sudo`/root privileges
 - Docker installed and running
 - A stable internet connection (required for dependency downloads)
 - Node.js (for frontend/admin work)
 ### Fork & Clone
 1. Click **Fork** at the top right of this repository
 2. Clone your fork locally:
   ```bash
   git clone https://github.com/YOUR_USERNAME/project-nomad.git
   cd project-nomad
   ```
 3. Add the upstream remote so you can stay in sync:
   ```bash
   git remote add upstream https://github.com/Crosstalk-Solutions/project-nomad.git
   ```
 ### Avoid Installing a Release Version Locally
 Because NOMAD relies heavily on Docker, we actually recommend against installing a release version of the project on the same local machine where you are developing. This can lead to conflicts with ports, volumes, and other resources. Instead, you can run your development version in a separate Docker environment while keeping your local machine clean. It certainly __can__ be done, but it adds complexity to your setup and workflow. If you choose to install a release version locally, please ensure you have a clear strategy for managing potential conflicts and resource usage.
 ---
 ## Development Workflow
 1. **Sync with upstream** before starting any new work. We prefer rebasing over merge commits to keep a clean, linear git history as much as possible (this also makes it easier for maintainers to review and merge your changes). To sync with upstream:
   ```bash
   git fetch upstream
   git checkout dev
   git rebase upstream/dev
   ```
 2. **Create a feature branch** off `dev` with a descriptive name:
   ```bash
   git checkout -b fix/issue-123
   # or
   git checkout -b feature/add-new-tool
   ```
 3. **Make your changes.** Follow existing code style and conventions. Test your changes locally against a running NOMAD instance before submitting.
 4. **Add release notes** (see [Release Notes](#release-notes) below).
 5. **Commit your changes** using [Conventional Commits](#commit-messages).
 6. **Push your branch** and open a pull request.
 ---
 ## UI Consistency
 NOMAD's guiding principle is that **user-friendliness is paramount**: a control that looks or behaves differently from the rest of the app reads as broken to a non-technical user. New frontend (inertia/React) work should be visually and behaviorally uniform with what is already there. Before adding a UI element, look at its neighbors and reuse the shared building blocks rather than hand-rolling a one-off.
 **Reuse the shared components** in `admin/inertia/components/` (and `.../components/inputs/`):
 | Need | Use | Not |
 |------|-----|-----|
 | Binary on/off setting | `Switch` | a raw `<input type="checkbox">` |
 | Explanatory hover help | `InfoTooltip` | a raw `title=` attribute or a bespoke tooltip |
 | Modal / confirmation dialog | `StyledModal` | a hand-built overlay |
 | Text field | `Input` | a bare `<input>` |
 | Section heading | `StyledSectionHeader` | ad-hoc heading markup |
 Grep for an existing component before building a new one.
 **Match the neighbors.** Copy the exact classes and conventions of adjacent elements:
 - **Labels:** match punctuation and casing of sibling labels. If the field beside yours reads `Model:` (with a colon), yours should read `Thinking:`, not `Thinking`. Reuse the same typography tokens (e.g. `text-sm text-text-secondary`).
 - **Theme:** use design tokens (`text-*`, `bg-*`, `border-*`) so the element works in light and dark mode. Never hardcode colors.
 - **Placement:** make sure popovers and tooltips are not clipped or crushed against a viewport edge, including when the trigger sits near an edge of the screen.
 **When a raw control is fine.** These conventions are about matching intent, not banning primitives. A raw checkbox is appropriate for a multi-select list or a consent box; radio groups and native selects are fine where a shared component does not exist. The point is to reach for the shared component when your case matches its intent (a binary setting toggle should be a `Switch`), not to eliminate primitives.
 **Test UI changes in a real browser.** Most of these conventions are judgment calls that tooling cannot fully enforce, so the single most important habit is to load your change in a browser against a running instance before submitting. Several classes of issue (clipped or cramped tooltips, layout breaking at different window widths, blank-screen render errors) are invisible to type-checking and only show up when you actually look at the page. Check the states that should appear *and* the states that should be hidden, and try more than one window width when layout or positioning is involved.
 ---
 ## Commit Messages
 This project uses [Conventional Commits](https://www.conventionalcommits.org/). All commit messages must follow this format:
 ```
 <type>(<scope>): <description>
 ```
 **Common types:**
 | Type | When to use |
 |------|-------------|
 | `feat` | A new user-facing feature |
 | `fix` | A bug fix |
 | `docs` | Documentation changes only |
 | `refactor` | Code change that isn't a fix or feature and does not affect functionality |
 | `chore` | Build process, dependency updates, tooling |
 | `test` | Adding or updating tests |
 **Scope** is optional but encouraged — use it to indicate the area of the codebase affected (e.g., `api`, `ui`, `maps`).
 **Examples:**
 ```
 feat(ui): add dark mode toggle to Command Center
 fix(api): resolve container status not updating after restart
 docs: update hardware requirements in README
 chore(deps): bump docker-compose to v2.24
 ```
 ---
 ## Release Notes
 Human-readable release notes live in [`admin/docs/release-notes.md`](admin/docs/release-notes.md) and are displayed directly in the Command Center UI.
 If your PR is merged in, the maintainers will update the release notes with a summary of your contribution and credit you as the author. You do not need to add this yourself in the PR (please don't, as it may cause merge conflicts), but you can include a suggested note in the PR description if you like.
 ---
 ## Versioning
 This project uses [Semantic Versioning](https://semver.org/). Versions are managed in the root `package.json` and updated automatically by `semantic-release`. The `project-nomad` Docker image uses this version. The `admin/package.json` version stays at `0.0.0` and should not be changed manually.
 ---
 ## Submitting a Pull Request
 1. Push your branch to your fork:
   ```bash
   git push origin your-branch-name
   ```
 2. Open a pull request against the `dev` branch of this repository
 3. In the PR description:
   - Summarize what your changes do and why
   - Reference the related issue (e.g., `Closes #123`) — required for non-trivial changes
   - Note any relevant testing steps or environment details
 4. Be responsive to feedback — maintainers may request changes. Pull requests with no activity for an extended period may be closed.
 ---
 ## Feedback & Community
 Have questions or want to discuss ideas before opening an issue? Join the community:
 - **Discord:** [Join the Crosstalk Solutions server](https://discord.com/invite/crosstalksolutions) — the best place to get help, share your builds, and talk with other NOMAD users
 - **Website:** [www.projectnomad.us](https://www.projectnomad.us)
 - **Benchmark Leaderboard:** [benchmark.projectnomad.us](https://benchmark.projectnomad.us)
 ---
 *Project NOMAD is licensed under the [Apache License 2.0](LICENSE).*
--- a/93
+++ b/93
@ -1,15 +1,7 @@
-FROM node:22-slim AS base
+FROM node:22.16.0-alpine3.22 AS base
-# Install bash & curl for entrypoint script compatibility, graphicsmagick for pdf2pic, and vips-dev & build-base for sharp 
+# Install bash & curl for entrypoint script compatibility
-RUN apt-get update && apt-get install -y \
+RUN apk add --no-cache bash curl
      bash \
      curl \
      openssl \
      graphicsmagick \
      libvips-dev \
      build-essential \
      pciutils \
      && rm -rf /var/lib/apt/lists/*
 # All deps stage
 FROM base AS deps
@ -28,89 +20,16 @@ FROM base AS build
 WORKDIR /app
 COPY --from=deps /app/node_modules /app/node_modules
 ADD admin/ ./
 # Regenerate the curated drug-reference data modules
 # (app/data/{conditions,natural_remedies,home_remedies}.ts) from their single
 # source of truth — the repo-root collections/*.json — so the JSON is what gets
 # compiled into the image and the committed .ts can never silently drift from it
 # in a build. The gen script resolves ../../collections relative to admin/scripts,
 # which is /collections once admin/ has been copied to /app.
 COPY collections/ /collections/
 RUN npm run gen:curated-data
 RUN node ace build
 # Production stage
 FROM base
 ARG VERSION=dev
 ARG BUILD_DATE
 ARG VCS_REF
 ARG TARGETARCH
 # go-pmtiles (regional map extracts). Pinned so the CLI's stdout format stays
 # in sync with parseDryRunOutput().
 ARG PMTILES_VERSION=1.30.2
 # Upstream releases don't ship a checksums file, so pin per-arch SHA256 here.
 # When bumping PMTILES_VERSION, regenerate these with:
 #   curl -fsSL <release-url> | sha256sum
 ARG PMTILES_SHA256_AMD64=2cd3aa18868297fc88425038f794efdc0995e0275f4ca16fa496dd79e245a40c
 ARG PMTILES_SHA256_ARM64=804cdf071834e1156af554c1a26cc42b56b9cde5a2db9c6e3653d16fb846d5fa
 RUN set -eux; \
    case "${TARGETARCH:-amd64}" in \
      amd64) PMTILES_ARCH=x86_64; PMTILES_SHA256="${PMTILES_SHA256_AMD64}" ;; \
      arm64) PMTILES_ARCH=arm64;  PMTILES_SHA256="${PMTILES_SHA256_ARM64}" ;; \
      *) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \
    esac; \
    TARBALL="go-pmtiles_${PMTILES_VERSION}_Linux_${PMTILES_ARCH}.tar.gz"; \
    cd /tmp; \
    curl -fsSL -o "$TARBALL" \
      "https://github.com/protomaps/go-pmtiles/releases/download/v${PMTILES_VERSION}/${TARBALL}"; \
    echo "${PMTILES_SHA256}  ${TARBALL}" | sha256sum -c -; \
    tar -xzf "$TARBALL" -C /usr/local/bin pmtiles; \
    rm -f "$TARBALL"; \
    chmod +x /usr/local/bin/pmtiles; \
    /usr/local/bin/pmtiles version
 # Labels
 LABEL org.opencontainers.image.title="Project NOMAD" \
      org.opencontainers.image.description="The Project NOMAD Official Docker image" \
      org.opencontainers.image.version="${VERSION}" \
      org.opencontainers.image.created="${BUILD_DATE}" \
      org.opencontainers.image.revision="${VCS_REF}" \
      org.opencontainers.image.vendor="Crosstalk Solutions, LLC" \
      org.opencontainers.image.documentation="https://github.com/CrosstalkSolutions/project-nomad/blob/main/README.md" \
      org.opencontainers.image.source="https://github.com/CrosstalkSolutions/project-nomad" \
      org.opencontainers.image.licenses="Apache-2.0"
 ENV NODE_ENV=production
 # Creator Packs entitlement key, injected into OFFICIAL release builds at build
 # time (--build-arg CREATOR_PACKS_APP_KEY=... from the CREATOR_PACKS_APP_KEY CI
 # secret; see build-primary-image.yml). Baked as an ENV so admin/start/env.ts
 # reads it at runtime. Empty by default, so builds from source (and any build
 # without the secret) ship UNCONFIGURED and hide the Creator Packs UI. The key
 # lands in this public image layer (extractable — the accepted ceiling); rotate
 # via `wrangler secret put APP_KEY` + a new image if it leaks.
 ARG CREATOR_PACKS_APP_KEY=""
 ENV CREATOR_PACKS_APP_KEY=$CREATOR_PACKS_APP_KEY
 WORKDIR /app
 COPY --from=production-deps /app/node_modules /app/node_modules
 COPY --from=build /app/build /app
-# Generate version.json from the VERSION build-arg so the image tag is the
+# Copy root package.json for version info
-# single source of truth (previously copied root package.json, which drifted
+COPY package.json /app/version.json
 # from the tag when semantic-release did not commit the bump back).
 RUN echo "{\"version\":\"${VERSION}\"}" > /app/version.json
 # Copy docs and README for access within the container
 COPY admin/docs /app/docs
 COPY README.md /app/README.md
 # Empty Calibre library, seeded into storage/books on Calibre-Web install
 # (see DockerService._runPreinstallActions__CalibreWeb)
 COPY install/calibre-empty-library/metadata.db /app/assets/calibre/metadata.db
 # Copy entrypoint script and ensure it's executable
 COPY install/entrypoint.sh /usr/local/bin/entrypoint.sh
 RUN chmod +x /usr/local/bin/entrypoint.sh
 EXPOSE 8080
-ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
+CMD ["node", "./bin/server.js"]
--- a/FAQ.md
+++ b/FAQ.md
@ -1,108 +0,0 @@
 # Frequently Asked Questions (FAQ)
 Find answers to some of the most common questions about Project NOMAD
 ## Can I customize the port(s) that NOMAD uses?
 Yes, you can customize the ports that NOMAD's core services (Command Center, MySQL, Redis) use. Please refer to the [Advanced Installation](README.md#advanced-installation) section of the README for more details on how to do this.
 Note: As of 3/24/2026, only the core services defined in the `docker-compose.yml` file currently support port customization - the installable applications (e.g. Ollama, Kiwix, etc.) do not yet support this, but we have multiple PR's in the works to add this feature for all installable applications in a future release.
 ## Can I customize the storage location for NOMAD's data?
 Yes, you can customize the storage location for NOMAD's content by modifying the `docker-compose.yml` file to adjust the appropriate bind mounts to point to your desired storage location on your host machine. Please refer to the [Advanced Installation](README.md#advanced-installation) section of the README for more details on how to do this.
 ## Can I store NOMAD's data on an external drive or network storage?
 Short answer: yes, but we can't do it for you (and we recommend a local drive for best performance).
 Long answer: Custom storage paths, mount points, and external drives (like iSCSI or SMB/NFS volumes) **are possible**, but this will be up to your individual configuration on the host before NOMAD starts, and then passed in via the compose.yml as this is a *host-level concern*, not a NOMAD-level concern (see above for details). NOMAD itself can't configure this for you, nor could we support all possible configurations in the install script.
 ## Can I run NOMAD on MAC, WSL2, or a non-Debian-based Distro?
 **WSL2 on Windows** is community-supported via the [WSL2 install guide](https://www.projectnomad.us/install/wsl2) — covers two install paths (native Docker and Docker Desktop) with all known gotchas documented and empirical performance numbers comparing WSL2 to bare-metal.
 **macOS and other non-Debian Linux distros** aren't officially supported. See [Why does NOMAD require a Debian-based OS?](#why-does-nomad-require-a-debian-based-os) for details.
 ## Why does NOMAD require a Debian-based OS?
 Project NOMAD is currently designed to run on Debian-based Linux distributions (with Ubuntu 26.04 LTS being the recommended version) because our installation scripts and Docker configurations are optimized for this environment. While it's technically possible to run the Docker containers on other operating systems that support Docker, we have not tested or optimized the installation process for non-Debian-based systems, so we cannot guarantee a smooth experience on those platforms at this time.
 Support for other operating systems will come in the future, but because our development resources are limited as a free and open-source project, we needed to prioritize our efforts and focus on a narrower set of supported platforms for the initial release. We chose Debian-based Linux as our starting point because it's widely used, easy to spin up, and provides a stable environment for running Docker containers.
 For Windows users, the [WSL2 install guide](https://www.projectnomad.us/install/wsl2) provides a community-supported path. Community members have also published guides for other platforms (e.g. macOS) in our Discord community and [Github Discussions](https://github.com/Crosstalk-Solutions/project-nomad/discussions), so if you're interested in running NOMAD on a non-Debian-based system, we recommend checking there for any available resources or guides. However, keep in mind that if you choose to run NOMAD on a non-Debian-based system, you may encounter issues that we won't be able to provide support for, and you may need to have a higher level of technical expertise to troubleshoot and resolve any problems that arise.
 ## Can I run NOMAD on a Raspberry Pi or other ARM-based device?
 Project NOMAD is currently designed to run on x86-64 architecture, and we have not yet tested or optimized it for ARM-based devices like the Raspberry Pi (and have not published any official images for ARM architecture).
 Support for ARM-based devices is on our roadmap, but our initial focus was on x86-64 hardware due to its widespread use and compatibility with a wide range of applications.
 Community members have forked and published their own ARM-compatible images and installation guides for running NOMAD on Raspberry Pi and other ARM-based devices in our Discord community and [Github Discussions](https://github.com/Crosstalk-Solutions/project-nomad/discussions), but these are not officially supported by the core development team, and we cannot guarantee their functionality or provide support for any issues that arise when using these community-created resources.
 ## What are the hardware requirements for running NOMAD?
 Project NOMAD itself is quite lightweight and can run on even modest x86-64 hardware, but the tools and resources you choose to install with NOMAD will determine the specs required for your unique deployment. Please see the [Hardware Guide](https://www.projectnomad.us/hardware) for detailed build recommendations at various price points.
 ## Does NOMAD support languages other than English?
 As of March 2026, Project NOMAD's UI is only available in English, and the majority of the tools and resources available through NOMAD are also primarily in English. However, we have multi-language support on our roadmap for a future release, and we are actively working on adding support for additional languages both in the UI and in the available tools/resources. If you're interested in contributing to this effort, please check out our [CONTRIBUTING.md](CONTRIBUTING.md) file for guidelines on how to get involved.
 ## What technologies is NOMAD built with?
 Project NOMAD is built using a combination of technologies, including:
 - **Docker:** for containerization of the Command Center and its dependencies
 - **Node.js & TypeScript:** for the backend of the Command Center, particularly the [AdonisJS](https://adonisjs.com/) framework
 - **React:** for the frontend of the Command Center, utilizing [Vite](https://vitejs.dev/) and [Inertia.js](https://inertiajs.com/) under the hood
 - **MySQL:** for the Command Center's database
 - **Redis:** for various caching, background jobs, "cron" tasks, and other internal processes within the Command Center
 NOMAD makes use of the Docker-outside-of-Docker ("DooD") pattern, which allows the Command Center to manage and orchestrate other Docker containers on the host machine without needing to run Docker itself inside a container. This approach provides better performance and compatibility with a wider range of host environments while still allowing for powerful container management capabilities through the Command Center's UI.
 ## Can I run NOMAD if I have existing Docker containers on my machine?
 Yes, you can safely run Project NOMAD on a machine that already has existing Docker containers. NOMAD is designed to coexist with other Docker containers and will not interfere with them as long as there are no port conflicts or resource constraints.
 All of NOMAD's containers are prefixed with `nomad_` in their names, so they can be easily identified and managed separately from any other containers you may have running. Just make sure to review the ports that NOMAD's core services (Command Center, MySQL, Redis) use during installation and adjust them if necessary to avoid conflicts with your existing containers.
 ## Why does NOMAD require access to the Docker socket?
 See [What technologies is NOMAD built with?](#what-technologies-is-nomad-built-with)
 ## Can I use any AI models?
 NOMAD by default uses Ollama inside of a docker container to run LLM Models for the AI Assistant. So if you find a model on HuggingFace for example, you won't be able to use that model in NOMAD. The list of available models in the AI Assistant settings (/settings/models) may not show all of the models you are looking for. If you found a model from https://ollama.com/search that you'd like to try and its not in the settings page, you can use a curl command to download the model.  
 `curl -X POST -H "Content-Type: application/json" -d '{"model":"MODEL_NAME_HERE"}' http://localhost:8080/api/ollama/models` replacing MODEL_NAME_HERE with the model name from whats in the ollama website.
 ## Do I have to install the AI features in NOMAD?
 No, the AI features in NOMAD (Ollama, Qdrant, custom RAG pipeline, etc.) are all optional and not required to use the core functionality of NOMAD.
 ## Is NOMAD actually free? Are there any hidden costs?
 Yes, Project NOMAD is completely free and open-source software licensed under the Apache License 2.0. There are no hidden costs or fees associated with using NOMAD itself, and we don't have any plans to introduce "premium" features or paid tiers.
 Aside from the cost of the hardware you choose to run it on, there are no costs associated with using NOMAD.
 ## Do you sell hardware or pre-built devices with NOMAD pre-installed?
 No, we do not sell hardware or pre-built devices with NOMAD pre-installed at this time. Project NOMAD is a free and open-source software project, and we provide detailed installation instructions and hardware recommendations for users to set up their own NOMAD instances on compatible hardware of their choice. The tradeoff to this DIY approach is some additional setup time and technical know-how required on the user's end, but it also allows for greater flexibility and customization in terms of hardware selection and configuration to best suit each user's unique needs, budget, and preferences.
 ## How quickly are issues resolved when reported?
 We strive to address and resolve issues as quickly as possible, but please keep in mind that Project NOMAD is a free and open-source project maintained by a small team of volunteers. We prioritize issues based on their severity, impact on users, and the resources required to resolve them. Critical issues that affect a large number of users are typically addressed more quickly, while less severe issues may take longer to resolve. Aside from the development efforts needed to address the issue, we do our best to conduct thorough testing and validation to ensure that any fix we implement doesn't introduce new issues or regressions, which also adds to the time it takes to resolve an issue.
 We also encourage community involvement in troubleshooting and resolving issues, so if you encounter a problem, please consider checking our Discord community and Github Discussions for potential solutions or workarounds while we work on an official fix.
 ## How often are new features added or updates released?
 We aim to release updates and new features on a regular basis, but the exact timing can vary based on the complexity of the features being developed, the resources available to our volunteer development team, and the feedback and needs of our community. We typically release smaller "patch" versions more frequently to address bugs and make minor improvements, while larger feature releases may take more time to develop and test before they're ready for release.
 ## I opened a PR to contribute a new feature or fix a bug. How long does it usually take for PRs to be reviewed and merged?
 We appreciate all contributions to the project and strive to review and merge pull requests (PRs) as quickly as possible. The time it takes for a PR to be reviewed and merged can vary based on several factors, including the complexity of the changes, the current workload of our maintainers, and the need for any additional testing or revisions.
 Because NOMAD is still a young project, some PRs (particularly those for new features) may take longer to review and merge as we prioritize building out the core functionality and ensuring stability before adding new features. However, we do our best to provide timely feedback on all PRs and keep contributors informed about the status of their contributions.
 ## I have a question that isn't answered here. Where can I ask for help?
 If you have a question that isn't answered in this FAQ, please feel free to ask for help in our Discord community (https://discord.com/invite/crosstalksolutions) or on our Github Discussions page (https://github.com/Crosstalk-Solutions/project-nomad/discussions).
 ## I have a suggestion for a new feature or improvement. How can I share it?
 We welcome and encourage suggestions for new features and improvements! We highly encourage sharing your ideas (or upvoting existing suggestions) on our public roadmap at https://roadmap.projectnomad.us, where we track new feature requests. This is the best way to ensure that your suggestion is seen by the development team and the community, and it also allows other community members to upvote and show support for your idea, which can help prioritize it for future development.
--- a/190
+++ b/190
@ -1,190 +0,0 @@
                                 Apache License
                           Version 2.0, January 2004
                        http://www.apache.org/licenses/
   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
   1. Definitions.
      "License" shall mean the terms and conditions for use, reproduction,
      and distribution as defined by Sections 1 through 9 of this document.
      "Licensor" shall mean the copyright owner or entity authorized by
      the copyright owner that is granting the License.
      "Legal Entity" shall mean the union of the acting entity and all
      other entities that control, are controlled by, or are under common
      control with that entity. For the purposes of this definition,
      "control" means (i) the power, direct or indirect, to cause the
      direction or management of such entity, whether by contract or
      otherwise, or (ii) ownership of fifty percent (50%) or more of the
      outstanding shares, or (iii) beneficial ownership of such entity.
      "You" (or "Your") shall mean an individual or Legal Entity
      exercising permissions granted by this License.
      "Source" form shall mean the preferred form for making modifications,
      including but not limited to software source code, documentation
      source, and configuration files.
      "Object" form shall mean any form resulting from mechanical
      transformation or translation of a Source form, including but
      not limited to compiled object code, generated documentation,
      and conversions to other media types.
      "Work" shall mean the work of authorship, whether in Source or
      Object form, made available under the License, as indicated by a
      copyright notice that is included in or attached to the work
      (an example is provided in the Appendix below).
      "Derivative Works" shall mean any work, whether in Source or Object
      form, that is based on (or derived from) the Work and for which the
      editorial revisions, annotations, elaborations, or other modifications
      represent, as a whole, an original work of authorship. For the purposes
      of this License, Derivative Works shall not include works that remain
      separable from, or merely link (or bind by name) to the interfaces of,
      the Work and Derivative Works thereof.
      "Contribution" shall mean any work of authorship, including
      the original version of the Work and any modifications or additions
      to that Work or Derivative Works thereof, that is intentionally
      submitted to the Licensor for inclusion in the Work by the copyright owner
      or by an individual or Legal Entity authorized to submit on behalf of
      the copyright owner. For the purposes of this definition, "submitted"
      means any form of electronic, verbal, or written communication sent
      to the Licensor or its representatives, including but not limited to
      communication on electronic mailing lists, source code control systems,
      and issue tracking systems that are managed by, or on behalf of, the
      Licensor for the purpose of discussing and improving the Work, but
      excluding communication that is conspicuously marked or otherwise
      designated in writing by the copyright owner as "Not a Contribution."
      "Contributor" shall mean Licensor and any individual or Legal Entity
      on behalf of whom a Contribution has been received by the Licensor and
      subsequently incorporated within the Work.
   2. Grant of Copyright License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      copyright license to reproduce, prepare Derivative Works of,
      publicly display, publicly perform, sublicense, and distribute the
      Work and such Derivative Works in Source or Object form.
   3. Grant of Patent License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      (except as stated in this section) patent license to make, have made,
      use, offer to sell, sell, import, and otherwise transfer the Work,
      where such license applies only to those patent claims licensable
      by such Contributor that are necessarily infringed by their
      Contribution(s) alone or by combination of their Contribution(s)
      with the Work to which such Contribution(s) was submitted. If You
      institute patent litigation against any entity (including a
      cross-claim or counterclaim in a lawsuit) alleging that the Work
      or a Contribution incorporated within the Work constitutes direct
      or contributory patent infringement, then any patent licenses
      granted to You under this License for that Work shall terminate
      as of the date such litigation is filed.
   4. Redistribution. You may reproduce and distribute copies of the
      Work or Derivative Works thereof in any medium, with or without
      modifications, and in Source or Object form, provided that You
      meet the following conditions:
      (a) You must give any other recipients of the Work or
          Derivative Works a copy of this License; and
      (b) You must cause any modified files to carry prominent notices
          stating that You changed the files; and
      (c) You must retain, in the Source form of any Derivative Works
          that You distribute, all copyright, patent, trademark, and
          attribution notices from the Source form of the Work,
          excluding those notices that do not pertain to any part of
          the Derivative Works; and
      (d) If the Work includes a "NOTICE" text file as part of its
          distribution, then any Derivative Works that You distribute must
          include a readable copy of the attribution notices contained
          within such NOTICE file, excluding any notices that do not
          pertain to any part of the Derivative Works, in at least one
          of the following places: within a NOTICE text file distributed
          as part of the Derivative Works; within the Source form or
          documentation, if provided along with the Derivative Works; or,
          within a display generated by the Derivative Works, if and
          wherever such third-party notices normally appear. The contents
          of the NOTICE file are for informational purposes only and
          do not modify the License. You may add Your own attribution
          notices within Derivative Works that You distribute, alongside
          or as an addendum to the NOTICE text from the Work, provided
          that such additional attribution notices cannot be construed
          as modifying the License.
      You may add Your own copyright statement to Your modifications and
      may provide additional or different license terms and conditions
      for use, reproduction, or distribution of Your modifications, or
      for any such Derivative Works as a whole, provided Your use,
      reproduction, and distribution of the Work otherwise complies with
      the conditions stated in this License.
   5. Submission of Contributions. Unless You explicitly state otherwise,
      any Contribution intentionally submitted for inclusion in the Work
      by You to the Licensor shall be under the terms and conditions of
      this License, without any additional terms or conditions.
      Notwithstanding the above, nothing herein shall supersede or modify
      the terms of any separate license agreement you may have executed
      with Licensor regarding such Contributions.
   6. Trademarks. This License does not grant permission to use the trade
      names, trademarks, service marks, or product names of the Licensor,
      except as required for reasonable and customary use in describing the
      origin of the Work and reproducing the content of the NOTICE file.
   7. Disclaimer of Warranty. Unless required by applicable law or
      agreed to in writing, Licensor provides the Work (and each
      Contributor provides its Contributions) on an "AS IS" BASIS,
      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
      implied, including, without limitation, any warranties or conditions
      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
      PARTICULAR PURPOSE. You are solely responsible for determining the
      appropriateness of using or redistributing the Work and assume any
      risks associated with Your exercise of permissions under this License.
   8. Limitation of Liability. In no event and under no legal theory,
      whether in tort (including negligence), contract, or otherwise,
      unless required by applicable law (such as deliberate and grossly
      negligent acts) or agreed to in writing, shall any Contributor be
      liable to You for damages, including any direct, indirect, special,
      incidental, or consequential damages of any character arising as a
      result of this License or out of the use or inability to use the
      Work (including but not limited to damages for loss of goodwill,
      work stoppage, computer failure or malfunction, or any and all
      other commercial damages or losses), even if such Contributor
      has been advised of the possibility of such damages.
   9. Accepting Warranty or Additional Liability. While redistributing
      the Work or Derivative Works thereof, You may choose to offer,
      and charge a fee for, acceptance of support, warranty, indemnity,
      or other liability obligations and/or rights consistent with this
      License. However, in accepting such obligations, You may act only
      on Your own behalf and on Your sole responsibility, not on behalf
      of any other Contributor, and only if You agree to indemnify,
      defend, and hold each Contributor harmless for any liability
      incurred by, or claims asserted against, such Contributor by reason
      of your accepting any such warranty or additional liability.
   END OF TERMS AND CONDITIONS
   Copyright 2024-2026 Crosstalk Solutions LLC
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at
       http://www.apache.org/licenses/LICENSE-2.0
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
--- a/README.md
+++ b/README.md
@ -1,174 +1,75 @@
-<div align="center">
+# *NOTE: Project N.O.M.A.D. is still in active development and should not be considered stable!*
 <img src="admin/public/project_nomad_logo.webp" width="200" height="200"/>
 # Project NOMAD
 ### Knowledge That Never Goes Offline
 [![Website](https://img.shields.io/badge/Website-projectnomad.us-blue)](https://www.projectnomad.us)
 [![Discord](https://img.shields.io/badge/Discord-Join%20Community-5865F2)](https://discord.com/invite/crosstalksolutions)
 [![Benchmark](https://img.shields.io/badge/Benchmark-Leaderboard-green)](https://benchmark.projectnomad.us)
 <div style="width: 100;text-align: center;margin-bottom: 25px;">
 <img src="https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/master/admin/public/project_nomad_logo.png" width="200" height="200"/>
 </div>
---
+# Project N.O.M.A.D. (Node for Offline Media, Archives, and Data)
-
+Project N.O.M.A.D., is a self-contained, offline survival computer packed with critical tools, knowledge, and AI to keep you informed and empowered—anytime, anywhere.
 Project NOMAD is a self-contained, offline-first knowledge and education server packed with critical tools, knowledge, and AI to keep you informed and empowered — anytime, anywhere.
 ## Installation & Quickstart
-Project NOMAD can be installed on any Debian-based operating system (we recommend Ubuntu 26.04 LTS; 24.04 LTS and Debian 12 are also supported). Installation is completely terminal-based, and all tools and resources are designed to be accessed through the browser, so there's no need for a desktop environment if you'd rather setup NOMAD as a "server" and access it through other clients.
+Project N.O.M.A.D. can be installed on any Debian-based operating system (we recommend Ubuntu). Installation is completely terminal-based, and all tools and resources are designed to be accessed through the browser, so there's no need for a desktop environment if you'd rather setup N.O.M.A.D. as a "server" and access it through other clients.
 *Note: sudo/root privileges are required to run the install script*
 ### Quick Install (Debian-based OS Only)
 ```bash
-sudo apt-get update && \
+curl -fsSL https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/master/install/install_nomad.sh -o install_nomad.sh
-sudo apt-get install -y curl && \
+```
-curl -fsSL https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/install/install_nomad.sh \
+
-  -o install_nomad.sh && \
+```bash
 sudo bash install_nomad.sh
 ```
-Project NOMAD is now installed on your device! Open a browser and navigate to `http://localhost:8080` (or `http://DEVICE_IP:8080`) to start exploring!
+Project N.O.M.A.D. is now installed on your device! Open a browser and navigate to `http://localhost:8080` (or `http://DEVICE_IP:8080`) to start exploring!
 For a complete step-by-step walkthrough (including Ubuntu installation), see the [Installation Guide](https://www.projectnomad.us/install). For Windows users, see the [WSL2 install guide](https://www.projectnomad.us/install/wsl2) — community-supported path covering native Docker and Docker Desktop install routes.
 ### Advanced Installation
 For more control over the installation process, copy and paste the [Docker Compose template](https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/install/management_compose.yaml) into a `docker-compose.yml` file and customize it to your liking (be sure to replace any placeholders with your actual values). Then, run `docker compose up -d` to start the Command Center and its dependencies. Note: this method is recommended for advanced users only, as it requires familiarity with Docker and manual configuration before starting.
 ## How It Works
-NOMAD is a management UI ("Command Center") and API that orchestrates a collection of containerized tools and resources via [Docker](https://www.docker.com/). It handles installation, configuration, and updates for everything — so you don't have to.
+From a technical standpoint, N.O.M.A.D. is primarily a management UI ("Command Center") and API that orchestrates a goodie basket of containerized offline archive tools and resources such as 
 [Kiwix](https://kiwix.org/), [OpenStreetMap](https://www.openstreetmap.org/), [Ollama](https://ollama.com/), [OpenWebUI](https://openwebui.com/), and more.
-**Built-in capabilities include:**
+By abstracting the installation of each of these awesome tools, N.O.M.A.D. makes getting your offline survival computer up and running a breeze! N.O.M.A.D. also includes some additional built-in handy tools, such as a ZIM library managment interface, calculators, and more.
 - **AI Chat with Knowledge Base** — local AI chat powered by [Ollama](https://ollama.com/) or you can use OpenAI API compatible software such as LM Studio or llama.cpp, with document upload and semantic search (RAG via [Qdrant](https://qdrant.tech/))
 - **Information Library** — offline Wikipedia, medical references, ebooks, and more via [Kiwix](https://kiwix.org/)
 - **Education Platform** — Khan Academy courses with progress tracking via [Kolibri](https://learningequality.org/kolibri/)
 - **Offline Maps** — downloadable regional maps via [ProtoMaps](https://protomaps.com)
 - **Data Tools** — encryption, encoding, and analysis via [CyberChef](https://gchq.github.io/CyberChef/)
 - **Notes** — local note-taking via [FlatNotes](https://github.com/dullage/flatnotes)
 - **System Benchmark** — hardware scoring with a [community leaderboard](https://benchmark.projectnomad.us)
 - **Supply Depot** — a one-click app catalog (PDF tools, file browser, e-book library, password manager, and more) plus the ability to run your own custom Docker containers
 - **Automatic Updates** — opt-in, hands-off updates for the core software, installed apps, and offline content, on a schedule you control
 - **Easy Setup Wizard** — guided first-time configuration with curated content collections
 NOMAD also includes built-in tools like a Wikipedia content selector, ZIM library manager, and content explorer.
 ## What's Included
 | Capability | Powered By | What You Get |
 |-----------|-----------|-------------|
 | Information Library | Kiwix | Offline Wikipedia, medical references, survival guides, ebooks |
 | AI Assistant | Ollama + Qdrant | Built-in chat with document upload and semantic search |
 | Education Platform | Kolibri | Khan Academy courses, progress tracking, multi-user support |
 | Offline Maps | ProtoMaps | Downloadable regional maps for offline viewing and search |
 | Data Tools | CyberChef | Encryption, encoding, hashing, and data analysis |
 | Notes | FlatNotes | Local note-taking with markdown support |
 | System Benchmark | Built-in | Hardware scoring, Builder Tags, and community leaderboard |
 | Supply Depot | Built-in | One-click app catalog + bring-your-own custom Docker containers |
 ## Device Requirements
-While many similar offline survival computers are designed to be run on bare-minimum, lightweight hardware, Project NOMAD is quite the opposite. To install and run the
+While many similar offline survival computers are designed to be run on bare-minimum, lightweight hardware, Project N.O.M.A.D. is quite the opposite. To install and run the
 available AI tools, we highly encourage the use of a beefy, GPU-backed device to make the most of your install.
-At its core, however, NOMAD is still very lightweight. For a barebones installation of the management application itself, the following minimal specs are required:
+At it's core, however, N.O.M.A.D. is still very lightweight. For a barebones installation of the management application itself, the following minimal specs are required:
-*Note: Project NOMAD is not sponsored by any hardware manufacturer and is designed to be as hardware-agnostic as possible. The hardware listed below is for example/comparison use only*
+*Note: Project N.O.M.A.D. is not sponsored by any hardware manufacturer and is designed to be as hardware-agnostic as possible. The harware listed below is for example/comparison use only*
 #### Minimum Specs
 - Processor: 2 GHz dual-core processor or better
 - RAM: 4GB system memory
 - Storage: At least 5 GB free disk space
- OS: Debian-based (Ubuntu 26.04 LTS recommended)
+- OS: Debian-based (Ubuntu recommended)
 - Stable internet connection (required during install only)
-To run LLMs and other included AI tools:
+To run LLM's and other included AI tools:
 #### Optimal Specs
 - Processor: AMD Ryzen 7 or Intel Core i7 or better
 - RAM: 32 GB system memory
- Graphics: NVIDIA RTX 3060 or AMD equivalent or better (more VRAM = run larger models)
+- Graphics: NVIDIA RTX 3060 or better (more VRAM = run larger models)
 - Storage: At least 250 GB free disk space (preferably on SSD)
- OS: Debian-based (Ubuntu 26.04 LTS recommended)
+- OS: Debian-based (Ubuntu recommended)
 - Stable internet connection (required during install only)
-**For detailed build recommendations at three price points ($150–$1,000+), see the [Hardware Guide](https://www.projectnomad.us/hardware).**
+Again, Project N.O.M.A.D. itself is quite lightweight - it's the tools and resources you choose to install with N.O.M.A.D. that will determine the specs required for your unique deployment
 Again, Project NOMAD itself is quite lightweight — it's the tools and resources you choose to install with NOMAD that will determine the specs required for your unique deployment
 #### Running AI models on a different host
 By default, NOMAD's installer will attempt to setup Ollama on the host when the AI Assistant is installed. However, if you would like to run the AI model on a different host, you can go to the settings of the AI assistant and input a URL for either an ollama or OpenAI-compatible API server (such as LM Studio).  
 Note that if you use Ollama on a different host, you must start the server with this option: `OLLAMA_HOST=0.0.0.0`.  
 Ollama is the preferred way to use the AI assistant, as it has features such as model download that OpenAI API does not support. So when using LM Studio, for example, you will have to use LM Studio to download models.
 You are responsible for the setup of Ollama/OpenAI server on the other host.
 ## Frequently Asked Questions (FAQ)
 For answers to common questions about Project NOMAD, please see our [FAQ](FAQ.md) page.
 ## About Internet Usage & Privacy
-Project NOMAD is designed for offline usage. An internet connection is only required during the initial installation (to download dependencies) and if you (the user) decide to download additional tools and resources at a later time. Otherwise, NOMAD does not require an internet connection and has ZERO built-in telemetry.
+Project N.O.M.A.D. is designed for offline usage. An internet connection is only required during the initial installation (to download dependencies) and if you (the user) decide to download additional tools and resources at a later time. Otherwise, N.O.M.A.D. does not require an internet connection and has ZERO built-in telemetry.
-To test internet connectivity, NOMAD first attempts to make a request to Cloudflare's utility endpoint, `https://1.1.1.1/cdn-cgi/trace`. If that endpoint is unreachable (for example, because your network blocks `1.1.1.1`), it falls back to other endpoints the application already contacts (the GitHub API and the Project NOMAD API) and considers the connection online if any of them respond.
+To test internet connectivity, N.O.M.A.D. attempts to make a request to Cloudflare's utility endpoint, `https://1.1.1.1/cdn-cgi/trace` and checks for a successful response.
 You can override the endpoint used for this check in two ways. The connectivity test URL can be configured from the UI under **Settings → Advanced** (stored locally on your instance), or you can set the `INTERNET_STATUS_TEST_URL` environment variable. When set, the environment variable always takes precedence over the UI-configured value. If neither is set, the built-in defaults above are used.
 ## About Security
-By design, Project NOMAD is intended to be open and available without hurdles — it includes no authentication. If you decide to connect your device to a local network after install (e.g. for allowing other devices to access its resources), you can block/open ports to control which services are exposed.
+By design, Project N.O.M.A.D. is intended to be open and available without hurdles - it includes no authentication. If you decide to connect your device to a local network after install (e.g. for allowing other devices to access it's resources), you can block/open ports to control which services are exposed.
-**Will authentication be added in the future?** Maybe. It's not currently a priority, but if there's enough demand for it, we may consider building in an optional authentication layer in a future release to support use cases where multiple users need access to the same instance but with different permission levels (e.g. family use with parental controls, classroom use with teacher/admin accounts, etc.). We have a suggestion for this on our public roadmap, so if this is something you'd like to see, please upvote it here: https://roadmap.projectnomad.us/posts/1/user-authentication-please-build-in-user-auth-with-admin-user-roles
+## Versioning
-
+This project uses semantic versioning. The version is managed in the root `package.json` 
-For now, we recommend using network-level controls to manage access if you're planning to expose your NOMAD instance to other devices on a local network. NOMAD is not designed to be exposed directly to the internet, and we strongly advise against doing so unless you really know what you're doing, have taken appropriate security measures, and understand the risks involved.
+and automatically updated by semantic-release. For simplicity's sake, the "project-nomad" container
-
+uses the same version defined there instead of the version in `admin/package.json` (stays at 0.0.0), as it's the only container derived from the code.
 ## Contributing
 Contributions are welcome and appreciated! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines on how to contribute to the project.
 ### Testing Auto-Updates (Dry Run)
 The Command Center can automatically install **minor/patch** updates of itself during a configurable window, after a cool-off period, and only when pre-flight checks pass (sufficient disk for the new image, no downloads or app installs in progress). Major versions always require a manual update.
 Because exercising this logic with real version bumps is impractical, an Ace command runs the **entire decision pipeline without ever triggering an update**. Run it from the `admin/` directory:
 ```bash
 # 1) Deterministic scenario suite — no network, DB, or Docker required.
 #    Proves every branch (major-only, cool-off, prerelease/draft, window wrap, …)
 #    and exits non-zero on failure, so it's safe to wire into CI.
 node ace auto-update:dry-run --scenarios
 # 2) Simulate "what would happen if I were running 1.32.0 right now?"
 #    against the LIVE GitHub releases feed and real pre-flight checks:
 node ace auto-update:dry-run --current=1.32.0 --force-enabled
 # 3) Fully offline simulation with a canned release list and a fixed clock:
 node ace auto-update:dry-run --current=1.32.0 --force-enabled \
  --releases-file=./fixtures/releases.json --now=2026-06-04T21:00:00Z \
  --window-start=20:00 --window-end=23:00 --cooloff=72 --skip-preflight
 ```
 It prints the resolved decision — current version, whether the clock is inside the window, the eligible target (if any), and pre-flight blockers — ending in a clear verdict such as `WOULD UPDATE → v1.33.2` or `WOULD NOT UPDATE (outside-window): …`. **No real update is ever requested.**
 | Flag | Description |
 |------|-------------|
 | `--scenarios` | Run the built-in deterministic scenario suite and exit |
 | `--current=<version>` | Simulate this currently-running version (e.g. `1.32.0`) |
 | `--force-enabled` | Treat auto-update as enabled, ignoring the saved setting |
 | `--cooloff=<hours>` | Override the cool-off period |
 | `--window-start=<HH:MM>` / `--window-end=<HH:MM>` | Override the update window |
 | `--now=<ISO timestamp>` | Simulate the clock at a specific time |
 | `--releases-file=<path>` | Use a local JSON releases array instead of fetching GitHub (offline) |
 | `--skip-preflight` | Bypass the Docker/disk/queue pre-flight checks |
 ## Community & Resources
 - **Website:** [www.projectnomad.us](https://www.projectnomad.us) - Learn more about the project
 - **Discord:** [Join the Community](https://discord.com/invite/crosstalksolutions) - Get help, share your builds, and connect with other NOMAD users
 - **Benchmark Leaderboard:** [benchmark.projectnomad.us](https://benchmark.projectnomad.us) - See how your hardware stacks up against other NOMAD builds
 - **FAQ:** [FAQ.md](FAQ.md) - Find answers to frequently asked questions
 - **Community Add-Ons:** [admin/docs/community-add-ons.md](admin/docs/community-add-ons.md) - Third-party content packs built by the community
 ## License
 Project NOMAD is licensed under the [Apache License 2.0](LICENSE).
 ## Helper Scripts
-Once installed, Project NOMAD has a few helper scripts should you ever need to troubleshoot issues or perform maintenance that can't be done through the Command Center. All of these scripts are found in Project NOMAD's install directory, `/opt/project-nomad`
+Once installed, Project N.O.M.A.D. has a few helper scripts should you ever need to troubleshoot issues or perform maintenance that can't be done through the Command Center. All of these scripts are found in Project N.O.M.A.D.'s install directory, `/opt/project-nomad`
 ###
@ -180,7 +81,7 @@ sudo bash /opt/project-nomad/start_nomad.sh
 ###### Stop Script - Stops all installed project containers
 ```bash
-sudo bash /opt/project-nomad/stop_nomad.sh
+sudo bash /opt/project-nomad/start_nomad.sh
 ```
 ###
@ -191,5 +92,9 @@ sudo bash /opt/project-nomad/update_nomad.sh
 ###### Uninstall Script - Need to start fresh? Use the uninstall script to make your life easy. Note: this cannot be undone!
 ```bash
-curl -fsSL https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/install/uninstall_nomad.sh -o uninstall_nomad.sh && sudo bash uninstall_nomad.sh
+curl -fsSL https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/master/install/uninstall_nomad.sh -o uninstall_nomad.sh
 ```
 ```bash
 sudo bash uninstall_nomad.sh
 ```
--- a/admin/.env.example
+++ b/admin/.env.example
@ -1,10 +1,6 @@
 PORT=8080
 HOST=localhost
 LOG_LEVEL=info
 # Optional: override the URL used to test internet connectivity.
 # Defaults to https://1.1.1.1/cdn-cgi/trace with fallbacks to hosts the app
 # already contacts. Leave unset to use the defaults.
 # INTERNET_STATUS_TEST_URL=https://1.1.1.1/cdn-cgi/trace
 APP_KEY=some_random_key
 NODE_ENV=development
 SESSION_DRIVER=cookie
@ -16,15 +12,7 @@ DB_PASSWORD=password
 DB_SSL=false
 REDIS_HOST=localhost
 REDIS_PORT=6379
 # Optional: Redis logical database index (0-15). Defaults to 0 if unset.
 # Set this when sharing a Redis instance across services to avoid key collisions.
 # REDIS_DB=0
 # Storage path for NOMAD content (ZIM files, maps, etc.)
 # On Windows dev, use an absolute path like: C:/nomad-storage
 # On Linux production, use: /opt/project-nomad/storage
-NOMAD_STORAGE_PATH=/opt/project-nomad/storage
+NOMAD_STORAGE_PATH=/opt/project-nomad/storage
 # Creator Packs (gated content downloads). Official release builds inject
 # CREATOR_PACKS_APP_KEY at build time; leave unset for a build that can't install
 # packs. CREATOR_PACKS_WORKER_BASE overrides the entitlement Worker origin.
 # CREATOR_PACKS_APP_KEY=
 # CREATOR_PACKS_WORKER_BASE=https://nomad-packs-worker.chris-556.workers.dev
--- a/admin/README.md
+++ b/admin/README.md
@ -0,0 +1,5 @@
 ## Docker container
 ```
 docker run --rm -it -p 8080:8080 jturnercosmistack/projectnomad:admin-latest -e PORT=8080 -e HOST=0.0.0.0 -e APP_KEY=secretlongpasswordsecret -e LOG_LEVEL=debug -e DRIVE_DISK=fs
 ```
--- a/admin/adonisrc.ts
+++ b/admin/adonisrc.ts
@ -53,11 +53,7 @@ export default defineConfig({
    () => import('@adonisjs/lucid/database_provider'),
    () => import('@adonisjs/inertia/inertia_provider'),
    () => import('@adonisjs/transmit/transmit_provider'),
-    () => import('#providers/map_static_provider'),
+    () => import('#providers/map_static_provider')
    () => import('#providers/kiwix_migration_provider'),
    () => import('#providers/qdrant_restart_policy_provider'),
    () => import('#providers/version_check_provider'),
    () => import('#providers/gpu_passthrough_remediation_provider'),
  ],
  /*
@ -109,10 +105,6 @@ export default defineConfig({
      pattern: 'resources/views/**/*.edge',
      reloadServer: false,
    },
    {
      pattern: 'resources/geodata/**/*.geojson',
      reloadServer: false,
    },
    {
      pattern: 'public/**',
      reloadServer: false,
--- a/admin/app/controllers/benchmark_controller.ts
+++ b/admin/app/controllers/benchmark_controller.ts
@ -1,298 +0,0 @@
 import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
 import { BenchmarkService } from '#services/benchmark_service'
 import { runBenchmarkValidator, submitBenchmarkValidator } from '#validators/benchmark'
 import { RunBenchmarkJob } from '#jobs/run_benchmark_job'
 import type { BenchmarkType } from '../../types/benchmark.js'
 import { randomUUID } from 'node:crypto'
 import logger from '@adonisjs/core/services/logger'
@inject()
 export default class BenchmarkController {
  constructor(private benchmarkService: BenchmarkService) {}
  /**
   * Start a benchmark run (async via job queue, or sync if specified)
   */
  async run({ request, response }: HttpContext) {
    const payload = await request.validateUsing(runBenchmarkValidator)
    const benchmarkType: BenchmarkType = payload.benchmark_type || 'full'
    const runSync = request.input('sync') === 'true' || request.input('sync') === true
    // Check if a benchmark is already running
    const status = this.benchmarkService.getStatus()
    if (status.status !== 'idle') {
      return response.status(409).send({
        success: false,
        error: 'A benchmark is already running',
        current_benchmark_id: status.benchmarkId,
      })
    }
    // Run synchronously if requested (useful for local dev without Redis)
    if (runSync) {
      try {
        let result
        switch (benchmarkType) {
          case 'full':
            result = await this.benchmarkService.runFullBenchmark()
            break
          case 'system':
            result = await this.benchmarkService.runSystemBenchmarks()
            break
          case 'ai':
            result = await this.benchmarkService.runAIBenchmark()
            break
          default:
            result = await this.benchmarkService.runFullBenchmark()
        }
        return response.send({
          success: true,
          benchmark_id: result.benchmark_id,
          nomad_score: result.nomad_score,
          result,
        })
      } catch (error) {
        logger.error({ err: error }, '[BenchmarkController] Benchmark run failed')
        return response.status(500).send({
          success: false,
          error: 'An internal error occurred while running the benchmark.',
        })
      }
    }
    // Generate benchmark ID and dispatch job (async)
    const benchmarkId = randomUUID()
    const { job, created } = await RunBenchmarkJob.dispatch({
      benchmark_id: benchmarkId,
      benchmark_type: benchmarkType,
      include_ai: benchmarkType === 'full' || benchmarkType === 'ai',
    })
    return response.status(201).send({
      success: true,
      job_id: job?.id || benchmarkId,
      benchmark_id: benchmarkId,
      message: created
        ? `${benchmarkType} benchmark started`
        : 'Benchmark job already exists',
    })
  }
  /**
   * Run a system-only benchmark (CPU, memory, disk)
   */
  async runSystem({ response }: HttpContext) {
    const status = this.benchmarkService.getStatus()
    if (status.status !== 'idle') {
      return response.status(409).send({
        success: false,
        error: 'A benchmark is already running',
      })
    }
    const benchmarkId = randomUUID()
    await RunBenchmarkJob.dispatch({
      benchmark_id: benchmarkId,
      benchmark_type: 'system',
      include_ai: false,
    })
    return response.status(201).send({
      success: true,
      benchmark_id: benchmarkId,
      message: 'System benchmark started',
    })
  }
  /**
   * Run an AI-only benchmark
   */
  async runAI({ response }: HttpContext) {
    const status = this.benchmarkService.getStatus()
    if (status.status !== 'idle') {
      return response.status(409).send({
        success: false,
        error: 'A benchmark is already running',
      })
    }
    const benchmarkId = randomUUID()
    await RunBenchmarkJob.dispatch({
      benchmark_id: benchmarkId,
      benchmark_type: 'ai',
      include_ai: true,
    })
    return response.status(201).send({
      success: true,
      benchmark_id: benchmarkId,
      message: 'AI benchmark started',
    })
  }
  /**
   * Get all benchmark results
   */
  async results({}: HttpContext) {
    const results = await this.benchmarkService.getAllResults()
    return {
      results,
      total: results.length,
    }
  }
  /**
   * Get the latest benchmark result
   */
  async latest({}: HttpContext) {
    const result = await this.benchmarkService.getLatestResult()
    if (!result) {
      return { result: null }
    }
    return { result }
  }
  /**
   * Get a specific benchmark result by ID
   */
  async show({ params, response }: HttpContext) {
    const result = await this.benchmarkService.getResultById(params.id)
    if (!result) {
      return response.status(404).send({
        error: 'Benchmark result not found',
      })
    }
    return { result }
  }
  /**
   * Submit benchmark results to central repository
   */
  async submit({ request, response }: HttpContext) {
    const payload = await request.validateUsing(submitBenchmarkValidator)
    const anonymous = request.input('anonymous') === true || request.input('anonymous') === 'true'
    try {
      const submitResult = await this.benchmarkService.submitToRepository(payload.benchmark_id, anonymous)
      return response.send({
        success: true,
        repository_id: submitResult.repository_id,
        percentile: submitResult.percentile,
      })
    } catch (error) {
      // Pass through the status code from the service if available, otherwise default to 400
      const statusCode = (error as any).statusCode || 400
      logger.error({ err: error }, '[BenchmarkController] Benchmark submit failed')
      // Surface a clear, actionable reason to the UI instead of a generic failure.
      // The rate limiter (429) is the most common cause, so name it explicitly;
      // otherwise pass through the underlying detail (repository error or a service
      // validation message) and fall back to a safe generic only when we have none.
      let errorMessage: string
      if (statusCode === 429) {
        errorMessage = 'You can only submit one benchmark per hour. Please wait a bit and try again.'
      } else {
        errorMessage =
          (error as any).detail || (error as any).message || 'Failed to submit benchmark results.'
      }
      return response.status(statusCode).send({
        success: false,
        error: errorMessage,
      })
    }
  }
  /**
   * Update builder tag for a benchmark result
   */
  async updateBuilderTag({ request, response }: HttpContext) {
    const benchmarkId = request.input('benchmark_id')
    const builderTag = request.input('builder_tag')
    if (!benchmarkId) {
      return response.status(400).send({
        success: false,
        error: 'benchmark_id is required',
      })
    }
    const result = await this.benchmarkService.getResultById(benchmarkId)
    if (!result) {
      return response.status(404).send({
        success: false,
        error: 'Benchmark result not found',
      })
    }
    // Validate builder tag format if provided
    if (builderTag) {
      const tagPattern = /^[A-Za-z]+-[A-Za-z]+-\d{4}$/
      if (!tagPattern.test(builderTag)) {
        return response.status(400).send({
          success: false,
          error: 'Invalid builder tag format. Expected: Word-Word-0000',
        })
      }
    }
    result.builder_tag = builderTag || null
    await result.save()
    return response.send({
      success: true,
      builder_tag: result.builder_tag,
    })
  }
  /**
   * Get comparison stats from central repository
   */
  async comparison({}: HttpContext) {
    const stats = await this.benchmarkService.getComparisonStats()
    return { stats }
  }
  /**
   * Get current benchmark status
   */
  async status({}: HttpContext) {
    return this.benchmarkService.getStatus()
  }
  /**
   * Whether to show the dashboard "re-run under Score v2" banner
   */
  async rerunBanner({}: HttpContext) {
    return { show: await this.benchmarkService.shouldShowRerunBanner() }
  }
  /**
   * Get benchmark settings
   */
  async settings({}: HttpContext) {
    const { default: BenchmarkSetting } = await import('#models/benchmark_setting')
    return await BenchmarkSetting.getAllSettings()
  }
  /**
   * Update benchmark settings
   */
  async updateSettings({ request, response }: HttpContext) {
    const { default: BenchmarkSetting } = await import('#models/benchmark_setting')
    const body = request.body()
    if (body.allow_anonymous_submission !== undefined) {
      await BenchmarkSetting.setValue(
        'allow_anonymous_submission',
        body.allow_anonymous_submission ? 'true' : 'false'
      )
    }
    return response.send({
      success: true,
      settings: await BenchmarkSetting.getAllSettings(),
    })
  }
 }
--- a/admin/app/controllers/chats_controller.ts
+++ b/admin/app/controllers/chats_controller.ts
@ -1,120 +0,0 @@
 import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
 import { ChatService } from '#services/chat_service'
 import { createSessionSchema, updateSessionSchema, addMessageSchema } from '#validators/chat'
 import KVStore from '#models/kv_store'
 import { SystemService } from '#services/system_service'
 import { SERVICE_NAMES } from '../../constants/service_names.js'
 import logger from '@adonisjs/core/services/logger'
@inject()
 export default class ChatsController {
  constructor(private chatService: ChatService, private systemService: SystemService) {}
  async inertia({ inertia, response }: HttpContext) {
    const aiAssistantInstalled = await this.systemService.checkServiceInstalled(SERVICE_NAMES.OLLAMA)
    if (!aiAssistantInstalled) {
      return response.status(404).json({ error: 'AI Assistant service not installed' })
    }
    const chatSuggestionsEnabled = await KVStore.getValue('chat.suggestionsEnabled')
    return inertia.render('chat', {
      settings: {
        chatSuggestionsEnabled: chatSuggestionsEnabled ?? false,
      },
    })
  }
  async index({}: HttpContext) {
    return await this.chatService.getAllSessions()
  }
  async show({ params, response }: HttpContext) {
    const sessionId = parseInt(params.id)
    const session = await this.chatService.getSession(sessionId)
    if (!session) {
      return response.status(404).json({ error: 'Session not found' })
    }
    return session
  }
  async store({ request, response }: HttpContext) {
    try {
      const data = await request.validateUsing(createSessionSchema)
      const session = await this.chatService.createSession(data.title, data.model)
      return response.status(201).json(session)
    } catch (error) {
      logger.error({ err: error }, '[ChatsController] Failed to create session')
      return response.status(500).json({
        error: 'Failed to create session',
      })
    }
  }
  async suggestions({ response }: HttpContext) {
    try {
      const suggestions = await this.chatService.getChatSuggestions()
      return response.status(200).json({ suggestions })
    } catch (error) {
      logger.error({ err: error }, '[ChatsController] Failed to get suggestions')
      return response.status(500).json({
        error: 'Failed to get suggestions',
      })
    }
  }
  async update({ params, request, response }: HttpContext) {
    try {
      const sessionId = parseInt(params.id)
      const data = await request.validateUsing(updateSessionSchema)
      const session = await this.chatService.updateSession(sessionId, data)
      return session
    } catch (error) {
      logger.error({ err: error }, '[ChatsController] Failed to update session')
      return response.status(500).json({
        error: 'Failed to update session',
      })
    }
  }
  async destroy({ params, response }: HttpContext) {
    try {
      const sessionId = parseInt(params.id)
      await this.chatService.deleteSession(sessionId)
      return response.status(204)
    } catch (error) {
      logger.error({ err: error }, '[ChatsController] Failed to delete session')
      return response.status(500).json({
        error: 'Failed to delete session',
      })
    }
  }
  async addMessage({ params, request, response }: HttpContext) {
    try {
      const sessionId = parseInt(params.id)
      const data = await request.validateUsing(addMessageSchema)
      const message = await this.chatService.addMessage(sessionId, data.role, data.content)
      return response.status(201).json(message)
    } catch (error) {
      logger.error({ err: error }, '[ChatsController] Failed to add message')
      return response.status(500).json({
        error: 'Failed to add message',
      })
    }
  }
  async destroyAll({ response }: HttpContext) {
    try {
      const result = await this.chatService.deleteAllSessions()
      return response.status(200).json(result)
    } catch (error) {
      logger.error({ err: error }, '[ChatsController] Failed to delete all sessions')
      return response.status(500).json({
        error: 'Failed to delete all sessions',
      })
    }
  }
 }
--- a/admin/app/controllers/collection_updates_controller.ts
+++ b/admin/app/controllers/collection_updates_controller.ts
@ -1,30 +0,0 @@
 import { CollectionUpdateService } from '#services/collection_update_service'
 import {
  assertNotPrivateUrl,
  applyContentUpdateValidator,
  applyAllContentUpdatesValidator,
 } from '#validators/common'
 import type { HttpContext } from '@adonisjs/core/http'
 export default class CollectionUpdatesController {
  async checkForUpdates({}: HttpContext) {
    const service = new CollectionUpdateService()
    return await service.checkForUpdates()
  }
  async applyUpdate({ request }: HttpContext) {
    const update = await request.validateUsing(applyContentUpdateValidator)
    assertNotPrivateUrl(update.download_url)
    const service = new CollectionUpdateService()
    return await service.applyUpdate(update)
  }
  async applyAllUpdates({ request }: HttpContext) {
    const { updates } = await request.validateUsing(applyAllContentUpdatesValidator)
    for (const update of updates) {
      assertNotPrivateUrl(update.download_url)
    }
    const service = new CollectionUpdateService()
    return await service.applyAllUpdates(updates)
  }
 }
--- a/admin/app/controllers/conditions_controller.ts
+++ b/admin/app/controllers/conditions_controller.ts
@ -1,114 +0,0 @@
 import type { HttpContext } from '@adonisjs/core/http'
 import logger from '@adonisjs/core/services/logger'
 import { ConditionService } from '#services/condition_service'
 import { conditionDrugsValidator } from '#validators/conditions'
 import { affirmativeRemediesEnabled } from '../utils/affirmative_remedies.js'
 /**
 * "When to use what" — condition-first HTTP boundary (Phase 1).
 *
 * Two Inertia pages (index / show) + a small JSON API (drugs). Mirrors the
 * DrugReferenceController chain:
 *   - index/show render Inertia
 *   - the JSON action returns a plain object
 *   - slug guard on show (404 when not in the curated spine)
 *   - never leak exceptions to the UI
 */
 export default class ConditionsController {
  private get service() {
    return new ConditionService()
  }
  /**
   * GET /conditions — legacy browse route.
   * Situation browsing now lives directly on the unified Drug Reference page, so
   * the standalone browse route permanently redirects there. Any old bookmark or
   * in-app link lands on the same content. The condition detail route
   * (/conditions/:slug) is unchanged — situation chips still deep-link to it via
   * /drug-reference?situation=<slug>.
   */
  async index({ response }: HttpContext) {
    return response.redirect('/drug-reference')
  }
  /**
   * GET /conditions/:slug — condition detail page.
   * 404s when the slug is not a curated condition.
   */
  async show({ inertia, params, response }: HttpContext) {
    const slug = String(params.slug ?? '').trim()
    if (!slug) {
      return response.notFound({ error: 'invalid condition' })
    }
    try {
      const condition = this.service.findCondition(slug)
      if (!condition) {
        return response.notFound({ error: 'Condition not found' })
      }
      const [result, drugRowCount, remediesOn] = await Promise.all([
        this.service.drugsForSlug(slug),
        this.service.drugRowCount(),
        affirmativeRemediesEnabled(),
      ])
      return inertia.render('conditions/show', {
        condition: result?.condition ?? null,
        drugs: result?.drugs ?? [],
        // Affirmative remedies gated off by default (#1040); the OTC/condition
        // match above is regulated label text and stays live.
        remedies: remediesOn ? (result?.remedies ?? []) : [],
        drugRowCount,
      })
    } catch (err) {
      const msg = err instanceof Error ? err.message : String(err)
      logger.error(`[ConditionsController] show(${slug}) failed: ${msg}`)
      return response.internalServerError({ error: 'Could not load condition' })
    }
  }
  /**
   * GET /api/conditions/drugs?slug=… | ?q=…
   * Returns { condition, drugs } for a curated condition or a free-text
   * situation. Requires exactly one of slug/q.
   */
  async drugsApi({ request, response }: HttpContext) {
    try {
      const params = await request.validateUsing(conditionDrugsValidator)
      if (params.slug && params.q) {
        return response.badRequest({ error: 'Provide either slug or q, not both' })
      }
      const filterOpts = {
        route: params.route,
        sort: params.sort,
      }
      // Strip affirmative remedies from the situation-search response when the
      // gate is closed (#1040); the OTC drug matches are regulated label text and
      // are returned either way.
      const remediesOn = await affirmativeRemediesEnabled()
      if (params.slug) {
        const result = await this.service.drugsForSlug(params.slug, params.limit, filterOpts)
        if (!result) {
          return response.notFound({ error: 'Condition not found' })
        }
        return remediesOn ? result : { ...result, remedies: [] }
      }
      if (params.q) {
        const result = await this.service.drugsForFreeText(params.q, params.limit, filterOpts)
        return remediesOn ? result : { ...result, remedies: [] }
      }
      return response.badRequest({ error: 'Provide a slug or q query parameter' })
    } catch (err) {
      const msg = err instanceof Error ? err.message : String(err)
      logger.warn(`[ConditionsController] drugsApi failed: ${msg}`)
      return response.badRequest({ error: msg })
    }
  }
 }
--- a/admin/app/controllers/creator_packs_controller.ts
+++ b/admin/app/controllers/creator_packs_controller.ts
@ -1,62 +0,0 @@
 import { CreatorPackService } from '#services/creator_pack_service'
 import { DownloadService } from '#services/download_service'
 import { QueueService } from '#services/queue_service'
 import logger from '@adonisjs/core/services/logger'
 import type { HttpContext } from '@adonisjs/core/http'
 export default class CreatorPacksController {
  private creatorPackService = new CreatorPackService()
  /**
   * Catalog with per-pack status, plus live progress for any in-flight pack
   * downloads (ZIM download jobs — filtered client-side by collection_ref once
   * the UI needs it; for now the raw zim job list is enough for progress).
   */
  async index({ response }: HttpContext) {
    try {
      const configured = this.creatorPackService.isConfigured()
      const packs = await this.creatorPackService.listPacksWithStatus()
      const downloadService = new DownloadService(QueueService.getInstance())
      const downloads = await downloadService.listDownloadJobs('zim')
      return { configured, packs, downloads }
    } catch (error: any) {
      logger.error('[CreatorPacksController] Failed to list creator packs:', error?.message || error)
      return response.status(500).send({ message: 'Failed to load creator packs' })
    }
  }
  async install({ params, response }: HttpContext) {
    const packId = params.id as string
    const result = await this.creatorPackService.installPack(packId)
    switch (result.code) {
      case 'dispatched':
        return response.status(202).send({
          message: 'Pack download started',
          filename: result.filename,
        })
      case 'already_installed':
        return { message: 'Pack is already installed' }
      case 'already_downloading':
        return { message: 'Pack download is already in progress' }
      case 'not_found':
        return response.status(404).send({ message: `Creator pack not found: ${packId}` })
      case 'not_configured':
        return response.status(503).send({
          message: 'Creator Packs are not configured on this server',
        })
    }
  }
  async uninstall({ params, response }: HttpContext) {
    const packId = params.id as string
    const result = await this.creatorPackService.uninstallPack(packId)
    switch (result.code) {
      case 'uninstalled':
        return { message: 'Pack uninstalled', filename: result.filename }
      case 'not_installed':
        return response.status(404).send({ message: `Creator pack is not installed: ${packId}` })
    }
  }
 }
--- a/admin/app/controllers/downloads_controller.ts
+++ b/admin/app/controllers/downloads_controller.ts
@ -15,17 +15,4 @@ export default class DownloadsController {
    const payload = await request.validateUsing(downloadJobsByFiletypeSchema)
    return this.downloadService.listDownloadJobs(payload.params.filetype)
  }
  async removeJob({ params }: HttpContext) {
    await this.downloadService.removeFailedJob(params.jobId)
    return { success: true }
  }
  async cancelJob({ params }: HttpContext) {
    return this.downloadService.cancelJob(params.jobId)
  }
  async retryJob({ params }: HttpContext) {
    return this.downloadService.retryFailedJob(params.jobId)
  }
 }
--- a/admin/app/controllers/drug_reference_controller.ts
+++ b/admin/app/controllers/drug_reference_controller.ts
@ -1,295 +0,0 @@
 import type { HttpContext } from '@adonisjs/core/http'
 import logger from '@adonisjs/core/services/logger'
 import { DrugReferenceService } from '#services/drug_reference_service'
 import { ConditionService } from '#services/condition_service'
 import { searchDrugValidator, interactionsValidator } from '#validators/drug_reference'
 import { parseCompareIds } from '../../util/compare_ids.js'
 import { situationsForIndications } from '../../util/conditions.js'
 import { affirmativeRemediesEnabled } from '../utils/affirmative_remedies.js'
 /**
 * Drug Reference v1 — HTTP boundary.
 *
 * Two Inertia pages (index / show) + a small JSON API (search / status /
 * download). Mirrors the WorkshopController / InventoryController chain:
 *   - index/show render Inertia
 *   - JSON actions return plain objects
 *   - Integer-id guard on show
 *   - Never leak exceptions to the UI
 */
 export default class DrugReferenceController {
  private get service() {
    return new DrugReferenceService()
  }
  /**
   * GET /drug-reference — unified search page.
   * Passes the current row count and ingest status so the empty-state
   * "download first" prompt can render server-side. Also passes the curated
   * condition spine so the always-visible situation chips (and the situation→
   * drugs direction of the unified surface) can render server-side.
   */
  async index({ inertia }: HttpContext) {
    try {
      const conditionService = new ConditionService()
      const [status, count, remediesOn] = await Promise.all([
        this.service.getIngestStatus(),
        this.service.rowCount(),
        affirmativeRemediesEnabled(),
      ])
      return inertia.render('drug-reference/index', {
        ingestStatus: status,
        rowCount: count,
        conditions: conditionService.listConditions(),
        // Affirmative remedy content is gated off by default (#1040): keep it out
        // of the payload entirely when disabled, and tell the page so it can hide
        // the "Natural" filter too. Drug search + condition matching are unaffected.
        remedies: remediesOn ? conditionService.listRemedies() : [],
        remediesEnabled: remediesOn,
      })
    } catch (err) {
      const msg = err instanceof Error ? err.message : String(err)
      logger.error(`[DrugReferenceController] index failed: ${msg}`)
      return inertia.render('drug-reference/index', {
        ingestStatus: null,
        rowCount: 0,
        conditions: [],
        remedies: [],
        remediesEnabled: false,
      })
    }
  }
  /**
   * GET /drug-reference/:id — detail page.
   */
  async show({ inertia, params, response }: HttpContext) {
    const id = Number(params.id)
    if (!Number.isInteger(id) || id <= 0) {
      return response.notFound({ error: 'invalid id' })
    }
    try {
      const label = await this.service.find(id)
      if (!label) {
        return response.notFound({ error: 'Drug label not found' })
      }
      // Reverse link — the other direction of the symbiotic relationship: which
      // curated situations does THIS label's indications text treat? Matched
      // server-side against the curated spine so searchTerms stay server-only.
      const situations = situationsForIndications(
        label.indications,
        new ConditionService().allConditions()
      )
      return inertia.render('drug-reference/show', { label, situations })
    } catch (err) {
      const msg = err instanceof Error ? err.message : String(err)
      logger.error(`[DrugReferenceController] show(${id}) failed: ${msg}`)
      return response.internalServerError({ error: 'Could not load drug label' })
    }
  }
  /**
   * GET /api/drug-reference/search
   * Returns a slim collapsed result list (brand+generic pairs).
   */
  async search({ request, response }: HttpContext) {
    try {
      const params = await request.validateUsing(searchDrugValidator)
      const results = await this.service.search(params.q, {
        productType: params.product_type,
        route: params.route,
        sort: params.sort,
        limit: params.limit,
        offset: params.offset,
        scope: params.scope,
      })
      return { results }
    } catch (err) {
      const msg = err instanceof Error ? err.message : String(err)
      logger.warn(`[DrugReferenceController] search failed: ${msg}`)
      return response.badRequest({ error: msg })
    }
  }
  /**
   * GET /api/drug-reference/status
   * Returns the live ingest status DTO.
   */
  async status({ response }: HttpContext) {
    try {
      const status = await this.service.getIngestStatus()
      return status
    } catch (err) {
      const msg = err instanceof Error ? err.message : String(err)
      logger.error(`[DrugReferenceController] status failed: ${msg}`)
      return response.internalServerError({ error: 'Could not read ingest status' })
    }
  }
  /**
   * GET /drug-reference/interactions — side-by-side label comparison page.
   * Passes rowCount + ingestStatus so the empty-state prompt can render,
   * mirroring the index() pattern. The actual entry data is loaded client-side
   * via /api/drug-reference/interactions?ids=… so the page is shareable via URL.
   */
  async interactions({ inertia }: HttpContext) {
    try {
      const [status, count] = await Promise.all([
        this.service.getIngestStatus(),
        this.service.rowCount(),
      ])
      return inertia.render('drug-reference/interactions', {
        ingestStatus: status,
        rowCount: count,
      })
    } catch (err) {
      const msg = err instanceof Error ? err.message : String(err)
      logger.error(`[DrugReferenceController] interactions page failed: ${msg}`)
      return inertia.render('drug-reference/interactions', {
        ingestStatus: null,
        rowCount: 0,
      })
    }
  }
  /**
   * GET /api/drug-reference/interactions?ids=1,2,3
   * Validates → parses → fetches and returns { entries: DrugInteractionEntry[] }.
   * Never leaks exceptions; integer-guards ids via parseCompareIds.
   */
  async interactionsApi({ request, response }: HttpContext) {
    try {
      const params = await request.validateUsing(interactionsValidator)
      const ids = parseCompareIds(params.ids ?? '')
      const entries = await this.service.getInteractionsFor(ids)
      return { entries }
    } catch (err) {
      const msg = err instanceof Error ? err.message : String(err)
      logger.warn(`[DrugReferenceController] interactionsApi failed: ${msg}`)
      return response.badRequest({ error: msg })
    }
  }
  /**
   * POST /api/drug-reference/download
   * Triggers the download phase (idempotent — deduped on deterministic jobId).
   * The download auto-chains the ingest phase on completion.
   */
  async download({ response }: HttpContext) {
    try {
      const result = await this.service.triggerDownload()
      return { success: true, created: result.created, message: result.message }
    } catch (err) {
      const msg = err instanceof Error ? err.message : String(err)
      logger.error(`[DrugReferenceController] download trigger failed: ${msg}`)
      return response.internalServerError({ error: 'Could not trigger download' })
    }
  }
  /**
   * POST /api/drug-reference/ingest
   * Manually (re-)runs the ingest phase from the already-downloaded on-disk
   * parts, with no re-download. Returns 404 when nothing is on disk so the UI
   * can keep its guard honest even if the button is reached out of band.
   */
  async ingest({ response }: HttpContext) {
    try {
      const result = await this.service.triggerIngestFromDisk()
      if (result.nothingDownloaded) {
        return response.notFound({ error: result.message })
      }
      return { success: true, created: result.created, message: result.message }
    } catch (err) {
      const msg = err instanceof Error ? err.message : String(err)
      logger.error(`[DrugReferenceController] ingest trigger failed: ${msg}`)
      return response.internalServerError({ error: 'Could not trigger ingest' })
    }
  }
  /**
   * POST /api/drug-reference/reset-ingest
   * Force-clears a wedged ingest job (e.g. one left 'active' by a worker killed
   * mid-ingest during an upgrade) and restarts it from the on-disk parts. The
   * escape hatch for a stuck "Indexing…" state.
   */
  async resetIngest({ response }: HttpContext) {
    try {
      const result = await this.service.resetAndReingest()
      if (result.nothingDownloaded) {
        return response.notFound({ error: result.message })
      }
      return { success: true, created: result.created, message: result.message }
    } catch (err) {
      const msg = err instanceof Error ? err.message : String(err)
      logger.error(`[DrugReferenceController] reset-ingest failed: ${msg}`)
      return response.internalServerError({ error: 'Could not reset ingest' })
    }
  }
  /**
   * POST /api/drug-reference/uninstall
   * Uninstall the offline FDA drug dataset: stop in-flight jobs, delete on-disk
   * parts, TRUNCATE drug_labels, clear KV markers, and remove the install-state
   * row (which auto-hides the home tiles). The curated-tier "remove" action.
   * Reports partial failures rather than masking them.
   */
  async uninstall({ response }: HttpContext) {
    try {
      const result = await this.service.uninstall()
      if (!result.success) {
        return response.internalServerError({
          success: false,
          rowsDropped: result.rowsDropped,
          error: result.message,
        })
      }
      return { success: true, rowsDropped: result.rowsDropped, message: result.message }
    } catch (err) {
      const msg = err instanceof Error ? err.message : String(err)
      logger.error(`[DrugReferenceController] uninstall failed: ${msg}`)
      return response.internalServerError({ error: 'Could not uninstall drug reference' })
    }
  }
  /**
   * GET /api/drug-reference/ingest-log
   * Tail the persisted app log for ingest/download lines. In production the logger
   * writes JSON to /app/storage/logs/admin.log (both the admin and worker
   * containers share that volume), so the worker's [IngestDrugDataJob] trace lands
   * there even when its stdout never reaches the log viewer. This surfaces it over
   * HTTP so the exact stall stage (zip-open vs first-record vs batch) is visible
   * without container access. Reads only the last slice of the file.
   */
  async ingestLog({ request, response }: HttpContext) {
    const LOG_PATH = '/app/storage/logs/admin.log'
    const TAIL_BYTES = 128 * 1024
    const limit = Math.min(Number(request.input('lines', 400)) || 400, 2000)
    try {
      const { stat, open } = await import('node:fs/promises')
      const st = await stat(LOG_PATH)
      const start = Math.max(0, st.size - TAIL_BYTES)
      const fh = await open(LOG_PATH, 'r')
      try {
        const buf = Buffer.alloc(st.size - start)
        await fh.read(buf, 0, buf.length, start)
        const all = buf.toString('utf8').split('\n')
        // Keep only ingest/download/worker-relevant lines; for JSON pino lines the
        // substring match still works against the embedded "msg" field.
        const re =
          /IngestDrugDataJob|DownloadDrugDataJob|DrugReference|drug-ingest|drug-download|unhandledRejection|uncaughtException|queue:work|stalled/i
        const matched = all.filter((l) => re.test(l)).slice(-limit)
        return { ok: true, path: LOG_PATH, size: st.size, count: matched.length, lines: matched }
      } finally {
        await fh.close()
      }
    } catch (err) {
      const msg = err instanceof Error ? err.message : String(err)
      return response.notFound({ ok: false, path: LOG_PATH, error: msg })
    }
  }
 }
--- a/admin/app/controllers/easy_setup_controller.ts
+++ b/admin/app/controllers/easy_setup_controller.ts
@ -1,7 +1,5 @@
 import { SystemService } from '#services/system_service'
 import { ZimService } from '#services/zim_service'
 import { CollectionManifestService } from '#services/collection_manifest_service'
 import KVStore from '#models/kv_store'
 import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
@ -13,14 +11,10 @@ export default class EasySetupController {
  ) {}
  async index({ inertia }: HttpContext) {
-    const [services, remoteOllamaUrl] = await Promise.all([
+    const services = await this.systemService.getServices({ installedOnly: false })
      this.systemService.getServices({ installedOnly: false }),
      KVStore.getValue('ai.remoteOllamaUrl'),
    ])
    return inertia.render('easy-setup/index', {
      system: {
        services: services,
        remoteOllamaUrl: remoteOllamaUrl ?? '',
      },
    })
  }
@ -32,22 +26,4 @@ export default class EasySetupController {
  async listCuratedCategories({}: HttpContext) {
    return await this.zimService.listCuratedCategories()
  }
  async refreshManifests({}: HttpContext) {
    const manifestService = new CollectionManifestService()
    const [zimChanged, mapsChanged, wikiChanged] = await Promise.all([
      manifestService.fetchAndCacheSpec('zim_categories'),
      manifestService.fetchAndCacheSpec('maps'),
      manifestService.fetchAndCacheSpec('wikipedia'),
    ])
    return {
      success: true,
      changed: {
        zim_categories: zimChanged,
        maps: mapsChanged,
        wikipedia: wikiChanged,
      },
    }
  }
 }
--- a/admin/app/controllers/home_controller.ts
+++ b/admin/app/controllers/home_controller.ts
@ -1,6 +1,4 @@
 import { SystemService } from '#services/system_service'
 import { DrugReferenceService } from '#services/drug_reference_service'
 import logger from '@adonisjs/core/services/logger'
 import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
@ -20,38 +18,7 @@ export default class HomeController {
        return inertia.render('home', {
            system: {
                services
-            },
+            }
            // Gate the Drug Reference / "When to use what" tiles behind the FDA
            // dataset install state. Installed when the curated-tier ingest has
            // reached 'ready' OR an install is in flight (downloading/ingesting) —
            // so the tile appears the moment the user opts in and persists through
            // the long install, rather than popping in only at the very end.
            drugReferenceInstalled: await this.computeDrugReferenceInstalled(),
        })
    }
    /**
     * True when the offline FDA drug dataset is installed or installing. Reads the
     * two-phase ingest status: ready (fully installed) or an active phase
     * (downloading/downloaded/ingesting). rowCount > 0 covers a populated table
     * whose job history was pruned. Never throws — a status read failure hides the
     * tiles (fail-closed) rather than 500-ing the dashboard.
     */
    private async computeDrugReferenceInstalled(): Promise<boolean> {
        try {
            const status = await new DrugReferenceService().getIngestStatus()
            const installing =
                status.phase === 'downloading' ||
                status.phase === 'downloaded' ||
                status.phase === 'ingesting'
            return status.phase === 'ready' || installing || status.rowCount > 0
        } catch (err) {
            logger.error(
                `[HomeController] drug-reference install check failed: ${
                    err instanceof Error ? err.message : String(err)
                }`
            )
            return false
        }
    }
 }
--- a/admin/app/controllers/maps_controller.ts
+++ b/admin/app/controllers/maps_controller.ts
@ -1,65 +1,41 @@
 import { MapService } from '#services/map_service'
 import MapMarker from '#models/map_marker'
 import {
  assertNotPrivateUrl,
  downloadCollectionValidator,
  filenameParamValidator,
  mapExtractPreflightValidator,
  mapExtractValidator,
  remoteDownloadValidator,
  remoteDownloadValidatorOptional,
 } from '#validators/common'
 import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
 import vine from '@vinejs/vine'
@inject()
 export default class MapsController {
  constructor(private mapService: MapService) {}
  async index({ inertia }: HttpContext) {
-    const baseAssetsCheck = await this.mapService.ensureBaseAssets()
+    const baseAssetsCheck = await this.mapService.checkBaseAssetsExist()
-    const [regionFiles, worldBasemapExists] = await Promise.all([
+    const regionFiles = await this.mapService.listRegions()
      this.mapService.listRegions(),
      this.mapService.checkWorldBasemapExists(),
    ])
    return inertia.render('maps', {
      maps: {
        baseAssetsExist: baseAssetsCheck,
        worldBasemapExists,
        regionFiles: regionFiles.files,
      },
    })
  }
  async checkBaseAssets({}: HttpContext) {
    const exists = await this.mapService.checkBaseAssetsExist()
    return { exists }
  }
  async downloadBaseAssets({ request }: HttpContext) {
    const payload = await request.validateUsing(remoteDownloadValidatorOptional)
    if (payload.url) assertNotPrivateUrl(payload.url)
    await this.mapService.downloadBaseAssets(payload.url)
    return { success: true }
  }
  async setupWorldBasemap({ response }: HttpContext) {
    try {
      const ready = await this.mapService.provisionWorldBasemap()
      if (!ready) {
        return response.status(500).send({
          message:
            'Could not download the base map. Please connect this NOMAD to the internet and try again.',
        })
      }
      return { success: true }
    } catch {
      return response.status(500).send({
        message:
          'Could not download the base map. Please connect this NOMAD to the internet and try again.',
      })
    }
  }
  async downloadRemote({ request }: HttpContext) {
    const payload = await request.validateUsing(remoteDownloadValidator)
    assertNotPrivateUrl(payload.url)
    const filename = await this.mapService.downloadRemote(payload.url)
    return {
      message: 'Download started successfully',
@ -81,7 +57,6 @@ export default class MapsController {
  // For providing a "preflight" check in the UI before actually starting a background download
  async downloadRemotePreflight({ request }: HttpContext) {
    const payload = await request.validateUsing(remoteDownloadValidator)
    assertNotPrivateUrl(payload.url)
    const info = await this.mapService.downloadRemotePreflight(payload.url)
    return info
  }
@ -99,57 +74,8 @@ export default class MapsController {
    return await this.mapService.listRegions()
  }
-  async globalMapInfo({}: HttpContext) {
+  async styles({ response }: HttpContext) {
-    return await this.mapService.getGlobalMapInfo()
+    const styles = await this.mapService.generateStylesJSON()
  }
  async downloadGlobalMap({}: HttpContext) {
    const result = await this.mapService.downloadGlobalMap()
    return {
      message: 'Download started successfully',
      ...result,
    }
  }
  async listCountries({}: HttpContext) {
    return { countries: await this.mapService.listCountries() }
  }
  async listCountryGroups({}: HttpContext) {
    return { groups: await this.mapService.listCountryGroups() }
  }
  async extractPreflight({ request }: HttpContext) {
    const payload = await request.validateUsing(mapExtractPreflightValidator)
    return await this.mapService.extractPreflight(payload)
  }
  async extractRegion({ request }: HttpContext) {
    const payload = await request.validateUsing(mapExtractValidator)
    const result = await this.mapService.extractRegion(payload)
    return {
      message: 'Extract started successfully',
      ...result,
    }
  }
  async styles({ request, response }: HttpContext) {
    // Automatically ensure base assets are present before generating styles
    const baseAssetsExist = await this.mapService.ensureBaseAssets()
    if (!baseAssetsExist) {
      return response.status(500).send({
        message:
          'Base map assets are missing and could not be downloaded. Please check your connection and try again.',
      })
    }
    const forwardedProto = request.headers()['x-forwarded-proto'];
    const protocol: string = forwardedProto
      ? (typeof forwardedProto === 'string' ? forwardedProto : request.protocol())
      : request.protocol();
    const styles = await this.mapService.generateStylesJSON(request.host(), protocol)
    return response.json(styles)
  }
@ -171,72 +97,4 @@ export default class MapsController {
      message: 'Map file deleted successfully',
    }
  }
  // --- Map Markers ---
  async listMarkers({}: HttpContext) {
    return await MapMarker.query().orderBy('created_at', 'asc')
  }
  async createMarker({ request }: HttpContext) {
    const payload = await request.validateUsing(
      vine.compile(
        vine.object({
          name: vine.string().trim().minLength(1).maxLength(255),
          longitude: vine.number().min(-180).max(180),
          latitude: vine.number().min(-90).max(90),
          color: vine.string().trim().maxLength(20).optional(),
          notes: vine.string().trim().nullable().optional(),
          marker_type: vine.string().trim().maxLength(20).optional(),
        })
      )
    )
    const marker = await MapMarker.create({
      name: payload.name,
      longitude: payload.longitude,
      latitude: payload.latitude,
      color: payload.color ?? 'orange',
      notes: payload.notes ?? null,
      marker_type: payload.marker_type ?? 'pin',
    })
    return marker
  }
  async updateMarker({ request, response }: HttpContext) {
    const { id } = request.params()
    const marker = await MapMarker.find(id)
    if (!marker) {
      return response.status(404).send({ message: 'Marker not found' })
    }
    const payload = await request.validateUsing(
      vine.compile(
        vine.object({
          name: vine.string().trim().minLength(1).maxLength(255).optional(),
          color: vine.string().trim().maxLength(20).optional(),
          longitude: vine.number().min(-180).max(180).optional(),
          latitude: vine.number().min(-90).max(90).optional(),
          notes: vine.string().trim().nullable().optional(),
          marker_type: vine.string().trim().maxLength(20).optional(),
        })
      )
    )
    if (payload.name !== undefined) marker.name = payload.name
    if (payload.color !== undefined) marker.color = payload.color
    if (payload.longitude !== undefined) marker.longitude = payload.longitude
    if (payload.latitude !== undefined) marker.latitude = payload.latitude
    if (payload.notes !== undefined) marker.notes = payload.notes
    if (payload.marker_type !== undefined) marker.marker_type = payload.marker_type
    await marker.save()
    return marker
  }
  async deleteMarker({ request, response }: HttpContext) {
    const { id } = request.params()
    const marker = await MapMarker.find(id)
    if (!marker) {
      return response.status(404).send({ message: 'Marker not found' })
    }
    await marker.delete()
    return { message: 'Marker deleted' }
  }
 }
--- a/admin/app/controllers/nomad_md_controller.ts
+++ b/admin/app/controllers/nomad_md_controller.ts
@ -1,22 +0,0 @@
 import { NomadMdService } from '#services/nomad_md_service'
 import { updateNomadMdSchema } from '#validators/nomad_md'
 import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
@inject()
 export default class NomadMdController {
  constructor(private nomadMdService: NomadMdService) {}
  async show({ response }: HttpContext) {
    const content = await this.nomadMdService.read()
    return response.status(200).json({ content })
  }
  async update({ request, response }: HttpContext) {
    const { content } = await request.validateUsing(updateNomadMdSchema)
    // Empty request strings arrive as undefined (AdonisJS null-coerces them);
    // treat that as an explicit "clear the file".
    await this.nomadMdService.write(content ?? '')
    return response.status(200).json({ success: true, message: 'NOMAD.md saved successfully' })
  }
 }
--- a/admin/app/controllers/ollama_controller.ts
+++ b/admin/app/controllers/ollama_controller.ts
@ -1,507 +0,0 @@
 import { ChatService } from '#services/chat_service'
 import { DockerService } from '#services/docker_service'
 import { NomadMdService } from '#services/nomad_md_service'
 import { OllamaService } from '#services/ollama_service'
 import { RagService } from '#services/rag_service'
 import Service from '#models/service'
 import KVStore from '#models/kv_store'
 import { modelNameSchema } from '#validators/download'
 import { chatSchema, getAvailableModelsSchema, unloadChatModelsSchema } from '#validators/ollama'
 import { assertNotCloudMetadataUrl } from '#validators/common'
 import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
 import { RAG_CONTEXT_LIMITS, SYSTEM_PROMPTS } from '../../constants/ollama.js'
 import { SERVICE_NAMES } from '../../constants/service_names.js'
 import logger from '@adonisjs/core/services/logger'
 type Message = { role: 'system' | 'user' | 'assistant'; content: string }
@inject()
 export default class OllamaController {
  constructor(
    private chatService: ChatService,
    private dockerService: DockerService,
    private ollamaService: OllamaService,
    private ragService: RagService,
    private nomadMdService: NomadMdService
  ) { }
  async availableModels({ request }: HttpContext) {
    const reqData = await request.validateUsing(getAvailableModelsSchema)
    return await this.ollamaService.getAvailableModels({
      sort: reqData.sort,
      recommendedOnly: reqData.recommendedOnly,
      query: reqData.query || null,
      limit: reqData.limit || 15,
      force: reqData.force,
    })
  }
  /**
   * Send Ollama `keep_alive: 0` hints to every currently-loaded chat model
   * except the embedding model and (optionally) a target model to preserve.
   * Used by the chat UI to enforce the "one chat model at a time" invariant
   * on model-switch, session-switch, and page-load. Best-effort: a failure
   * here should not block the calling flow.
   */
  async unloadChatModels({ request, response }: HttpContext) {
    const { targetModel } = await request.validateUsing(unloadChatModelsSchema)
    const unloaded = await this.ollamaService.unloadAllChatModelsExcept(targetModel ?? null)
    return response.status(200).json({ unloaded })
  }
  async chat({ request, response }: HttpContext) {
    const reqData = await request.validateUsing(chatSchema)
    // Flush SSE headers immediately so the client connection is open while
    // pre-processing (query rewriting, RAG lookup) runs in the background.
    if (reqData.stream) {
      response.response.setHeader('Content-Type', 'text/event-stream')
      response.response.setHeader('Cache-Control', 'no-cache')
      response.response.setHeader('Connection', 'keep-alive')
      response.response.flushHeaders()
    }
    try {
      // If there are no system messages in the chat inject system prompts
      const hasSystemMessage = reqData.messages.some((msg) => msg.role === 'system')
      if (!hasSystemMessage) {
        const systemPrompt = {
          role: 'system' as const,
          content: SYSTEM_PROMPTS.default,
        }
        logger.debug('[OllamaController] Injecting system prompt')
        reqData.messages.unshift(systemPrompt)
      }
      // Inject the user-managed NOMAD.md as its own leading system message so the
      // user's persistent instructions take precedence, while the default
      // formatting prompt and any RAG context below remain intact. A missing or
      // blank file yields null and changes nothing.
      const nomadPrompt = await this.nomadMdService.getSystemPrompt()
      if (nomadPrompt) {
        logger.debug('[OllamaController] Injecting NOMAD.md system prompt')
        reqData.messages.unshift({ role: 'system' as const, content: nomadPrompt })
      }
      // Query rewriting for better RAG retrieval with manageable context
      // Will return user's latest message if no rewriting is needed
      const rewrittenQuery = await this.rewriteQueryWithContext(reqData.messages, reqData.model)
      logger.debug(`[OllamaController] Rewritten query for RAG: "${rewrittenQuery}"`)
      if (rewrittenQuery) {
        const collectionFilter: string | null = request.input('collection', null)
        const relevantDocs = await this.ragService.searchSimilarDocuments(
          rewrittenQuery,
          5, // Top 5 most relevant chunks
          0.3, // Minimum similarity score of 0.3
          collectionFilter ?? undefined
        )
        logger.debug(`[RAG] Retrieved ${relevantDocs.length} relevant documents for query: "${rewrittenQuery}"`)
        // If relevant context is found, inject as a system message with adaptive limits
        if (relevantDocs.length > 0) {
          // Determine context budget based on model size
          const { maxResults, maxTokens } = this.getContextLimitsForModel(reqData.model)
          let trimmedDocs = relevantDocs.slice(0, maxResults)
          // Apply token cap if set (estimate ~3.5 chars per token)
          // Always include the first (most relevant) result — the cap only gates subsequent results
          if (maxTokens > 0) {
            const charCap = maxTokens * 3.5
            let totalChars = 0
            trimmedDocs = trimmedDocs.filter((doc, idx) => {
              totalChars += doc.text.length
              return idx === 0 || totalChars <= charCap
            })
          }
          logger.debug(
            `[RAG] Injecting ${trimmedDocs.length}/${relevantDocs.length} results (model: ${reqData.model}, maxResults: ${maxResults}, maxTokens: ${maxTokens || 'unlimited'})`
          )
          // Label each context block with its source title when available (a neutral,
          // honest provenance signal) but never the raw relevance score — nomic cosine
          // scores for genuinely relevant passages sit ~0.4-0.6, and surfacing e.g.
          // "42%" primes the model to distrust correct context. Scores stay in the logs
          // above for debugging.
          const contextText = trimmedDocs
            .map((doc, idx) => {
              const title = doc.metadata?.full_title || doc.metadata?.article_title
              const label = title ? `[Context ${idx + 1} — ${title}]` : `[Context ${idx + 1}]`
              return `${label}\n${doc.text}`
            })
            .join('\n\n')
          const systemMessage = {
            role: 'system' as const,
            content: SYSTEM_PROMPTS.rag_context(contextText),
          }
          // Insert system message at the beginning (after any existing system messages)
          const firstNonSystemIndex = reqData.messages.findIndex((msg) => msg.role !== 'system')
          const insertIndex = firstNonSystemIndex === -1 ? 0 : firstNonSystemIndex
          reqData.messages.splice(insertIndex, 0, systemMessage)
        }
      }
      // If system messages are large (e.g. due to RAG context), request a context window big
      // enough to fit them. Ollama respects num_ctx per-request; LM Studio ignores it gracefully.
      const systemChars = reqData.messages
        .filter((m) => m.role === 'system')
        .reduce((sum, m) => sum + m.content.length, 0)
      const estimatedSystemTokens = Math.ceil(systemChars / 3.5)
      let numCtx: number | undefined
      if (estimatedSystemTokens > 3000) {
        const needed = estimatedSystemTokens + 2048 // leave room for conversation + response
        numCtx = [8192, 16384, 32768, 65536].find((n) => n >= needed) ?? 65536
        logger.debug(`[OllamaController] Large system prompt (~${estimatedSystemTokens} tokens), requesting num_ctx: ${numCtx}`)
      }
      // Check if the model supports "thinking" capability for enhanced response generation.
      // Thinking is only enabled when the model supports it AND the user wants it: the explicit
      // per-request preference wins, otherwise the global default (ai.autoThinking, default OFF).
      // If gpt-oss model, it requires a text param for "think" https://docs.ollama.com/api/chat
      const thinkingCapability = await this.ollamaService.checkModelHasThinking(reqData.model)
      let thinkingEnabled = false
      if (thinkingCapability) {
        thinkingEnabled = reqData.think ?? ((await KVStore.getValue('ai.autoThinking')) ?? false)
      }
      const think: boolean | 'medium' =
        thinkingEnabled ? (reqData.model.startsWith('gpt-oss') ? 'medium' : true) : false
      // Separate sessionId and the resolved thinking preference from the Ollama request payload —
      // Ollama rejects unknown fields, and `think` is re-derived above (not forwarded raw).
      const { sessionId, think: _thinkPref, ...ollamaRequest } = reqData
      // Save user message to DB before streaming if sessionId provided
      let userContent: string | null = null
      if (sessionId) {
        const lastUserMsg = [...reqData.messages].reverse().find((m) => m.role === 'user')
        if (lastUserMsg) {
          userContent = lastUserMsg.content
          await this.chatService.addMessage(sessionId, 'user', userContent)
        }
      }
      if (reqData.stream) {
        logger.debug(`[OllamaController] Initiating streaming response for model: "${reqData.model}" with think: ${think}`)
        // Headers already flushed above.
        // Abort the upstream generation if the client disconnects — otherwise an abandoned
        // request keeps decoding server-side and, with Ollama's default OLLAMA_NUM_PARALLEL=1,
        // blocks every later chat/RAG request until the model is manually stopped (#1065).
        const abortController = new AbortController()
        response.response.on('close', () => abortController.abort())
        const stream = await this.ollamaService.chatStream({
          ...ollamaRequest,
          think,
          thinkingCapable: thinkingCapability,
          numCtx,
          signal: abortController.signal,
        })
        let fullContent = ''
        try {
          for await (const chunk of stream) {
            if (chunk.message?.content) {
              fullContent += chunk.message.content
            }
            response.response.write(`data: ${JSON.stringify(chunk)}\n\n`)
          }
        } catch (err) {
          if (abortController.signal.aborted) {
            logger.debug('[OllamaController] Client disconnected; aborted upstream Ollama generation')
            return
          }
          throw err
        }
        response.response.end()
        // Save assistant message and optionally generate title
        if (sessionId && fullContent) {
          await this.chatService.addMessage(sessionId, 'assistant', fullContent)
          const messageCount = await this.chatService.getMessageCount(sessionId)
          if (messageCount <= 2 && userContent) {
            this.chatService.generateTitle(sessionId, userContent, fullContent, reqData.model).catch((err) => {
              logger.error(`[OllamaController] Title generation failed: ${err instanceof Error ? err.message : err}`)
            })
          }
        }
        return
      }
      // Non-streaming (legacy) path
      const result = await this.ollamaService.chat({ ...ollamaRequest, think, thinkingCapable: thinkingCapability, numCtx })
      if (sessionId && result?.message?.content) {
        await this.chatService.addMessage(sessionId, 'assistant', result.message.content)
        const messageCount = await this.chatService.getMessageCount(sessionId)
        if (messageCount <= 2 && userContent) {
          this.chatService.generateTitle(sessionId, userContent, result.message.content, reqData.model).catch((err) => {
            logger.error(`[OllamaController] Title generation failed: ${err instanceof Error ? err.message : err}`)
          })
        }
      }
      return result
    } catch (error) {
      if (reqData.stream) {
        response.response.write(`data: ${JSON.stringify({ error: true })}\n\n`)
        response.response.end()
        return
      }
      throw error
    }
  }
  async remoteStatus() {
    const remoteUrl = await KVStore.getValue('ai.remoteOllamaUrl')
    if (!remoteUrl) {
      return { configured: false, connected: false }
    }
    try {
      const testResponse = await fetch(`${remoteUrl.replace(/\/$/, '')}/v1/models`, {
        signal: AbortSignal.timeout(3000),
      })
      return { configured: true, connected: testResponse.ok }
    } catch {
      return { configured: true, connected: false }
    }
  }
  async configureRemote({ request, response }: HttpContext) {
    const remoteUrl: string | null = request.input('remoteUrl', null)
    const ollamaService = await Service.query().where('service_name', SERVICE_NAMES.OLLAMA).first()
    if (!ollamaService) {
      return response.status(404).send({ success: false, message: 'Ollama service record not found.' })
    }
    // Clear path: null or empty URL removes remote config. If a local nomad_ollama container
    // still exists (user had previously installed AI Assistant locally), restart it and keep
    // the service marked installed. Otherwise fall back to uninstalled.
    if (!remoteUrl || remoteUrl.trim() === '') {
      await KVStore.clearValue('ai.remoteOllamaUrl')
      const hasLocalContainer = await this._startLocalOllamaContainerIfExists()
      ollamaService.installed = hasLocalContainer
      ollamaService.installation_status = 'idle'
      await ollamaService.save()
      return {
        success: true,
        message: hasLocalContainer
          ? 'Remote Ollama cleared. Local Ollama container restored.'
          : 'Remote Ollama configuration cleared.',
      }
    }
    try {
      assertNotCloudMetadataUrl(remoteUrl)
    } catch (err) {
      return response.status(400).send({
        success: false,
        message: err instanceof Error ? err.message : 'Invalid URL.',
      })
    }
    // Test connectivity via OpenAI-compatible /v1/models endpoint (works with Ollama, LM Studio, llama.cpp, etc.)
    try {
      const testResponse = await fetch(`${remoteUrl.replace(/\/$/, '')}/v1/models`, {
        signal: AbortSignal.timeout(5000),
      })
      if (!testResponse.ok) {
        return response.status(400).send({
          success: false,
          message: `Could not connect to ${remoteUrl} (HTTP ${testResponse.status}). Make sure the server is running and accessible. For Ollama, start it with OLLAMA_HOST=0.0.0.0.`,
        })
      }
    } catch (error) {
      return response.status(400).send({
        success: false,
        message: `Could not connect to ${remoteUrl}. Make sure the server is running and reachable. For Ollama, start it with OLLAMA_HOST=0.0.0.0.`,
      })
    }
    // Save remote URL and mark service as installed
    await KVStore.setValue('ai.remoteOllamaUrl', remoteUrl.trim())
    ollamaService.installed = true
    ollamaService.installation_status = 'idle'
    await ollamaService.save()
    // Stop the local nomad_ollama container (if running) so it doesn't compete with the
    // remote host for GPU / port 11434. Preserves the container and its models volume.
    await this._stopLocalOllamaContainer()
    // Install Qdrant if not already installed (fire-and-forget)
    const qdrantService = await Service.query().where('service_name', SERVICE_NAMES.QDRANT).first()
    if (qdrantService && !qdrantService.installed) {
      this.dockerService.createContainerPreflight(SERVICE_NAMES.QDRANT).catch((error) => {
        logger.error('[OllamaController] Failed to start Qdrant preflight:', error)
      })
    }
    // Mirror post-install side effects: disable suggestions, trigger docs discovery
    await KVStore.setValue('chat.suggestionsEnabled', false)
    this.ragService.discoverNomadDocs().catch((error) => {
      logger.error('[OllamaController] Failed to discover Nomad docs:', error)
    })
    return { success: true, message: 'Remote Ollama configured.' }
  }
  private async _stopLocalOllamaContainer(): Promise<void> {
    try {
      const containers = await this.dockerService.docker.listContainers({ all: true })
      const ollamaContainer = containers.find((c) =>
        c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`)
      )
      if (!ollamaContainer || ollamaContainer.State !== 'running') {
        return
      }
      await this.dockerService.docker.getContainer(ollamaContainer.Id).stop()
      this.dockerService.invalidateServicesStatusCache()
      logger.info('[OllamaController] Stopped local nomad_ollama (remote Ollama configured)')
    } catch (error: any) {
      logger.error(
        { err: error },
        '[OllamaController] Failed to stop local nomad_ollama; remote Ollama is still active'
      )
    }
  }
  private async _startLocalOllamaContainerIfExists(): Promise<boolean> {
    try {
      const containers = await this.dockerService.docker.listContainers({ all: true })
      const ollamaContainer = containers.find((c) =>
        c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`)
      )
      if (!ollamaContainer) {
        return false
      }
      if (ollamaContainer.State !== 'running') {
        await this.dockerService.docker.getContainer(ollamaContainer.Id).start()
        this.dockerService.invalidateServicesStatusCache()
        logger.info('[OllamaController] Started local nomad_ollama (remote Ollama cleared)')
      }
      return true
    } catch (error: any) {
      logger.error(
        { err: error },
        '[OllamaController] Failed to start local nomad_ollama on remote clear'
      )
      return false
    }
  }
  async deleteModel({ request }: HttpContext) {
    const reqData = await request.validateUsing(modelNameSchema)
    await this.ollamaService.deleteModel(reqData.model)
    return {
      success: true,
      message: `Model deleted: ${reqData.model}`,
    }
  }
  async dispatchModelDownload({ request }: HttpContext) {
    const reqData = await request.validateUsing(modelNameSchema)
    await this.ollamaService.dispatchModelDownload(reqData.model)
    return {
      success: true,
      message: `Download job dispatched for model: ${reqData.model}`,
    }
  }
  async installedModels({ }: HttpContext) {
    const models = await this.ollamaService.getModels()
    // Enrich each model with its thinking capability so the chat picker knows which models
    // to show the per-model thinking toggle for. checkModelHasThinking memoizes /api/show
    // results, so this stays cheap on repeat loads. Best-effort per model.
    const thinking = await Promise.all(
      models.map((m) => this.ollamaService.checkModelHasThinking(m.name))
    )
    return models.map((m, i) => ({ ...m, thinking: thinking[i] }))
  }
  /**
   * Determines RAG context limits based on model size extracted from the model name.
   * Parses size indicators like "1b", "3b", "8b", "70b" from model names/tags.
   */
  private getContextLimitsForModel(modelName: string): { maxResults: number; maxTokens: number } {
    // Extract parameter count from model name (e.g., "llama3.2:3b", "qwen2.5:1.5b", "gemma:7b")
    const sizeMatch = modelName.match(/(\d+\.?\d*)[bB]/)
    const paramBillions = sizeMatch ? parseFloat(sizeMatch[1]) : 8 // default to 8B if unknown
    for (const tier of RAG_CONTEXT_LIMITS) {
      if (paramBillions <= tier.maxParams) {
        return { maxResults: tier.maxResults, maxTokens: tier.maxTokens }
      }
    }
    // Fallback: no limits
    return { maxResults: 5, maxTokens: 0 }
  }
  private async rewriteQueryWithContext(
    messages: Message[],
    model: string
  ): Promise<string | null> {
    const lastUserMessage = [...messages].reverse().find(msg => msg.role === 'user')
    try {
      // Skip the entire RAG pipeline if there are no documents to search
      const hasDocuments = await this.ragService.hasDocuments()
      if (!hasDocuments) {
        return null
      }
      // Get recent conversation history (last 6 messages for 3 turns)
      const recentMessages = messages.slice(-6)
      // Skip rewriting on the very first turn — with only one user message
      // there is no prior context to fold in, so the rewrite would just echo
      // the message back at the cost of an extra LLM round-trip. From the
      // first follow-up onward we need the rewrite so the RAG query carries
      // entities and topics from earlier turns ("the bars" → "Hershey's bars
      // chocolate poisoning dog"); without it, embeddings match nothing and
      // the assistant loses the thread.
      const userMessages = recentMessages.filter(msg => msg.role === 'user')
      if (userMessages.length < 2) {
        return lastUserMessage?.content || null
      }
      const conversationContext = recentMessages
        .map(msg => {
          const role = msg.role === 'user' ? 'User' : 'Assistant'
          // Truncate assistant messages to first 200 chars to keep context manageable
          const content = msg.role === 'assistant'
            ? msg.content.slice(0, 200) + (msg.content.length > 200 ? '...' : '')
            : msg.content
          return `${role}: "${content}"`
        })
        .join('\n')
      const response = await this.ollamaService.chat({
        model,
        messages: [
          {
            role: 'system',
            content: SYSTEM_PROMPTS.query_rewrite,
          },
          {
            role: 'user',
            content: `Conversation:\n${conversationContext}\n\nRewritten Query:`,
          },
        ],
      })
      const rewrittenQuery = response.message.content.trim()
      logger.info(`[RAG] Query rewritten: "${rewrittenQuery}"`)
      return rewrittenQuery
    } catch (error) {
      logger.error(
        `[RAG] Query rewriting failed: ${error instanceof Error ? error.message : error}`
      )
      // Fallback to last user message if rewriting fails
      return lastUserMessage?.content || null
    }
  }
 }
--- a/admin/app/controllers/openapi_controller.ts
+++ b/admin/app/controllers/openapi_controller.ts
@ -1,74 +0,0 @@
 import { createRequire } from 'node:module'
 import { readFileSync } from 'node:fs'
 import type { HttpContext } from '@adonisjs/core/http'
 import { buildOpenApiDocument } from '#start/openapi/generator'
 const require = createRequire(import.meta.url)
 /**
 * The self-contained Scalar browser bundle, read from the installed
 * `@scalar/api-reference` package and cached in memory. Served from this app
 * (not a CDN) so the API reference works on an offline appliance.
 */
 let scalarBundle: string | null = null
 function getScalarBundle(): string {
  if (scalarBundle === null) {
    const entry = require.resolve('@scalar/api-reference')
    const marker = '/node_modules/@scalar/api-reference/'
    const root = entry.slice(0, entry.lastIndexOf(marker) + marker.length - 1)
    scalarBundle = readFileSync(`${root}/dist/browser/standalone.js`, 'utf-8')
  }
  return scalarBundle
 }
 /**
 * The reference page mounts Scalar against our locally served bundle and spec.
 * Scalar auto-detects the `#api-reference` element and reads `data-url`.
 */
 const REFERENCE_HTML = `<!doctype html>
 <html lang="en">
  <head>
    <meta charset="utf-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1" />
    <title>Nomad Admin API Reference</title>
  </head>
  <body>
    <div id="app"></div>
    <script src="/reference/assets/standalone.js"></script>
    <script>
      Scalar.createApiReference('#app', {
        url: '/api/openapi.json',
        hideClientButton: true,
        telemetry: false,
        theme: 'saturn',
      })
    </script>
  </body>
 </html>`
 export default class OpenApiController {
  /**
   * The generated OpenAPI 3.1 document.
   */
  async spec({ response }: HttpContext) {
    return response.json(buildOpenApiDocument())
  }
  /**
   * The interactive Scalar API reference page.
   */
  async reference({ response }: HttpContext) {
    return response.header('content-type', 'text/html').send(REFERENCE_HTML)
  }
  /**
   * The locally bundled Scalar JS (kept off any CDN for offline use).
   */
  async standalone({ response }: HttpContext) {
    return response
      .header('content-type', 'application/javascript; charset=utf-8')
      .header('cache-control', 'public, max-age=86400')
      .send(getScalarBundle())
  }
 }
--- a/admin/app/controllers/openwebui_controller.ts
+++ b/admin/app/controllers/openwebui_controller.ts
@ -0,0 +1,40 @@
 import { OpenWebUIService } from '#services/openwebui_service'
 import { modelNameSchema } from '#validators/download'
 import { getAvailableModelsSchema } from '#validators/openwebui'
 import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
@inject()
 export default class OpenWebUIController {
  constructor(private openWebUIService: OpenWebUIService) {}
  async models({ request }: HttpContext) {
    const reqData = await request.validateUsing(getAvailableModelsSchema)
    return await this.openWebUIService.getAvailableModels({
      sort: reqData.sort,
      recommendedOnly: reqData.recommendedOnly,
    })
  }
  async installedModels({}: HttpContext) {
    return await this.openWebUIService.getInstalledModels()
  }
  async deleteModel({ request }: HttpContext) {
    const reqData = await request.validateUsing(modelNameSchema)
    await this.openWebUIService.deleteModel(reqData.model)
    return {
      success: true,
      message: `Model deleted: ${reqData.model}`,
    }
  }
  async dispatchModelDownload({ request }: HttpContext) {
    const reqData = await request.validateUsing(modelNameSchema)
    await this.openWebUIService.dispatchModelDownload(reqData.model)
    return {
      success: true,
      message: `Download job dispatched for model: ${reqData.model}`,
    }
  }
 }
--- a/admin/app/controllers/rag_controller.ts
+++ b/admin/app/controllers/rag_controller.ts
@ -1,247 +0,0 @@
 import { RagService } from '#services/rag_service'
 import { EmbedFileJob } from '#jobs/embed_file_job'
 import KbRatioRegistry from '#models/kb_ratio_registry'
 import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
 import app from '@adonisjs/core/services/app'
 import { randomBytes } from 'node:crypto'
 import { sanitizeFilename } from '../utils/fs.js'
 import { basename } from 'node:path'
 import { deleteFileSchema, embedFileSchema, estimateBatchSchema, fileSourceSchema, getJobStatusSchema } from '#validators/rag'
 import logger from '@adonisjs/core/services/logger'
 import { sanitizeCollectionName } from '../../constants/kb_collections.js'
@inject()
 export default class RagController {
  constructor(private ragService: RagService) { }
  public async upload({ request, response }: HttpContext) {
    const uploadedFile = request.file('file')
    if (!uploadedFile) {
      return response.status(400).json({ error: 'No file uploaded' })
    }
    const collection = sanitizeCollectionName(request.input('collection', null))
    const randomSuffix = randomBytes(6).toString('hex')
    const sanitizedName = sanitizeFilename(uploadedFile.clientName)
    const fileName = `${sanitizedName}-${randomSuffix}.${uploadedFile.extname || 'txt'}`
    const fullPath = app.makePath(RagService.UPLOADS_STORAGE_PATH, fileName)
    await uploadedFile.move(app.makePath(RagService.UPLOADS_STORAGE_PATH), {
      name: fileName,
    })
    // Dispatch background job for embedding
    const result = await EmbedFileJob.dispatch({
      filePath: fullPath,
      fileName,
      ...(collection ? { collection } : {}),
    })
    return response.status(202).json({
      message: result.message,
      jobId: result.jobId,
      fileName,
      filePath: `/${RagService.UPLOADS_STORAGE_PATH}/${fileName}`,
      alreadyProcessing: !result.created,
      ...(collection ? { collection } : {}),
    })
  }
  public async getActiveJobs({ response }: HttpContext) {
    const jobs = await EmbedFileJob.listActiveJobs()
    return response.status(200).json(jobs)
  }
  public async getJobStatus({ request, response }: HttpContext) {
    const reqData = await request.validateUsing(getJobStatusSchema)
    const fullPath = app.makePath(RagService.UPLOADS_STORAGE_PATH, reqData.filePath)
    const status = await EmbedFileJob.getStatus(fullPath)
    if (!status.exists) {
      return response.status(404).json({ error: 'Job not found for this file' })
    }
    return response.status(200).json(status)
  }
  public async getStoredFiles({ response }: HttpContext) {
    const files = await this.ragService.getStoredFiles()
    return response.status(200).json({ files })
  }
  public async getKnowledgeCollections({ response }: HttpContext) {
    const collections = await this.ragService.getKnowledgeCollections()
    return response.status(200).json({ collections })
  }
  public async updateFileCollection({ request, response }: HttpContext) {
    const source: string | null = request.input('source', null)
    // sanitizeCollectionName trims/lowercases/caps length, and returns null
    // for empty input — which doubles as "clear back to Uncategorized".
    const collection = sanitizeCollectionName(request.input('collection', null))
    if (!source) {
      return response.status(400).json({ error: 'source is required.' })
    }
    const result = await this.ragService.updateFileCollection(source, collection)
    if (!result.success) {
      return response.status(500).json({ error: result.message })
    }
    return response.status(200).json({ message: result.message })
  }
  public async renameKnowledgeCollection({ request, response }: HttpContext) {
    const oldName = sanitizeCollectionName(request.input('oldName', null))
    const newName = sanitizeCollectionName(request.input('newName', null))
    if (!oldName || !newName) {
      return response.status(400).json({ error: 'oldName and newName are required.' })
    }
    const result = await this.ragService.renameKnowledgeCollection(oldName, newName)
    if (!result.success) {
      return response.status(500).json({ error: result.message })
    }
    return response.status(200).json({ message: result.message })
  }
  public async deleteKnowledgeCollection({ request, response }: HttpContext) {
    const name = sanitizeCollectionName(request.input('name', null))
    if (!name) {
      return response.status(400).json({ error: 'name is required.' })
    }
    const result = await this.ragService.deleteKnowledgeCollection(name)
    if (!result.success) {
      return response.status(500).json({ error: result.message })
    }
    return response.status(200).json({ message: result.message })
  }
  public async getFileWarnings({ response }: HttpContext) {
    const result = await this.ragService.computeFileWarnings()
    return response.status(200).json(result)
  }
  public async deleteFile({ request, response }: HttpContext) {
    const { source } = await request.validateUsing(deleteFileSchema)
    const result = await this.ragService.deleteFileBySource(source)
    if (!result.success) {
      return response.status(500).json({ error: result.message })
    }
    return response.status(200).json({ message: result.message })
  }
  public async embedFile({ request, response }: HttpContext) {
    const { source, force } = await request.validateUsing(embedFileSchema)
    const result = await this.ragService.embedSingleFile(source, force ?? false)
    if (!result.success) {
      const status = {
        not_found: 404,
        inflight: 409,
        delete_failed: 500,
        dispatch_failed: 500,
      }[result.code]
      return response.status(status).json({ error: result.message, code: result.code })
    }
    return response.status(202).json({ message: result.message })
  }
  public async getFailedJobs({ response }: HttpContext) {
    const jobs = await EmbedFileJob.listFailedJobs()
    return response.status(200).json(jobs)
  }
  public async cleanupFailedJobs({ response }: HttpContext) {
    const result = await EmbedFileJob.cleanupFailedJobs()
    return response.status(200).json({
      message: `Cleaned up ${result.cleaned} failed job${result.cleaned !== 1 ? 's' : ''}${result.filesDeleted > 0 ? `, deleted ${result.filesDeleted} file${result.filesDeleted !== 1 ? 's' : ''}` : ''}.`,
      ...result,
    })
  }
  public async cancelAllJobs({ response }: HttpContext) {
    const result = await EmbedFileJob.cancelAllJobs()
    return response.status(200).json({
      message: `Cancelled ${result.cancelled} job${result.cancelled !== 1 ? 's' : ''}${result.filesDeleted > 0 ? `, deleted ${result.filesDeleted} file${result.filesDeleted !== 1 ? 's' : ''}` : ''}.`,
      ...result,
    })
  }
  public async policyPromptState({ response }: HttpContext) {
    const result = await this.ragService.getPolicyPromptState()
    return response.status(200).json(result)
  }
  public async scanAndSync({ response }: HttpContext) {
    try {
      const syncResult = await this.ragService.scanAndSyncStorage()
      return response.status(200).json(syncResult)
    } catch (error) {
      logger.error({ err: error }, '[RagController] Error scanning and syncing storage')
      return response.status(500).json({ error: 'Error scanning and syncing storage' })
    }
  }
  public async reembedAll({ response }: HttpContext) {
    try {
      const result = await this.ragService.reembedAll()
      return response.status(200).json(result)
    } catch (error) {
      logger.error({ err: error }, '[RagController] Error during re-embed all')
      return response.status(500).json({ error: 'Error during re-embed all' })
    }
  }
  public async resetAndRebuild({ response }: HttpContext) {
    try {
      const result = await this.ragService.resetAndRebuild()
      return response.status(200).json(result)
    } catch (error) {
      logger.error({ err: error }, '[RagController] Error during reset and rebuild')
      return response.status(500).json({ error: 'Error during reset and rebuild' })
    }
  }
  public async health({ response }: HttpContext) {
    const result = await this.ragService.checkQdrantHealth()
    return response.status(200).json(result)
  }
  public async estimateBatch({ request, response }: HttpContext) {
    const { files } = await request.validateUsing(estimateBatchSchema)
    // The registry matches on basename prefixes; if a caller passes a full path
    // (e.g. /app/storage/zim/wikipedia_en_simple_…), strip directories first so
    // patterns like `wikipedia_en_simple_` still match.
    const normalized = files.map((f) => ({
      filename: basename(f.filename),
      sizeBytes: f.sizeBytes,
    }))
    const result = await KbRatioRegistry.estimateBatch(normalized)
    return response.status(200).json(result)
  }
  public async getFileContent({ request, response }: HttpContext) {
    const { source } = await request.validateUsing(fileSourceSchema)
    const result = await this.ragService.readFileContent(source)
    if (!result) {
      return response.status(404).json({ error: 'File not found or not viewable' })
    }
    return response.status(200).json(result)
  }
  public async downloadFile({ request, response }: HttpContext) {
    const { source } = await request.validateUsing(fileSourceSchema)
    const filePath = await this.ragService.resolveDownloadPath(source)
    if (!filePath) {
      return response.status(404).json({ error: 'File not found' })
    }
    const fileName = filePath.split(/[/\\]/).at(-1) ?? 'download'
    return response.attachment(filePath, fileName)
  }
 }
--- a/admin/app/controllers/settings_controller.ts
+++ b/admin/app/controllers/settings_controller.ts
@ -1,152 +1,77 @@
-import KVStore from '#models/kv_store'
+import { MapService } from '#services/map_service';
-import { BenchmarkService } from '#services/benchmark_service'
+import { OpenWebUIService } from '#services/openwebui_service';
-import { MapService } from '#services/map_service'
+import { SystemService } from '#services/system_service';
-import { OllamaService } from '#services/ollama_service'
+import { inject } from '@adonisjs/core';
 import { SystemService } from '#services/system_service'
 import { getSettingSchema, updateSettingSchema, validateSettingValue } from '#validators/settings'
 import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
 import env from '#start/env'
@inject()
 export default class SettingsController {
-  constructor(
+    constructor(
-    private systemService: SystemService,
+        private systemService: SystemService,
-    private mapService: MapService,
+        private mapService: MapService,
-    private benchmarkService: BenchmarkService,
+        private openWebUIService: OpenWebUIService
-    private ollamaService: OllamaService
+    ) { }
  ) {}
-  async system({ inertia }: HttpContext) {
+    async system({ inertia }: HttpContext) {
-    const systemInfo = await this.systemService.getSystemInfo()
+        const systemInfo = await this.systemService.getSystemInfo();
-    return inertia.render('settings/system', {
+        return inertia.render('settings/system', {
-      system: {
+            system: {
-        info: systemInfo,
+                info: systemInfo
-      },
+            }
-    })
+        });
  }
  async apps({ inertia }: HttpContext) {
    const services = await this.systemService.getServices({ installedOnly: false })
    return inertia.render('settings/apps', {
      system: {
        services,
      },
    })
  }
  async legal({ inertia }: HttpContext) {
    return inertia.render('settings/legal')
  }
  async support({ inertia }: HttpContext) {
    return inertia.render('settings/support')
  }
  async maps({ inertia }: HttpContext) {
    const baseAssetsCheck = await this.mapService.ensureBaseAssets()
    const [regionFiles, worldBasemapExists] = await Promise.all([
      this.mapService.listRegions(),
      this.mapService.checkWorldBasemapExists(),
    ])
    return inertia.render('settings/maps', {
      maps: {
        baseAssetsExist: baseAssetsCheck,
        worldBasemapExists,
        regionFiles: regionFiles.files,
      },
    })
  }
  async models({ inertia }: HttpContext) {
    const availableModels = await this.ollamaService.getAvailableModels({
      sort: 'pulls',
      recommendedOnly: false,
      query: null,
      limit: 15,
    })
    const installedModels = await this.ollamaService.getModels().catch(() => [])
    const chatSuggestionsEnabled = await KVStore.getValue('chat.suggestionsEnabled')
    const aiAssistantCustomName = await KVStore.getValue('ai.assistantCustomName')
    const remoteOllamaUrl = await KVStore.getValue('ai.remoteOllamaUrl')
    const ollamaFlashAttention = await KVStore.getValue('ai.ollamaFlashAttention')
    const autoThinking = await KVStore.getValue('ai.autoThinking')
    return inertia.render('settings/models', {
      models: {
        availableModels: availableModels?.models || [],
        installedModels: installedModels || [],
        settings: {
          chatSuggestionsEnabled: chatSuggestionsEnabled ?? false,
          aiAssistantCustomName: aiAssistantCustomName ?? '',
          remoteOllamaUrl: remoteOllamaUrl ?? '',
          ollamaFlashAttention: ollamaFlashAttention ?? true,
          autoThinking: autoThinking ?? false,
        },
      },
    })
  }
  async update({ inertia }: HttpContext) {
    const updateInfo = await this.systemService.checkLatestVersion()
    return inertia.render('settings/update', {
      system: {
        updateAvailable: updateInfo.updateAvailable,
        latestVersion: updateInfo.latestVersion,
        currentVersion: updateInfo.currentVersion,
      },
    })
  }
  async zim({ inertia }: HttpContext) {
    return inertia.render('settings/zim/index')
  }
  async zimRemote({ inertia }: HttpContext) {
    return inertia.render('settings/zim/remote-explorer')
  }
  async creatorPacks({ inertia }: HttpContext) {
    return inertia.render('settings/creator-packs')
  }
  async benchmark({ inertia }: HttpContext) {
    const latestResult = await this.benchmarkService.getLatestResult()
    const status = this.benchmarkService.getStatus()
    return inertia.render('settings/benchmark', {
      benchmark: {
        latestResult,
        status: status.status,
        currentBenchmarkId: status.benchmarkId,
      },
    })
  }
  async advanced({ inertia }: HttpContext) {
    // When the env var is set it always takes precedence over the stored value,
    // so surface that to the UI to disable the field and explain the override.
    const envOverride = Boolean(env.get('INTERNET_STATUS_TEST_URL')?.trim())
    const internetStatusTestUrl = await KVStore.getValue('system.internetStatusTestUrl')
    return inertia.render('settings/advanced', {
      advanced: {
        internetStatusTestUrl: internetStatusTestUrl ?? '',
        internetStatusTestUrlEnvOverride: envOverride,
      },
    })
  }
  async getSetting({ request, response }: HttpContext) {
    const { key } = await getSettingSchema.validate({ key: request.qs().key });
    const value = await KVStore.getValue(key);
    return response.status(200).send({ key, value });
  }
  async updateSetting({ request, response }: HttpContext) {
    const reqData = await request.validateUsing(updateSettingSchema)
    const valueError = validateSettingValue(reqData.key, reqData.value)
    if (valueError) {
      return response.status(422).send({ success: false, message: valueError })
    }
-    await this.systemService.updateSetting(reqData.key, reqData.value)
+
-    return response.status(200).send({ success: true, message: 'Setting updated successfully' })
+    async apps({ inertia }: HttpContext) {
-  }
+        const services = await this.systemService.getServices({ installedOnly: false });
-}
+        return inertia.render('settings/apps', {
            system: {
                services
            }
        });
    }
    async legal({ inertia }: HttpContext) {
        return inertia.render('settings/legal');
    }
    async maps({ inertia }: HttpContext) {
        const baseAssetsCheck = await this.mapService.checkBaseAssetsExist();
        const regionFiles = await this.mapService.listRegions();
        return inertia.render('settings/maps', {
            maps: {
                baseAssetsExist: baseAssetsCheck,
                regionFiles: regionFiles.files
            }
        });
    }
    async models({ inertia }: HttpContext) {
        const availableModels = await this.openWebUIService.getAvailableModels();
        const installedModels = await this.openWebUIService.getInstalledModels();
        return inertia.render('settings/models', {
            models: {
                availableModels: availableModels || [],
                installedModels: installedModels || []
            }
        });
    }
    async update({ inertia }: HttpContext) {
        const updateInfo = await this.systemService.checkLatestVersion();
        return inertia.render('settings/update', {
            system: {
                updateAvailable: updateInfo.updateAvailable,
                latestVersion: updateInfo.latestVersion,
                currentVersion: updateInfo.currentVersion
            }
        });
    }
    async zim({ inertia }: HttpContext) {
        return inertia.render('settings/zim/index')
    }
    async zimRemote({ inertia }: HttpContext) {
        return inertia.render('settings/zim/remote-explorer');
    }
 }
--- a/admin/app/controllers/supply_depot_controller.ts
+++ b/admin/app/controllers/supply_depot_controller.ts
@ -1,13 +0,0 @@
 import { SystemService } from '#services/system_service'
 import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
@inject()
 export default class SupplyDepotController {
  constructor(private systemService: SystemService) {}
  async index({ inertia }: HttpContext) {
    const services = await this.systemService.getServices({ installedOnly: false })
    return inertia.render('supply-depot', { system: { services } })
  }
 }
--- a/admin/app/controllers/system_controller.ts
+++ b/admin/app/controllers/system_controller.ts
@ -1,47 +1,16 @@
 import { DockerService } from '#services/docker_service';
 import { SystemService } from '#services/system_service'
 import { SystemUpdateService } from '#services/system_update_service'
-import { ContainerRegistryService } from '#services/container_registry_service'
+import { affectServiceValidator, installServiceValidator } from '#validators/system';
 import { AutoUpdateService } from '#services/auto_update_service'
 import { AppAutoUpdateService } from '#services/app_auto_update_service'
 import { ContentAutoUpdateService } from '#services/content_auto_update_service'
 import { DownloadService } from '#services/download_service'
 import { QueueService } from '#services/queue_service'
 import { CheckServiceUpdatesJob } from '#jobs/check_service_updates_job'
 import {
  affectServiceValidator,
  checkLatestVersionValidator,
  customAppValidator,
  deleteCustomAppValidator,
  installServiceValidator,
  preflightCustomValidator,
  preflightValidator,
  serviceLogsValidator,
  subscribeToReleaseNotesValidator,
  uninstallServiceValidator,
  updateCustomAppValidator,
  updateServiceValidator,
  setServiceAutoUpdateValidator,
  setServiceCustomUrlValidator,
  normalizeCustomUrl,
 } from '#validators/system'
 import {
  DEFAULT_CPUS,
  DEFAULT_MEMORY_MB,
  evaluateCustomApp,
 } from '#services/custom_app_guard'
 import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
 import logger from '@adonisjs/core/services/logger'
 import Service from '#models/service'
@inject()
 export default class SystemController {
    constructor(
        private systemService: SystemService,
        private dockerService: DockerService,
-        private systemUpdateService: SystemUpdateService,
+        private systemUpdateService: SystemUpdateService
        private containerRegistryService: ContainerRegistryService
    ) { }
    async getInternetStatus({ }: HttpContext) {
@ -63,7 +32,7 @@ export default class SystemController {
        if (result.success) {
            response.send({ success: true, message: result.message });
        } else {
-            response.status(400).send({ success: false, message: result.message });
+            response.status(400).send({ error: result.message });
        }
    }
@ -77,9 +46,8 @@ export default class SystemController {
        response.send({ success: result.success, message: result.message });
    }
-    async checkLatestVersion({ request }: HttpContext) {
+    async checkLatestVersion({ }: HttpContext) {
-        const payload = await request.validateUsing(checkLatestVersionValidator)
+        return await this.systemService.checkLatestVersion();
        return await this.systemService.checkLatestVersion(payload.force);
    }
    async forceReinstallService({ request, response }: HttpContext) {
@ -134,674 +102,4 @@ export default class SystemController {
        const logs = this.systemUpdateService.getUpdateLogs();
        response.send({ logs });
    }
    async getAutoUpdateStatus({ response }: HttpContext) {
        // Construct inline reusing already-injected singletons + the QueueService
        // singleton (its constructor is private to prevent Redis connection leaks,
        // so we must not let the container new a fresh one).
        const autoUpdateService = new AutoUpdateService(
            this.dockerService,
            new DownloadService(QueueService.getInstance()),
            this.systemService,
            this.systemUpdateService,
            this.containerRegistryService
        )
        try {
            const status = await autoUpdateService.getStatus()
            response.send(status)
        } catch (error) {
            logger.error({ err: error }, '[SystemController] Failed to get auto-update status')
            response.status(500).send({ error: 'Failed to retrieve auto-update status' })
        }
    }
    async getAppAutoUpdateStatus({ response }: HttpContext) {
        // Constructed inline reusing already-injected singletons + the QueueService
        // singleton (its constructor is private to prevent Redis connection leaks),
        // mirroring getAutoUpdateStatus. Apps need no SystemUpdateService (no sidecar).
        const appAutoUpdateService = new AppAutoUpdateService(
            this.dockerService,
            new DownloadService(QueueService.getInstance()),
            this.systemService,
            this.containerRegistryService
        )
        try {
            const status = await appAutoUpdateService.getStatus()
            response.send(status)
        } catch (error) {
            logger.error({ err: error }, '[SystemController] Failed to get app auto-update status')
            response.status(500).send({ error: 'Failed to retrieve app auto-update status' })
        }
    }
    async getContentAutoUpdateStatus({ response }: HttpContext) {
        // Mirrors getAppAutoUpdateStatus. Content auto-update needs only the
        // DownloadService (for the active-download pre-flight); the catalog and
        // collection-update services default-construct inside the service.
        const contentAutoUpdateService = new ContentAutoUpdateService(
            new DownloadService(QueueService.getInstance())
        )
        try {
            const status = await contentAutoUpdateService.getStatus()
            response.send(status)
        } catch (error) {
            logger.error({ err: error }, '[SystemController] Failed to get content auto-update status')
            response.status(500).send({ error: 'Failed to retrieve content auto-update status' })
        }
    }
    async setServiceAutoUpdate({ request, response }: HttpContext) {
        const payload = await request.validateUsing(setServiceAutoUpdateValidator)
        const service = await Service.query().where('service_name', payload.service_name).first()
        if (!service) {
            return response.status(404).send({ error: `Service ${payload.service_name} not found` })
        }
        service.auto_update_enabled = payload.enabled
        // Re-enabling clears any prior self-disable so the app gets a fresh start.
        if (payload.enabled) {
            service.auto_update_consecutive_failures = 0
            service.auto_update_disabled_reason = null
        }
        await service.save()
        return response.send({ success: true, message: 'App auto-update preference updated' })
    }
    async subscribeToReleaseNotes({ request }: HttpContext) {
        const reqData = await request.validateUsing(subscribeToReleaseNotesValidator);
        return await this.systemService.subscribeToReleaseNotes(reqData.email);
    }
    async getDebugInfo({}: HttpContext) {
        const debugInfo = await this.systemService.getDebugInfo()
        return { debugInfo }
    }
    async checkServiceUpdates({ response }: HttpContext) {
        await CheckServiceUpdatesJob.dispatch()
        response.send({ success: true, message: 'Service update check dispatched' })
    }
    async getAvailableVersions({ params, response }: HttpContext) {
        const serviceName = params.name
        const service = await (await import('#models/service')).default
            .query()
            .where('service_name', serviceName)
            .where('installed', true)
            .first()
        if (!service) {
            return response.status(404).send({ error: `Service ${serviceName} not found or not installed` })
        }
        try {
            const hostArch = await this.getHostArch()
            const updates = await this.containerRegistryService.getAvailableUpdates(
                service.container_image,
                hostArch,
                service.source_repo
            )
            response.send({ versions: updates })
        } catch (error) {
            logger.error({ err: error }, `[SystemController] Failed to fetch versions for ${serviceName}`)
            response.status(500).send({ error: 'Failed to fetch available versions for this service.' })
        }
    }
    async updateService({ request, response }: HttpContext) {
        const payload = await request.validateUsing(updateServiceValidator)
        const result = await this.dockerService.updateContainer(
            payload.service_name,
            payload.target_version
        )
        if (result.success) {
            response.send({ success: true, message: result.message })
        } else {
            response.status(400).send({ error: result.message })
        }
    }
    private async getHostArch(): Promise<string> {
        try {
            const info = await this.dockerService.docker.info()
            const arch = info.Architecture || ''
            const archMap: Record<string, string> = {
                x86_64: 'amd64',
                aarch64: 'arm64',
                armv7l: 'arm',
                amd64: 'amd64',
                arm64: 'arm64',
            }
            return archMap[arch] || arch.toLowerCase()
        } catch {
            return 'amd64'
        }
    }
    /**
     * Pre-install preflight check: reports port conflicts and resource warnings for a service.
     * Results are advisory — the UI shows warnings but allows the user to force-proceed.
     */
    async preflightCheck({ request, response }: HttpContext) {
        const payload = await request.validateUsing(preflightValidator)
        const service = await Service.query().where('service_name', payload.service_name).first()
        if (!service) {
            return response.status(404).send({ error: `Service ${payload.service_name} not found` })
        }
        // Extract host ports from container_config — the MySQL driver may return JSON columns
        // as an already-parsed object rather than a string, so guard before calling JSON.parse.
        const rawConfig = service.container_config
        const config = rawConfig
            ? typeof rawConfig === 'object'
                ? rawConfig
                : JSON.parse(rawConfig as string)
            : null
        const portBindings: Record<string, [{ HostPort: string }]> =
            config?.HostConfig?.PortBindings ?? {}
        const hostPorts = Object.values(portBindings)
            .flat()
            .map((b) => parseInt(b.HostPort, 10))
            .filter((p) => !isNaN(p))
        // Parse resource requirements from metadata (same object-guard as container_config)
        let minMemoryMB = 256
        let minDiskMB = 512
        try {
            const rawMeta = service.metadata
            const meta = rawMeta
                ? typeof rawMeta === 'object'
                    ? rawMeta
                    : JSON.parse(rawMeta as string)
                : null
            if (meta?.minMemoryMB) minMemoryMB = meta.minMemoryMB
            if (meta?.minDiskMB) minDiskMB = meta.minDiskMB
        } catch {}
        const [{ conflicts: portConflicts }, resourceWarnings] = await Promise.all([
            this.dockerService.checkPortConflicts(hostPorts),
            this.systemService.checkResourceWarnings(minMemoryMB, minDiskMB),
        ])
        return response.send({ portConflicts, resourceWarnings })
    }
    /** Return the next suggested host port for a custom app (8600+ range). */
    async suggestCustomPort({ response }: HttpContext) {
        const port = await this.systemService.getNextSuggestedCustomPort()
        return response.send({ port })
    }
    /**
     * Service-less preflight for the custom-app form: given host ports, volumes and an image,
     * report port conflicts, host resource warnings, overridable guard warnings (risky bind
     * mounts / untrusted or moving-tag images), and hard blocks (docker socket, system dirs,
     * malformed image). Lets the form give live feedback before a Service record exists.
     */
    async preflightCustomApp({ request, response }: HttpContext) {
        const payload = await request.validateUsing(preflightCustomValidator)
        const [{ conflicts }, resourceWarnings] = await Promise.all([
            this.dockerService.checkPortConflicts(payload.ports ?? []),
            this.systemService.checkResourceWarnings(256, 512),
        ])
        // When editing, the app's own container legitimately holds its ports — don't flag those.
        const portConflicts = payload.exclude_service
            ? conflicts.filter((c) => c.usedBy !== payload.exclude_service)
            : conflicts
        const guard = evaluateCustomApp({ image: payload.image, volumes: payload.volumes })
        return response.send({
            portConflicts,
            resourceWarnings: [...resourceWarnings, ...guard.warnings],
            blocked: guard.blocked,
        })
    }
    /** Create and immediately begin installing a custom app container. */
    async createCustomApp({ request, response }: HttpContext) {
        const payload = await request.validateUsing(customAppValidator)
        // Derive a stable service_name from the friendly name
        const slug = payload.friendly_name
            .toLowerCase()
            .replace(/[^a-z0-9]+/g, '_')
            .replace(/^_+|_+$/g, '')
        const serviceName = `nomad_custom_${slug}`
        const existing = await Service.query().where('service_name', serviceName).first()
        if (existing) {
            return response.status(409).send({
                success: false,
                message: `A custom app named "${payload.friendly_name}" already exists. Choose a different name.`,
            })
        }
        // Reject duplicate host ports within the request — Docker would otherwise fail at
        // start time with an opaque "port is already allocated" error.
        const hostPorts = (payload.ports ?? []).map((p) => p.host)
        const duplicateHostPorts = [...new Set(hostPorts.filter((p, i) => hostPorts.indexOf(p) !== i))]
        if (duplicateHostPorts.length) {
            return response.status(422).send({
                success: false,
                message: `Duplicate host port(s): ${duplicateHostPorts.join(', ')}. Each host port can map to only one container.`,
            })
        }
        // Security guardrails: hard-block dangerous bind mounts / malformed images regardless of
        // force; surface overridable warnings (risky paths, untrusted/moving-tag images) unless forced.
        const guard = evaluateCustomApp({ image: payload.image, volumes: payload.volumes })
        if (guard.blocked.length) {
            return response.status(422).send({
                success: false,
                message: guard.blocked.join(' '),
                blocked: guard.blocked,
            })
        }
        if (!payload.force && guard.warnings.length) {
            return response.status(409).send({
                success: false,
                message: guard.warnings.join(' '),
                warnings: guard.warnings,
            })
        }
        // Advisory preflight: surface port conflicts before creating the record so a failed
        // install doesn't leave a phantom card. The user can re-submit with force=true to override.
        if (!payload.force && hostPorts.length) {
            const { conflicts } = await this.dockerService.checkPortConflicts(hostPorts)
            if (conflicts.length) {
                return response.status(409).send({
                    success: false,
                    message: `Port conflict: ${conflicts
                        .map((c) => `${c.port} (in use by ${c.usedBy})`)
                        .join(', ')}.`,
                    portConflicts: conflicts,
                })
            }
        }
        const { containerConfig, uiLocation } = this.buildCustomContainerConfig(payload)
        await Service.create({
            service_name: serviceName,
            friendly_name: payload.friendly_name,
            container_image: payload.image,
            container_config: JSON.stringify(containerConfig),
            ui_location: uiLocation,
            icon: payload.icon || 'IconBrandDocker',
            installed: false,
            installation_status: 'idle',
            is_dependency_service: false,
            is_custom: true,
            category: payload.category ?? 'custom',
            depends_on: null,
        })
        const result = await this.dockerService.createContainerPreflight(serviceName)
        if (result.success) {
            return response.send({ success: true, message: result.message, service_name: serviceName })
        }
        return response.status(400).send({ success: false, message: result.message })
    }
    /** Delete a custom app: stop + remove its container, then delete the DB record. */
    async deleteCustomApp({ request, response }: HttpContext) {
        const payload = await request.validateUsing(deleteCustomAppValidator)
        const service = await Service.query().where('service_name', payload.service_name).first()
        if (!service) {
            return response.status(404).send({ error: `Service ${payload.service_name} not found` })
        }
        if (!service.is_custom) {
            return response.status(403).send({ error: 'Only custom apps can be deleted.' })
        }
        await this.dockerService.removeCustomAppContainer(payload.service_name, payload.remove_image ?? false)
        await service.delete()
        return response.send({ success: true, message: `Custom app ${payload.service_name} deleted` })
    }
    /** Uninstall a curated catalog app: stop + remove its container (optionally its image) and
     * return the card to the available catalog. App data under the storage path stays on disk,
     * so a later reinstall picks it back up. Custom apps are removed via deleteCustomApp instead,
     * which also drops their DB record. */
    async uninstallService({ request, response }: HttpContext) {
        const payload = await request.validateUsing(uninstallServiceValidator)
        const service = await Service.query().where('service_name', payload.service_name).first()
        if (!service) {
            return response.status(404).send({ error: `Service ${payload.service_name} not found` })
        }
        if (service.is_custom) {
            return response.status(403).send({ error: 'Custom apps are removed via delete.' })
        }
        if (service.is_dependency_service) {
            return response.status(403).send({ error: 'Dependency services cannot be uninstalled directly.' })
        }
        if (!service.installed) {
            return response.status(409).send({ error: `Service ${payload.service_name} is not installed` })
        }
        const result = await this.dockerService.uninstallService(
            payload.service_name,
            payload.remove_image ?? false
        )
        if (!result.success) {
            return response.status(500).send({ success: false, message: result.message })
        }
        return response.send({ success: true, message: result.message })
    }
    /** Set or clear an app's custom launch URL (works for curated and custom apps). Purely a
     * metadata change — no container is touched. An empty/invalid value clears the override, after
     * which the default host + port link is used again. */
    async setServiceCustomUrl({ request, response }: HttpContext) {
        const payload = await request.validateUsing(setServiceCustomUrlValidator)
        const service = await Service.query().where('service_name', payload.service_name).first()
        if (!service) {
            return response.status(404).send({ success: false, message: `Service ${payload.service_name} not found` })
        }
        // Hidden dependency services (e.g. Qdrant) aren't user-launchable, so they have no link to set.
        if (service.is_dependency_service) {
            return response.status(403).send({ success: false, message: 'This service cannot be configured.' })
        }
        // Reject a non-empty value that isn't a valid http(s) URL; an empty value clears the override.
        const normalized = normalizeCustomUrl(payload.custom_url)
        if (payload.custom_url && payload.custom_url.trim() && !normalized) {
            return response.status(422).send({
                success: false,
                message: 'Custom URL must be a valid http(s) address (e.g. https://jellyfin.myhomelab.net).',
            })
        }
        service.custom_url = normalized
        await service.save()
        return response.send({ success: true, custom_url: service.custom_url })
    }
    /** Re-pull a custom app's image and recreate its container in place (preserving volumes). */
    async updateCustomApp_pullLatest({ request, response }: HttpContext) {
        const payload = await request.validateUsing(installServiceValidator)
        const service = await Service.query().where('service_name', payload.service_name).first()
        if (!service) {
            return response.status(404).send({ success: false, message: `Service ${payload.service_name} not found` })
        }
        if (!service.is_custom) {
            return response.status(403).send({ success: false, message: 'Only custom apps can be updated this way.' })
        }
        const result = await this.dockerService.recreateCustomAppContainer(payload.service_name, {
            forcePull: true,
        })
        if (result.success) {
            return response.send({ success: true, message: result.message })
        }
        return response.status(400).send({ success: false, message: result.message })
    }
    /** Return the last N lines of a service container's logs. */
    async getServiceLogs({ params, request, response }: HttpContext) {
        // Scope to managed services only — otherwise any sibling container's logs (admin app,
        // database) would be readable by name on this unauthenticated API surface.
        const service = await Service.query().where('service_name', params.name).first()
        if (!service) {
            return response.status(404).send({ success: false, message: `Service ${params.name} not found` })
        }
        const { tail } = await request.validateUsing(serviceLogsValidator)
        const result = await this.dockerService.getContainerLogs(params.name, tail ?? 200)
        if (!result.success) {
            return response.status(404).send({ success: false, message: result.message })
        }
        return response.send({ success: true, logs: result.logs })
    }
    /** Return a one-shot CPU/memory usage snapshot for a running service container. */
    async getServiceStats({ params, response }: HttpContext) {
        // Scope to managed services only (see getServiceLogs).
        const service = await Service.query().where('service_name', params.name).first()
        if (!service) {
            return response.status(404).send({ success: false, message: `Service ${params.name} not found` })
        }
        const result = await this.dockerService.getContainerStats(params.name)
        if (!result.success) {
            return response.status(404).send({ success: false, message: result.message })
        }
        return response.send({ success: true, running: result.running ?? false, stats: result.stats ?? null })
    }
    /** Return an app's current configuration in the editable form-shape. */
    async getCustomApp({ params, response }: HttpContext) {
        const service = await Service.query().where('service_name', params.name).first()
        if (!service) {
            return response.status(404).send({ error: `Service ${params.name} not found` })
        }
        // Custom and curated apps are both editable; hidden dependency services (e.g. Qdrant) are not.
        if (service.is_dependency_service) {
            return response.status(403).send({ error: 'This service cannot be edited.' })
        }
        return response.send({ success: true, app: this.parseCustomContainerConfig(service) })
    }
    /** Reconfigure an app: validate + guard, persist the new config, then recreate the container.
     * Works for both custom apps and curated (pre-configured) apps. Editing a curated app marks it
     * user-modified so the seeder stops overwriting the user's changes. */
    async updateCustomApp({ request, response }: HttpContext) {
        const payload = await request.validateUsing(updateCustomAppValidator)
        const service = await Service.query().where('service_name', payload.service_name).first()
        if (!service) {
            return response.status(404).send({ success: false, message: `Service ${payload.service_name} not found` })
        }
        // Custom and curated apps are both editable; hidden dependency services (e.g. Qdrant) are not.
        if (service.is_dependency_service) {
            return response.status(403).send({ success: false, message: 'This service cannot be edited.' })
        }
        // Reject duplicate host ports within the request.
        const hostPorts = (payload.ports ?? []).map((p) => p.host)
        const duplicateHostPorts = [...new Set(hostPorts.filter((p, i) => hostPorts.indexOf(p) !== i))]
        if (duplicateHostPorts.length) {
            return response.status(422).send({
                success: false,
                message: `Duplicate host port(s): ${duplicateHostPorts.join(', ')}. Each host port can map to only one container.`,
            })
        }
        // Security guardrails (same posture as create).
        const guard = evaluateCustomApp({ image: payload.image, volumes: payload.volumes })
        if (guard.blocked.length) {
            return response.status(422).send({ success: false, message: guard.blocked.join(' '), blocked: guard.blocked })
        }
        if (!payload.force && guard.warnings.length) {
            return response.status(409).send({ success: false, message: guard.warnings.join(' '), warnings: guard.warnings })
        }
        // Port conflicts — but ignore ports already held by this app's own container.
        if (!payload.force && hostPorts.length) {
            const { conflicts } = await this.dockerService.checkPortConflicts(hostPorts)
            const external = conflicts.filter((c) => c.usedBy !== payload.service_name)
            if (external.length) {
                return response.status(409).send({
                    success: false,
                    message: `Port conflict: ${external
                        .map((c) => `${c.port} (in use by ${c.usedBy})`)
                        .join(', ')}.`,
                    portConflicts: external,
                })
            }
        }
        // Merge the form fields into the app's existing config rather than rebuilding from scratch,
        // so advanced settings a curated app ships with (GPU device requests, special env, etc.) are
        // preserved across an edit.
        // Preserve an explicit scheme (e.g. ui_location "https:8480") across an edit — otherwise a
        // TLS-serving app's Open link would silently revert to http after any reconfigure.
        const prevScheme = (service.ui_location || '').match(/^(https?):\d+$/)?.[1]
        const { containerConfig, uiLocation } = this.mergeCustomContainerConfig(
            service.container_config,
            payload
        )
        service.friendly_name = payload.friendly_name
        service.container_image = payload.image
        service.container_config = JSON.stringify(containerConfig)
        service.ui_location = prevScheme && uiLocation && /^\d+$/.test(uiLocation)
            ? `${prevScheme}:${uiLocation}`
            : uiLocation
        service.category = payload.category ?? service.category ?? 'custom'
        if (payload.icon) service.icon = payload.icon
        // Flag as user-modified so the seeder stops overwriting this app's config on future runs.
        service.is_user_modified = true
        await service.save()
        const result = await this.dockerService.recreateCustomAppContainer(payload.service_name)
        if (result.success) {
            return response.send({ success: true, message: result.message, service_name: payload.service_name })
        }
        return response.status(400).send({ success: false, message: result.message })
    }
    /**
     * Build a Docker container config (HostConfig + ExposedPorts + Env) from custom-app form input,
     * applying default resource caps. Shared by create and update so both stay in lockstep.
     */
    private buildCustomContainerConfig(payload: {
        ports?: { container: number; host: number }[]
        volumes?: { host_path: string; container_path: string }[]
        env?: string[]
        memory_mb?: number
        cpus?: number
    }): { containerConfig: Record<string, any>; uiLocation: string | null } {
        const portBindings: Record<string, [{ HostPort: string }]> = {}
        const exposedPorts: Record<string, {}> = {}
        for (const { container, host } of payload.ports ?? []) {
            portBindings[`${container}/tcp`] = [{ HostPort: String(host) }]
            exposedPorts[`${container}/tcp`] = {}
        }
        const binds = (payload.volumes ?? []).map(
            ({ host_path, container_path }) => `${host_path}:${container_path}`
        )
        // Resource caps so a runaway custom container can't starve the host. Memory is bytes;
        // NanoCpus is CPUs × 1e9. Defaults are generous and user-overridable.
        const memoryBytes = (payload.memory_mb ?? DEFAULT_MEMORY_MB) * 1024 * 1024
        const nanoCpus = Math.round((payload.cpus ?? DEFAULT_CPUS) * 1e9)
        const containerConfig: Record<string, any> = {
            HostConfig: {
                RestartPolicy: { Name: 'unless-stopped' },
                PortBindings: portBindings,
                Memory: memoryBytes,
                NanoCpus: nanoCpus,
                ...(binds.length ? { Binds: binds } : {}),
            },
            ExposedPorts: exposedPorts,
            ...(payload.env?.length ? { Env: payload.env } : {}),
        }
        const firstHostPort = payload.ports?.[0]?.host
        const uiLocation = firstHostPort ? String(firstHostPort) : null
        return { containerConfig, uiLocation }
    }
    /**
     * Merge custom-app form input into an app's *existing* container config. Used by the edit path so
     * editing a curated app only changes the fields exposed in the form (image/ports/volumes/env and,
     * if supplied, resource caps) while preserving everything else it ships with (GPU DeviceRequests,
     * User, custom HostConfig keys, etc.). Unlike buildCustomContainerConfig, resource caps are NOT
     * defaulted here — a curated app intentionally left uncapped stays uncapped unless the user sets one.
     */
    private mergeCustomContainerConfig(
        existingRaw: string | null,
        payload: {
            ports?: { container: number; host: number }[]
            volumes?: { host_path: string; container_path: string }[]
            env?: string[]
            memory_mb?: number
            cpus?: number
        }
    ): { containerConfig: Record<string, any>; uiLocation: string | null } {
        const parsed = existingRaw
            ? typeof existingRaw === 'object'
                ? existingRaw
                : JSON.parse(existingRaw as string)
            : {}
        // Deep clone so we never mutate the parsed source.
        const containerConfig: Record<string, any> = JSON.parse(JSON.stringify(parsed ?? {}))
        containerConfig.HostConfig = containerConfig.HostConfig ?? {}
        // Keep a restart policy if the existing config lacked one.
        containerConfig.HostConfig.RestartPolicy =
            containerConfig.HostConfig.RestartPolicy ?? { Name: 'unless-stopped' }
        const portBindings: Record<string, [{ HostPort: string }]> = {}
        const exposedPorts: Record<string, {}> = {}
        for (const { container, host } of payload.ports ?? []) {
            portBindings[`${container}/tcp`] = [{ HostPort: String(host) }]
            exposedPorts[`${container}/tcp`] = {}
        }
        containerConfig.HostConfig.PortBindings = portBindings
        containerConfig.ExposedPorts = exposedPorts
        const binds = (payload.volumes ?? []).map(
            ({ host_path, container_path }) => `${host_path}:${container_path}`
        )
        if (binds.length) containerConfig.HostConfig.Binds = binds
        else delete containerConfig.HostConfig.Binds
        if (payload.env?.length) containerConfig.Env = payload.env
        else delete containerConfig.Env
        // Only touch resource caps when the user explicitly set them — preserve existing/uncapped otherwise.
        if (payload.memory_mb != null) {
            containerConfig.HostConfig.Memory = payload.memory_mb * 1024 * 1024
        }
        if (payload.cpus != null) {
            containerConfig.HostConfig.NanoCpus = Math.round(payload.cpus * 1e9)
        }
        const firstHostPort = payload.ports?.[0]?.host
        const uiLocation = firstHostPort ? String(firstHostPort) : null
        return { containerConfig, uiLocation }
    }
    /** Inverse of buildCustomContainerConfig: turn a stored Service into the editable form-shape. */
    private parseCustomContainerConfig(service: Service) {
        const raw = service.container_config
        const config = raw ? (typeof raw === 'object' ? raw : JSON.parse(raw as string)) : {}
        const hostConfig = config?.HostConfig ?? {}
        const ports = Object.entries(hostConfig.PortBindings ?? {}).map(([key, val]: [string, any]) => ({
            container: Number.parseInt(key, 10),
            host: Number.parseInt(val?.[0]?.HostPort, 10),
        }))
        const volumes = (hostConfig.Binds ?? []).map((bind: string) => {
            const idx = bind.indexOf(':')
            return { host_path: bind.slice(0, idx), container_path: bind.slice(idx + 1) }
        })
        return {
            service_name: service.service_name,
            friendly_name: service.friendly_name,
            image: service.container_image,
            category: service.category ?? 'custom',
            icon: service.icon ?? 'IconBrandDocker',
            ports,
            volumes,
            env: (config?.Env ?? []) as string[],
            memory_mb: hostConfig.Memory ? Math.round(hostConfig.Memory / (1024 * 1024)) : undefined,
            cpus: hostConfig.NanoCpus ? hostConfig.NanoCpus / 1e9 : undefined,
        }
    }
 }
--- a/admin/app/controllers/zim_controller.ts
+++ b/admin/app/controllers/zim_controller.ts
@ -1,19 +1,12 @@
 import { ZimService } from '#services/zim_service'
 import {
-  assertNotPrivateUrl,
+  downloadCollectionValidator,
  downloadCategoryTierValidator,
  filenameParamValidator,
-  remoteDownloadWithMetadataValidator,
+  remoteDownloadValidator,
  selectWikipediaValidator,
 } from '#validators/common'
-import { addCustomLibraryValidator, browseLibraryValidator, idParamValidator, listRemoteZimValidator } from '#validators/zim'
+import { listRemoteZimValidator } from '#validators/zim'
 import { inject } from '@adonisjs/core'
 import logger from '@adonisjs/core/services/logger'
 import type { HttpContext } from '@adonisjs/core/http'
 import { createWriteStream } from 'fs'
 import { rename } from 'fs/promises'
 import { join, resolve, sep } from 'path'
 import { ZIM_STORAGE_PATH, ensureDirectoryExists, sanitizeFilename } from '../utils/fs.js'
@inject()
 export default class ZimController {
@ -30,9 +23,8 @@ export default class ZimController {
  }
  async downloadRemote({ request }: HttpContext) {
-    const payload = await request.validateUsing(remoteDownloadWithMetadataValidator)
+    const payload = await request.validateUsing(remoteDownloadValidator)
-    assertNotPrivateUrl(payload.url)
+    const { filename, jobId } = await this.zimService.downloadRemote(payload.url)
    const { filename, jobId } = await this.zimService.downloadRemote(payload.url, payload.metadata)
    return {
      message: 'Download started successfully',
@ -42,31 +34,24 @@ export default class ZimController {
    }
  }
-  async listCuratedCategories({}: HttpContext) {
+  async downloadCollection({ request }: HttpContext) {
-    return await this.zimService.listCuratedCategories()
+    const payload = await request.validateUsing(downloadCollectionValidator)
-  }
+    const resources = await this.zimService.downloadCollection(payload.slug)
  async downloadCategoryTier({ request }: HttpContext) {
    const payload = await request.validateUsing(downloadCategoryTierValidator)
    const resources = await this.zimService.downloadCategoryTier(
      payload.categorySlug,
      payload.tierSlug
    )
    return {
      message: 'Download started successfully',
-      categorySlug: payload.categorySlug,
+      slug: payload.slug,
      tierSlug: payload.tierSlug,
      resources,
    }
  }
-  async rescanLibrary({}: HttpContext) {
+  async listCuratedCollections({}: HttpContext) {
-    const result = await this.zimService.rescanLibrary()
+    return this.zimService.listCuratedCollections()
-    return {
+  }
-      message: 'Kiwix library rescanned',
+
-      ...result,
+  async fetchLatestCollections({}: HttpContext) {
-    }
+    const success = await this.zimService.fetchLatestCollections()
    return { success }
  }
  async delete({ request, response }: HttpContext) {
@ -87,143 +72,4 @@ export default class ZimController {
      message: 'ZIM file deleted successfully',
    }
  }
  async upload({ request, response }: HttpContext) {
    let filename: string | null = null
    let tmpPath: string | null = null
    let uploadError: string | null = null
    try {
      const basePath = resolve(join(process.cwd(), ZIM_STORAGE_PATH))
      await ensureDirectoryExists(basePath)
      request.multipart.onFile('*', {}, async (part) => {
        const clientName = part.filename || ''
        if (!clientName.toLowerCase().endsWith('.zim')) {
          part.resume()
          uploadError = 'INVALID_TYPE'
          return
        }
        const sanitized = sanitizeFilename(clientName)
        const finalPath = resolve(join(basePath, sanitized))
        if (!finalPath.startsWith(basePath + sep)) {
          part.resume()
          uploadError = 'INVALID_FILENAME'
          return
        }
        const { access } = await import('fs/promises')
        const exists = await access(finalPath).then(() => true).catch(() => false)
        if (exists) {
          part.resume()
          uploadError = 'DUPLICATE_FILENAME'
          return
        }
        filename = sanitized
        tmpPath = finalPath + '.tmp'
        const ws = createWriteStream(tmpPath)
        await new Promise<void>((res, rej) => {
          ws.on('error', rej)
          ws.on('finish', res)
          part.on('error', rej)
          part.pipe(ws)
        })
        await rename(tmpPath, finalPath)
        tmpPath = null
      })
      await request.multipart.process()
      if (uploadError === 'INVALID_TYPE') {
        return response.status(422).send({ message: 'Only .zim files are accepted' })
      }
      if (uploadError === 'INVALID_FILENAME') {
        return response.status(422).send({ message: 'Invalid filename' })
      }
      if (uploadError === 'DUPLICATE_FILENAME') {
        return response.status(409).send({ message: 'A ZIM file with that name already exists' })
      }
      if (!filename) {
        return response.status(400).send({ message: 'No file received' })
      }
      const { added } = await this.zimService.registerLocalUpload(filename)
      return response.status(201).send({
        message: 'ZIM file uploaded and registered successfully',
        filename,
        added,
      })
    } catch (error) {
      logger.error('[ZimController] Upload failed:', error)
      if (tmpPath) {
        const { unlink } = await import('fs/promises')
        await unlink(tmpPath).catch(() => {})
      }
      return response.status(500).send({ message: 'Upload failed' })
    }
  }
  // Wikipedia selector endpoints
  async getWikipediaState({}: HttpContext) {
    return this.zimService.getWikipediaState()
  }
  async selectWikipedia({ request }: HttpContext) {
    const payload = await request.validateUsing(selectWikipediaValidator)
    return this.zimService.selectWikipedia(payload.optionId)
  }
  // Custom library endpoints
  async listCustomLibraries({}: HttpContext) {
    return this.zimService.listCustomLibraries()
  }
  async addCustomLibrary({ request, response }: HttpContext) {
    const payload = await request.validateUsing(addCustomLibraryValidator)
    assertNotPrivateUrl(payload.base_url)
    try {
      const source = await this.zimService.addCustomLibrary(payload.name, payload.base_url)
      return { message: 'Custom library added', library: source }
    } catch (error) {
      if (error.message === 'Maximum of 10 custom libraries allowed') {
        return response.status(400).send({ message: error.message })
      }
      throw error
    }
  }
  async removeCustomLibrary({ request, response }: HttpContext) {
    const payload = await request.validateUsing(idParamValidator)
    try {
      await this.zimService.removeCustomLibrary(payload.params.id)
      return { message: 'Custom library removed' }
    } catch (error) {
      if (error.message === 'Custom library not found') {
        return response.status(404).send({ message: error.message })
      }
      throw error
    }
  }
  async browseLibrary({ request, response }: HttpContext) {
    const payload = await request.validateUsing(browseLibraryValidator)
    try {
      return await this.zimService.browseLibraryUrl(payload.url)
    } catch (error) {
      if (error.message?.includes('loopback or link-local')) {
        return response.status(400).send({ message: error.message })
      }
      return response.status(502).send({
        message: 'Could not fetch directory listing from the provided URL',
      })
    }
  }
 }
--- a/admin/app/data/conditions.ts
+++ b/admin/app/data/conditions.ts
@ -1,438 +0,0 @@
 /* eslint-disable */
 /**
 * GENERATED FILE — DO NOT EDIT.
 *
 * Single source of truth: collections/conditions.json
 * Regenerate after editing the JSON: `npm run gen:curated-data` (from admin/).
 * curated_data_sync.standalone.ts fails CI if this file drifts from the JSON.
 *
 * Curated "when to use what" condition spine (Phase 1). US-government / public-domain sourcing.
 */
 import type { ConditionsFile } from '../../types/conditions.js'
 export const CONDITIONS_FILE: ConditionsFile = {
  "version": "2026-06-07",
  "conditions": [
    {
      "slug": "pain",
      "label": "Pain",
      "category": "Pain, fever & inflammation",
      "searchTerms": [
        "pain",
        "aches",
        "minor aches",
        "pain relief",
        "analgesic"
      ]
    },
    {
      "slug": "headache",
      "label": "Headache",
      "category": "Pain, fever & inflammation",
      "searchTerms": [
        "headache",
        "migraine",
        "head pain",
        "tension headache"
      ]
    },
    {
      "slug": "muscle-joint-pain",
      "label": "Muscle & joint pain",
      "category": "Pain, fever & inflammation",
      "searchTerms": [
        "muscular aches",
        "muscle aches",
        "backache",
        "arthritis",
        "joint pain",
        "minor pain of arthritis"
      ]
    },
    {
      "slug": "fever",
      "label": "Fever",
      "category": "Pain, fever & inflammation",
      "searchTerms": [
        "fever",
        "reduces fever",
        "fever reducer",
        "temporarily reduces fever"
      ]
    },
    {
      "slug": "menstrual-cramps",
      "label": "Menstrual cramps",
      "category": "Pain, fever & inflammation",
      "searchTerms": [
        "menstrual cramps",
        "menstrual pain",
        "period pain",
        "premenstrual"
      ]
    },
    {
      "slug": "cough",
      "label": "Cough",
      "category": "Cold, cough & allergy",
      "searchTerms": [
        "cough",
        "cough suppressant",
        "coughing",
        "controls cough"
      ]
    },
    {
      "slug": "nasal-congestion",
      "label": "Nasal congestion",
      "category": "Cold, cough & allergy",
      "searchTerms": [
        "nasal congestion",
        "stuffy nose",
        "sinus congestion",
        "decongestant",
        "nasal decongestant"
      ]
    },
    {
      "slug": "sore-throat",
      "label": "Sore throat",
      "category": "Cold, cough & allergy",
      "searchTerms": [
        "sore throat",
        "sore mouth",
        "minor sore throat",
        "throat pain"
      ]
    },
    {
      "slug": "common-cold",
      "label": "Common cold",
      "category": "Cold, cough & allergy",
      "searchTerms": [
        "common cold",
        "cold symptoms",
        "cold",
        "flu symptoms"
      ]
    },
    {
      "slug": "allergic-reaction",
      "label": "Allergies & allergic reactions",
      "category": "Cold, cough & allergy",
      "searchTerms": [
        "allergy",
        "allergic reactions",
        "hay fever",
        "antihistamine",
        "runny nose",
        "sneezing",
        "itchy watery eyes"
      ]
    },
    {
      "slug": "heartburn",
      "label": "Heartburn",
      "category": "Stomach & digestion",
      "searchTerms": [
        "heartburn",
        "acid indigestion",
        "acid reducer",
        "antacid",
        "sour stomach"
      ]
    },
    {
      "slug": "indigestion",
      "label": "Indigestion & upset stomach",
      "category": "Stomach & digestion",
      "searchTerms": [
        "indigestion",
        "upset stomach",
        "gas",
        "bloating",
        "fullness",
        "antacid"
      ]
    },
    {
      "slug": "nausea-vomiting",
      "label": "Nausea & vomiting",
      "category": "Stomach & digestion",
      "searchTerms": [
        "nausea",
        "vomiting",
        "upset stomach associated with nausea"
      ]
    },
    {
      "slug": "diarrhea",
      "label": "Diarrhea",
      "category": "Stomach & digestion",
      "searchTerms": [
        "diarrhea",
        "antidiarrheal",
        "loose stools",
        "traveler’s diarrhea"
      ]
    },
    {
      "slug": "constipation",
      "label": "Constipation",
      "category": "Stomach & digestion",
      "searchTerms": [
        "constipation",
        "laxative",
        "irregularity",
        "occasional constipation",
        "stool softener"
      ]
    },
    {
      "slug": "motion-sickness",
      "label": "Motion sickness",
      "category": "Stomach & digestion",
      "searchTerms": [
        "motion sickness",
        "travel sickness",
        "seasickness",
        "car sickness"
      ]
    },
    {
      "slug": "gas",
      "label": "Gas & bloating",
      "category": "Stomach & digestion",
      "searchTerms": [
        "gas",
        "bloating",
        "flatulence",
        "antigas",
        "pressure"
      ]
    },
    {
      "slug": "wounds-cuts",
      "label": "Wounds & cuts",
      "category": "Skin & wounds",
      "searchTerms": [
        "minor cuts",
        "scrapes",
        "wounds",
        "first aid antiseptic",
        "first aid to help prevent infection",
        "abrasions"
      ]
    },
    {
      "slug": "burns",
      "label": "Burns",
      "category": "Skin & wounds",
      "searchTerms": [
        "burns",
        "minor burns",
        "sunburn",
        "scald",
        "minor burn"
      ]
    },
    {
      "slug": "insect-bites-stings",
      "label": "Insect bites & stings",
      "category": "Skin & wounds",
      "searchTerms": [
        "insect bites",
        "insect stings",
        "bug bites",
        "bee sting",
        "itching from insect bites"
      ]
    },
    {
      "slug": "skin-rash-itch",
      "label": "Rash & itching",
      "category": "Skin & wounds",
      "searchTerms": [
        "itching",
        "rash",
        "skin irritation",
        "itchy skin",
        "minor skin irritations",
        "eczema"
      ]
    },
    {
      "slug": "poison-ivy",
      "label": "Poison ivy & plant rashes",
      "category": "Skin & wounds",
      "searchTerms": [
        "poison ivy",
        "poison oak",
        "poison sumac",
        "rashes due to poison ivy"
      ]
    },
    {
      "slug": "fungal-infection",
      "label": "Athlete’s foot & ringworm",
      "category": "Skin & wounds",
      "searchTerms": [
        "athlete’s foot",
        "ringworm",
        "jock itch",
        "antifungal",
        "fungal infection",
        "tinea"
      ]
    },
    {
      "slug": "dry-skin",
      "label": "Dry & chapped skin",
      "category": "Skin & wounds",
      "searchTerms": [
        "dry skin",
        "chapped skin",
        "chapped lips",
        "skin protectant",
        "cracked skin"
      ]
    },
    {
      "slug": "acne",
      "label": "Acne",
      "category": "Skin & wounds",
      "searchTerms": [
        "acne",
        "pimples",
        "blackheads",
        "acne treatment"
      ]
    },
    {
      "slug": "eye-irritation",
      "label": "Eye irritation & dryness",
      "category": "Eyes, ears & mouth",
      "searchTerms": [
        "eye irritation",
        "dry eyes",
        "red eyes",
        "eye redness",
        "itchy eyes",
        "lubricant eye"
      ]
    },
    {
      "slug": "earache",
      "label": "Earache & ear wax",
      "category": "Eyes, ears & mouth",
      "searchTerms": [
        "earache",
        "ear wax",
        "earwax removal",
        "ear pain",
        "swimmer’s ear"
      ]
    },
    {
      "slug": "canker-sores",
      "label": "Canker & cold sores",
      "category": "Eyes, ears & mouth",
      "searchTerms": [
        "canker sores",
        "cold sores",
        "mouth sores",
        "fever blisters",
        "oral pain"
      ]
    },
    {
      "slug": "toothache",
      "label": "Toothache",
      "category": "Eyes, ears & mouth",
      "searchTerms": [
        "toothache",
        "tooth pain",
        "dental pain",
        "oral analgesic"
      ]
    },
    {
      "slug": "sleeplessness",
      "label": "Sleeplessness",
      "category": "Sleep, stress & general",
      "searchTerms": [
        "sleeplessness",
        "insomnia",
        "sleep aid",
        "difficulty falling asleep",
        "nighttime"
      ]
    },
    {
      "slug": "dehydration",
      "label": "Dehydration",
      "category": "Sleep, stress & general",
      "searchTerms": [
        "dehydration",
        "oral rehydration",
        "electrolyte",
        "fluid loss",
        "replaces electrolytes"
      ]
    },
    {
      "slug": "hemorrhoids",
      "label": "Hemorrhoids",
      "category": "Sleep, stress & general",
      "searchTerms": [
        "hemorrhoids",
        "hemorrhoidal",
        "anal itching",
        "rectal"
      ]
    },
    {
      "slug": "yeast-infection",
      "label": "Vaginal yeast infection",
      "category": "Infections (OTC-treatable)",
      "searchTerms": [
        "vaginal yeast infection",
        "yeast infection",
        "vaginal antifungal",
        "candidiasis"
      ]
    },
    {
      "slug": "pinworm",
      "label": "Pinworm",
      "category": "Infections (OTC-treatable)",
      "searchTerms": [
        "pinworm",
        "pinworm infection",
        "pinworm treatment"
      ]
    },
    {
      "slug": "cold-sore-lip",
      "label": "Chapped & sun-protected lips",
      "category": "Sleep, stress & general",
      "searchTerms": [
        "lip protectant",
        "chapped lips",
        "sunburn protection lips",
        "lip balm"
      ]
    },
    {
      "slug": "eye-allergy",
      "label": "Eye allergies",
      "category": "Eyes, ears & mouth",
      "searchTerms": [
        "eye allergy",
        "itchy eyes due to allergies",
        "ocular itching",
        "allergic conjunctivitis"
      ]
    }
  ]
 }
--- a/admin/app/data/home_remedies.ts
+++ b/admin/app/data/home_remedies.ts
@ -1,328 +0,0 @@
 /* eslint-disable */
 /**
 * GENERATED FILE — DO NOT EDIT.
 *
 * Single source of truth: collections/home_remedies.json
 * Regenerate after editing the JSON: `npm run gen:curated-data` (from admin/).
 * curated_data_sync.standalone.ts fails CI if this file drifts from the JSON.
 *
 * Non-herbal home-care / self-care measures from US-government public-domain pages (CDC, NIH/NLM MedlinePlus, FDA); each entry carries its own sourceUrl.
 */
 import type { NaturalRemediesFile } from '../../types/conditions.js'
 export const HOME_REMEDIES_FILE: NaturalRemediesFile = {
  "version": "2026-06-10",
  "source": {
    "name": "US government health guidance (CDC, NIH/NHLBI, MedlinePlus/NLM, FDA)",
    "url": "https://www.cdc.gov",
    "license": "Public domain (US government works)"
  },
  "remedies": [
    {
      "slug": "honey-for-cough",
      "name": "Honey (for cough)",
      "commonNames": [],
      "conditions": [
        "cough",
        "sore-throat",
        "common-cold"
      ],
      "uses": "Honey may be used to relieve cough in adults and children at least 1 year old. One to two teaspoons can be taken directly or stirred into a warm (not hot) beverage.",
      "how": "Give one to two teaspoons of honey directly by mouth, or stir it into a warm (not hot) drink. The CDC recommends this as a home measure for cough associated with the common cold.",
      "evidence": "CDC lists honey among recommended home measures for easing cough and sore throat associated with the common cold; it is one of several non-medication strategies mentioned alongside rest, fluids, and humidifier use.",
      "cautions": "Never give honey to infants under 1 year old — it can contain Clostridium botulinum spores that cause infant botulism, a rare but serious illness. This warning applies regardless of honey type or brand.",
      "sourceUrl": "https://www.cdc.gov/common-cold/treatment/index.html"
    },
    {
      "slug": "fluids-and-rest",
      "name": "Fluids and rest",
      "commonNames": [],
      "conditions": [
        "common-cold",
        "fever",
        "diarrhea",
        "nausea-vomiting"
      ],
      "uses": "Getting plenty of rest and drinking adequate fluids (water, clear broths, juice, or sports drinks) supports recovery from colds, fever, diarrhea, and nausea. Adults with diarrhea should drink water, fruit juices, sports drinks, sodas without caffeine, and salty broths.",
      "how": "Get plenty of rest and drink plenty of fluids. If keeping liquids down is difficult, take small sips of water or suck on ice chips frequently rather than trying to drink large amounts at once.",
      "evidence": "CDC recommends rest and fluids as primary home care measures for the common cold. MedlinePlus (NIH/NIDDK) similarly lists these as the foundation of diarrhea self-care, and advises nausea patients to take in small amounts of clear liquids often to stay hydrated.",
      "cautions": "Severely ill individuals, those with signs of dehydration (no urination, sunken eyes, extreme thirst), or those unable to keep any fluid down should seek medical care promptly. Caffeine and alcohol are not effective rehydration choices.",
      "sourceUrl": "https://www.cdc.gov/common-cold/treatment/index.html"
    },
    {
      "slug": "oral-rehydration",
      "name": "Oral rehydration solution (ORS)",
      "commonNames": [
        "ORS",
        "rehydration salts"
      ],
      "conditions": [
        "dehydration",
        "diarrhea"
      ],
      "uses": "Oral rehydration solutions replace fluids and electrolytes lost through diarrhea or other causes of dehydration. For mild to moderate dehydration, drinking water is the first step; sports drinks or oral rehydration solutions (such as Pedialyte) are recommended when electrolytes are also depleted, especially for children.",
      "how": "Use a commercially prepared oral rehydration solution (available without a prescription) and follow the package directions. For adults with electrolyte losses, sports drinks can help; if liquids are hard to keep down, take small sips frequently or suck on ice chips rather than drinking large amounts at once.",
      "evidence": "MedlinePlus (NIH/NIDDK) states that treatment for dehydration involves replacing lost fluids and electrolytes, and that oral rehydration solutions for children are available without a prescription. The same source recommends sports drinks for adults when electrolytes have been lost alongside fluids.",
      "cautions": "Seek immediate medical care for signs of severe dehydration: no urination for 8 or more hours, rapid heartbeat, confusion, or inability to keep fluids down. Infants and small children with diarrhea should use formulated ORS (not plain water) to replace electrolytes safely.",
      "sourceUrl": "https://medlineplus.gov/dehydration.html"
    },
    {
      "slug": "cool-compress-fever",
      "name": "Cool compress (for fever and insect bites)",
      "commonNames": [],
      "conditions": [
        "fever",
        "insect-bites-stings",
        "eye-allergy",
        "eye-irritation"
      ],
      "uses": "A clean cloth soaked in cool (not ice-cold) water and placed on the forehead or bitten area can help reduce discomfort from fever, insect bites, and eye allergy symptoms. For insect stings, ice wrapped in a washcloth should be applied for 10 minutes on and 10 minutes off.",
      "how": "Soak a clean cloth in cool water, wring it out, and place it on the forehead or affected area. For insect stings, wrap ice in a cloth and apply for 10 minutes on, then 10 minutes off — never place ice directly on bare skin.",
      "evidence": "MedlinePlus (NIH/NIAID) notes that applying cool compresses is recommended for allergic conjunctivitis and eye burning and irritation. For insect bites and stings, a cool or iced compress is a standard first-line self-care step described in MedlinePlus search guidance consistent with NIH resources.",
      "cautions": "Do not apply ice or an ice-cold compress directly to bare skin for extended periods — wrap ice in cloth and limit applications to 10–15 minutes to avoid frostbite or tissue damage. For fever, cool compresses supplement (but do not replace) appropriate fever-reducing medicine when indicated; consult a healthcare provider if fever is high, prolonged, or accompanied by severe symptoms.",
      "sourceUrl": "https://medlineplus.gov/insectbitesandstings.html"
    },
    {
      "slug": "ice-and-elevation",
      "name": "Ice and elevation (RICE method)",
      "commonNames": [
        "RICE",
        "Rest-Ice-Compression-Elevation"
      ],
      "conditions": [
        "muscle-joint-pain",
        "insect-bites-stings"
      ],
      "uses": "Applying ice wrapped in cloth to a strained muscle, sprain, or bite site — combined with rest, compression, and elevation of the injured area — reduces swelling and pain. Ice should be applied for 10–15 minutes every 1–3 hours during the first few days of injury.",
      "how": "Wrap ice in a cloth or towel and apply to the injured area for 10–15 minutes at a time. Rest the area, wrap it snugly with a bandage to reduce swelling, and elevate it above heart level when possible.",
      "evidence": "MedlinePlus (NIH/NIAMS) describes the RICE method as standard first-line treatment for sprains and strains: resting the area, icing it, compressing it with a bandage, and elevating it above heart level when possible. Ice use for the first 3 days is specifically mentioned.",
      "cautions": "Never apply ice directly to skin — always wrap it in a cloth or towel. If swelling worsens significantly, numbness develops, or you suspect a fracture, seek medical evaluation. Do not use heat during the first 48–72 hours of an acute soft-tissue injury.",
      "sourceUrl": "https://medlineplus.gov/sprainsandstrains.html"
    },
    {
      "slug": "heating-pad",
      "name": "Heating pad or warm compress",
      "commonNames": [],
      "conditions": [
        "menstrual-cramps",
        "muscle-joint-pain",
        "earache"
      ],
      "uses": "Applying a heating pad or hot water bottle to the lower abdomen eases menstrual cramps. A warm compress applied to the ear can relieve earache discomfort. Heat may also be used on strained muscles after the first 48–72 hours of an acute injury.",
      "how": "Place a heating pad or hot water bottle on the lower abdomen for menstrual cramps, or hold a warm cloth against the affected ear for earache. Use a low or medium heat setting and put a cloth between the pad and skin to prevent burns.",
      "evidence": "MedlinePlus (NIH/NLM) lists using a heating pad or hot water bottle on the lower abdomen, along with taking a warm bath, as home care measures for period pain. For earache, placing a warm cloth on the affected ear is listed among comfort measures in NIH/NLM resources for acute ear infection self-care.",
      "cautions": "Never fall asleep with a heating pad on — burns can result. Use a low or medium setting and place a cloth between the pad and skin. For ear pain, do not insert anything into the ear canal; if pain is severe, accompanied by drainage, hearing loss, or fever, see a healthcare provider to rule out infection requiring antibiotics.",
      "sourceUrl": "https://medlineplus.gov/periodpain.html"
    },
    {
      "slug": "humidifier-and-steam",
      "name": "Humidifier or steam inhalation",
      "commonNames": [
        "cool-mist vaporizer",
        "steam inhalation"
      ],
      "conditions": [
        "nasal-congestion",
        "cough",
        "common-cold"
      ],
      "uses": "Using a clean humidifier or cool-mist vaporizer adds moisture to the air and can help relieve nasal congestion and cough. Breathing steam from a bowl of hot water or a running shower 2–4 times daily also loosens nasal secretions.",
      "how": "Fill a clean cool-mist humidifier or vaporizer with water and run it in the room. Clean the device daily per the manufacturer's instructions to prevent mold and bacteria buildup.",
      "evidence": "CDC recommends using a clean humidifier or cool-mist vaporizer as a home care measure for common cold symptoms including congestion and cough. MedlinePlus similarly notes that a humidifier can break up mucus and that steam inhalation from a shower is a recognized congestion-relief strategy.",
      "cautions": "Clean the humidifier daily per manufacturer instructions to prevent mold and bacterial growth. Use cool-mist humidifiers rather than warm-mist (steam) versions for children to avoid burn risk. When inhaling steam from hot water, use caution to avoid scalding; keep a safe distance and do not cover your head over a pot of boiling water.",
      "sourceUrl": "https://www.cdc.gov/common-cold/treatment/index.html"
    },
    {
      "slug": "saline-nasal-rinse",
      "name": "Saline nasal rinse",
      "commonNames": [
        "neti pot",
        "nasal irrigation",
        "saline nasal wash"
      ],
      "conditions": [
        "nasal-congestion",
        "common-cold"
      ],
      "uses": "Saline nasal rinses flush pollen, dust, and excess mucus from the nasal passages and add moisture. They can be performed with a neti pot, squeeze bottle, or bulb syringe using a prepared saline solution.",
      "how": "Use only distilled, sterile, or previously boiled-and-cooled water — never tap water. After each use, rinse the device with the same safe water, then air-dry it thoroughly or wipe dry before storing.",
      "evidence": "CDC recommends saline nasal spray or drops as a home care measure for the common cold. The FDA confirms that nasal irrigation devices are 'usually safe and effective products when used and cleaned properly,' with the critical safety requirement being the type of water used.",
      "cautions": "Use only distilled, sterile, or previously boiled (and cooled) water — never tap water. The FDA warns that tap water can harbor organisms including amoebas that are safe to swallow but can cause serious or potentially fatal infections in the nasal passages. Boiled water should be cooled to lukewarm and stored in a clean, closed container for no more than 24 hours. Always clean and dry the device after each use.",
      "sourceUrl": "https://www.fda.gov/consumers/consumer-updates/rinsing-your-sinuses-neti-pots-safe"
    },
    {
      "slug": "oatmeal-bath",
      "name": "Oatmeal or cool bath",
      "commonNames": [
        "colloidal oatmeal bath"
      ],
      "conditions": [
        "skin-rash-itch",
        "poison-ivy",
        "dry-skin"
      ],
      "uses": "Soaking in a lukewarm oatmeal bath or taking a cool bath can relieve itching and skin irritation from rashes, poison ivy, eczema, and dry skin. Colloidal oatmeal bath products are available at drugstores.",
      "how": "Fill a tub with lukewarm (not hot) water and add a colloidal oatmeal bath product according to package directions, or use plain cool water. After soaking, pat skin dry gently and apply a fragrance-free moisturizer immediately to lock in moisture.",
      "evidence": "MedlinePlus (NLM) recommends taking 'lukewarm or oatmeal baths' as a self-care measure for itching, alongside cool compresses and moisturizing lotion. Oatmeal bath products are specifically noted to relieve symptoms of eczema and psoriasis; short, cooler baths are described as better than long, hot baths for skin conditions.",
      "cautions": "Use lukewarm — not hot — water; hot water can worsen skin dryness and irritation. After bathing, pat skin dry gently (do not rub) and immediately apply a fragrance-free moisturizer to lock in moisture. If a rash is spreading rapidly, is accompanied by fever, or involves the face or genitals, consult a healthcare provider.",
      "sourceUrl": "https://medlineplus.gov/itching.html"
    },
    {
      "slug": "cool-running-water-on-burns",
      "name": "Cool running water on burns",
      "commonNames": [],
      "conditions": [
        "burns"
      ],
      "uses": "For minor burns, immediately run cool (not cold) water slowly over the burned area for 10–15 minutes to stop the burning process and reduce pain. After cooling, cover the burn with a clean, dry cloth or sterile bandage.",
      "how": "Run cool water slowly over the burned area for several minutes, then cover with a clean, dry cloth or bandage. Do not apply ice, butter, or any creams — these can worsen tissue damage.",
      "evidence": "CDC burn first-aid materials instruct: run cool water slowly over the burn area for several minutes, then cover with a clean, dry cloth or bandage. MedlinePlus (NIH/NIGMS) similarly specifies cool running water for 10–15 minutes followed by a dry sterile dressing as the appropriate first-aid response for minor burns.",
      "cautions": "Do not apply ice, ice water, butter, first-aid creams, sprays, or home remedies — these can worsen tissue damage or introduce infection. Do not break blisters unless directed by a healthcare provider. Do not try to remove clothing or debris stuck to the burn. Seek immediate medical care for large burns, burns on the face, eyes, hands, or feet, burns from chemicals or electricity, or any burn with extreme pain, numbness, or deep tissue involvement.",
      "sourceUrl": "https://medlineplus.gov/burns.html"
    },
    {
      "slug": "clean-and-cover-wounds",
      "name": "Clean and cover wounds and cuts",
      "commonNames": [],
      "conditions": [
        "wounds-cuts"
      ],
      "uses": "For minor cuts and scrapes, rinse the wound thoroughly with cool clean water to remove dirt, apply gentle pressure with gauze to stop bleeding, then cover with a clean bandage. Wash with soap and water to reduce infection risk.",
      "how": "Apply firm but gentle pressure with gauze to stop bleeding; if blood soaks through, add more gauze on top without removing the first layer. Rinse the wound with cool clean water, then cover it with a clean dry bandage.",
      "evidence": "CDC guidance on wound care states: put pressure on a bleeding cut until it stops, gently pour clean water over the wound to clean it, then apply a clean, dry bandage. MedlinePlus (NLM) similarly advises rinsing cuts with cool water and applying firm but gentle pressure to stop bleeding.",
      "cautions": "Watch for signs of infection in the days following — increasing redness, swelling, warmth, pus, or red streaks spreading from the wound require prompt medical evaluation. Seek immediate care for wounds that are deep, gaping, caused by an animal or human bite, or associated with a puncture from a potentially contaminated object (tetanus risk).",
      "sourceUrl": "https://medlineplus.gov/firstaid.html"
    },
    {
      "slug": "salt-water-gargle",
      "name": "Salt-water gargle",
      "commonNames": [],
      "conditions": [
        "sore-throat",
        "common-cold",
        "canker-sores"
      ],
      "uses": "Gargling with warm salt water several times a day can ease sore throat pain and may also help relieve canker sore discomfort. A standard preparation is ½ teaspoon (3 grams) of salt dissolved in 1 cup (240 mL) of warm water.",
      "how": "Dissolve salt in a cup of warm water, take a mouthful, tilt your head back, and gargle for several seconds before spitting it out. Repeat as needed throughout the day to help ease sore throat pain.",
      "evidence": "MedlinePlus (NLM) states that gargling may ease sore throat pain, listing it alongside lozenges and fluids. The same resource notes that salt-water rinses may help with canker sore discomfort, though mouthwashes containing alcohol should be avoided as they irritate the tissue.",
      "cautions": "Gargling with salt water provides symptomatic relief only and does not treat the underlying cause of a sore throat. Sore throat accompanied by high fever, difficulty swallowing or breathing, drooling, a stiff neck, or lasting more than a week should be evaluated by a healthcare provider, as strep throat and other conditions require different treatment.",
      "sourceUrl": "https://medlineplus.gov/sorethroat.html"
    },
    {
      "slug": "fiber-and-water-constipation",
      "name": "Dietary fiber and water (for constipation and hemorrhoids)",
      "commonNames": [],
      "conditions": [
        "constipation",
        "hemorrhoids"
      ],
      "uses": "Eating more fruits, vegetables, and whole grains (which are high in fiber) and drinking plenty of water each day are the foundational self-care steps for preventing and relieving constipation and reducing hemorrhoid discomfort.",
      "how": "Eat more fruits, vegetables, and whole grains each day and drink plenty of fluids. Increase fiber gradually to avoid gas and bloating.",
      "evidence": "MedlinePlus (NIH/NIDDK) lists increased dietary fiber and adequate fluid intake as primary self-care measures for both constipation and hemorrhoids. The hemorrhoids topic page specifically recommends eating high-fiber foods and drinking enough fluids every day as the first-line home treatment.",
      "cautions": "Increase dietary fiber gradually to avoid gas and bloating. If constipation is new, severe, accompanied by blood in the stool, or associated with significant weight loss, see a healthcare provider to rule out underlying conditions. Hemorrhoid symptoms persisting beyond one week of home treatment, or any rectal bleeding, warrant medical evaluation.",
      "sourceUrl": "https://medlineplus.gov/constipation.html"
    },
    {
      "slug": "sitz-bath",
      "name": "Sitz bath (for hemorrhoids)",
      "commonNames": [],
      "conditions": [
        "hemorrhoids"
      ],
      "uses": "A sitz bath — sitting in a few inches of warm water for 10–15 minutes, several times a day — relieves the pain and itching of hemorrhoids. A special sitz bath tub that fits over a toilet is available at pharmacies.",
      "how": "Fill a tub or sitz bath basin with a few inches of comfortably warm water and sit in it for 10 to 15 minutes. Repeat several times a day, keeping the area clean and dry between baths.",
      "evidence": "MedlinePlus (NLM) lists taking warm baths several times a day, including sitz baths, as a recommended home care measure to relieve hemorrhoid pain. The recommendation is to sit in warm water for 10 to 15 minutes per session.",
      "cautions": "The water should be comfortably warm — not hot — to avoid burns. Keep the area clean and dry between baths. If hemorrhoid symptoms do not improve after one week of home treatment, or if there is rectal bleeding, see a healthcare provider.",
      "sourceUrl": "https://medlineplus.gov/hemorrhoids.html"
    },
    {
      "slug": "elevate-head-heartburn",
      "name": "Elevate head of bed (for heartburn)",
      "commonNames": [],
      "conditions": [
        "heartburn",
        "indigestion"
      ],
      "uses": "Raising the head of the bed 4–6 inches (using blocks under the bed frame or a wedge support) prevents stomach acid from backing up into the esophagus during sleep, reducing nighttime heartburn and GERD symptoms.",
      "how": "Place blocks under the legs at the head of the bed frame, or use a foam wedge under the mattress, to raise the sleeping surface about 6 inches. Using extra pillows under only the head is less effective because it bends the body at the waist rather than tilting the whole torso.",
      "evidence": "MedlinePlus heartburn resources (sourced from NIH/NIDDK) consistently recommend elevating the head during sleep as a lifestyle measure for heartburn and GERD, noting that this position helps prevent reflux. Sleeping with the head raised about 6 inches is a specific recommendation described in the Medical Encyclopedia entry.",
      "cautions": "Using extra pillows under the head is less effective than raising the entire upper body — pillows can cause neck strain and do not sufficiently change the angle. Heartburn that is frequent, severe, or accompanied by difficulty swallowing, unexplained weight loss, or vomiting blood requires medical evaluation.",
      "sourceUrl": "https://medlineplus.gov/heartburn.html"
    },
    {
      "slug": "bland-small-meals",
      "name": "Small, frequent bland meals",
      "commonNames": [
        "BRAT diet",
        "bland diet"
      ],
      "conditions": [
        "nausea-vomiting",
        "indigestion",
        "diarrhea"
      ],
      "uses": "Eating 6–8 small bland meals throughout the day (crackers, toast, baked chicken, rice, potatoes) instead of 3 large meals reduces nausea and eases indigestion. As diarrhea symptoms improve, soft bland foods can be introduced gradually.",
      "how": "Eat smaller meals more often and stick to bland foods, avoiding spicy, fatty, or salty options. If you have trouble keeping food down, start with small sips of clear liquids frequently and add bland solids only when tolerated.",
      "evidence": "MedlinePlus (NLM) recommends small, frequent bland meals and avoiding spicy, fatty, or salty foods as the primary dietary self-care for nausea and vomiting. For indigestion, MedlinePlus notes that avoiding foods and situations that trigger symptoms is the main home strategy. For diarrhea, 'soft, bland food' is recommended as symptoms improve.",
      "cautions": "The BRAT diet (bananas, rice, applesauce, toast) was historically promoted but MedlinePlus notes there is not strong evidence it is better than a standard bland diet; it probably does not cause harm. If nausea or vomiting persists beyond 24–48 hours, is accompanied by severe pain, or prevents adequate fluid intake, seek medical care.",
      "sourceUrl": "https://medlineplus.gov/nauseaandvomiting.html"
    },
    {
      "slug": "dark-quiet-room-headache",
      "name": "Rest in a dark, quiet room (for headache)",
      "commonNames": [],
      "conditions": [
        "headache",
        "sleeplessness"
      ],
      "uses": "Resting with eyes closed in a dark, quiet room is a recommended non-medication self-care measure during a headache or migraine. Drinking water to prevent dehydration and placing a cool cloth on the forehead are often combined with this rest.",
      "how": "Go to a quiet, darkened room, close your eyes, and rest. Drink water and place a cool damp cloth on your forehead to help ease discomfort.",
      "evidence": "MedlinePlus (NLM) describes resting in a quiet, darkened room as one of the key things you can do to feel better during a headache or migraine, alongside drinking water and using relaxation techniques. These recommendations are attributed to NIH sources on headache and migraine management.",
      "cautions": "Sudden severe ('thunderclap') headache, headache with fever, stiff neck, confusion, or vision changes may signal a serious condition and require emergency evaluation. Frequent headaches that disrupt daily life should be discussed with a healthcare provider rather than managed solely at home.",
      "sourceUrl": "https://medlineplus.gov/headache.html"
    },
    {
      "slug": "sleep-hygiene",
      "name": "Sleep hygiene practices",
      "commonNames": [
        "good sleep habits"
      ],
      "conditions": [
        "sleeplessness"
      ],
      "uses": "A consistent set of behavioral practices — consistent bedtime and wake time, a cool and dark bedroom, avoiding screens and caffeine near bedtime, and regular exercise — helps adults achieve and maintain adequate sleep.",
      "how": "Go to bed and wake up at the same time every day. Keep the bedroom quiet, dark, and cool; turn off electronic devices at least 30 minutes before bedtime; and avoid caffeine in the afternoon and evening and large meals or alcohol before bed.",
      "evidence": "CDC and NHLBI both recommend these specific practices for healthy sleep: going to bed and waking at the same time daily, keeping the bedroom quiet, cool, and dark, turning off screens at least 30 minutes before bed, avoiding caffeine in the afternoon and evening, and avoiding large meals or alcohol before sleep. NHLBI notes these habits are particularly important for shift workers and people with insomnia.",
      "cautions": "Good sleep hygiene can help relieve short-term insomnia; persistent insomnia lasting more than a few weeks should be evaluated by a healthcare provider. Sleep difficulties accompanied by snoring, gasping during sleep, or excessive daytime sleepiness may indicate obstructive sleep apnea, which requires medical diagnosis.",
      "sourceUrl": "https://www.cdc.gov/sleep/about/index.html"
    },
    {
      "slug": "pinworm-hygiene",
      "name": "Hygiene measures for pinworm",
      "commonNames": [],
      "conditions": [
        "pinworm"
      ],
      "uses": "Thorough handwashing with soap and warm water (especially after toilet use and before eating), daily morning bathing with soap and water, daily underwear changes, short and clean fingernails, and laundering of bedding and pajamas in hot water are the key household self-care measures for managing pinworm infection.",
      "how": "Bathe after waking up each morning and wash hands regularly, especially after using the bathroom. Change underwear daily, wash pajamas and bed sheets often, and avoid nail biting to prevent reinfection.",
      "evidence": "CDC identifies handwashing as 'the most important way to prevent the spread of pinworms.' MedlinePlus (NIH/NIAID) lists bathe after waking up, wash pajamas and bed sheets often, wash hands regularly, change underwear every day, and avoid nail biting and scratching the anal area as the core preventive hygiene steps.",
      "cautions": "Hygiene alone typically cannot eliminate an active pinworm infection — over-the-counter antiparasitic medication (pyrantel pamoate) is generally needed, and all household members should be treated simultaneously. Medication is typically repeated after 2 weeks because it kills worms but not eggs; the second dose treats worms that hatched after the first dose.",
      "sourceUrl": "https://medlineplus.gov/pinworms.html"
    },
    {
      "slug": "keep-feet-dry-athlete-foot",
      "name": "Keep feet clean and dry (for fungal infection)",
      "commonNames": [
        "athlete's foot self-care"
      ],
      "conditions": [
        "fungal-infection"
      ],
      "uses": "Keeping the feet clean, dry, and cool — including washing daily with soap and water, drying carefully between the toes, wearing clean cotton socks, and not walking barefoot in public showers or locker rooms — supports treatment and prevents spread of athlete's foot and other tinea infections.",
      "how": "Keep feet clean, dry, and cool; wear clean socks and avoid walking barefoot in public areas such as locker room showers (use flip-flops instead). Apply an over-the-counter antifungal cream as directed on the package for most cases of athlete's foot.",
      "evidence": "MedlinePlus (CDC-sourced) advises: keep your feet clean, dry, and cool; wear clean socks; avoid walking barefoot in public areas; wear flip-flops in locker room showers; keep toenails clean and clipped short. Over-the-counter antifungal creams work for most cases of athlete's foot.",
      "cautions": "If over-the-counter antifungal treatment does not improve the infection within 2–4 weeks, see a healthcare provider. Spreading or worsening redness, warmth, and swelling — especially in people with diabetes or circulatory problems — warrants prompt medical care. Nail fungal infections are more difficult to treat than skin infections and often require prescription therapy.",
      "sourceUrl": "https://medlineplus.gov/athletesfoot.html"
    }
  ]
 }
--- a/admin/app/data/natural_remedies.ts
+++ b/admin/app/data/natural_remedies.ts
@ -1,316 +0,0 @@
 /* eslint-disable */
 /**
 * GENERATED FILE — DO NOT EDIT.
 *
 * Single source of truth: collections/natural_remedies.json
 * Regenerate after editing the JSON: `npm run gen:curated-data` (from admin/).
 * curated_data_sync.standalone.ts fails CI if this file drifts from the JSON.
 *
 * Hand-curated subset of NCCIH "Herbs at a Glance" (nccih.nih.gov), mapped to the condition spine. US-government / public domain; attribution lives in the file `source` field and the UI caveat.
 */
 import type { NaturalRemediesFile } from '../../types/conditions.js'
 export const NATURAL_REMEDIES_FILE: NaturalRemediesFile = {
  "version": "2026-06-10",
  "source": {
    "name": "NCCIH — Herbs at a Glance",
    "url": "https://www.nccih.nih.gov/health/herbsataglance",
    "license": "Public domain (US government work; NCCIH)"
  },
  "remedies": [
    {
      "slug": "aloe-vera",
      "name": "Aloe Vera",
      "commonNames": [
        "Aloe barbadensis miller"
      ],
      "conditions": [
        "burns",
        "acne",
        "dry-skin"
      ],
      "uses": "Aloe vera gel is applied topically for burns, acne, and various skin conditions including psoriasis and radiation-related skin damage. Oral use is promoted for digestive conditions, though topical applications have more research support.",
      "how": "Apply gel from the inner leaf of the aloe plant (or a commercially prepared aloe gel) directly to the affected skin area. Topical use is the application with the most research support; oral aloe products carry more serious risks and should not be used without consulting a healthcare provider.",
      "evidence": "Research suggests topical aloe gel may speed burn healing and reduce burn-related pain. Two small studies indicate aloe gel, combined with other treatments, may improve acne; evidence for other skin uses is limited.",
      "cautions": "Topical use is generally well tolerated, though occasional burning or itching may occur. Oral use carries more serious risks: the latex can cause abdominal pain and diarrhea, oral extracts have been linked to cases of acute hepatitis, and animal studies associated non-decolorized extracts with gastrointestinal cancer. Likely unsafe during pregnancy; may interact with medications such as digoxin.",
      "sourceUrl": "https://www.nccih.nih.gov/health/aloe-vera"
    },
    {
      "slug": "boswellia",
      "name": "Boswellia",
      "commonNames": [
        "Boswellia serrata",
        "Indian frankincense"
      ],
      "conditions": [
        "muscle-joint-pain"
      ],
      "uses": "Boswellia resin extract is traditionally used to reduce inflammation and pain, and is promoted as a dietary supplement to support joint health and mobility, particularly for osteoarthritis.",
      "how": "Boswellia is taken orally as a dietary supplement. Clinical trials have used doses up to 1,000 mg daily for up to 6 months; follow the product label and consult a healthcare provider before starting.",
      "evidence": "Some studies suggest oral boswellia may help reduce inflammation and pain associated with osteoarthritis, but larger rigorous trials are needed; topical use lacks sufficient evidence of effectiveness.",
      "cautions": "Extracts up to 1,000 mg daily have been used safely in clinical trials lasting up to 6 months. Consult a healthcare provider before use, especially if pregnant, breastfeeding, managing asthma, or taking medications, as interactions remain unclear.",
      "sourceUrl": "https://www.nccih.nih.gov/health/boswellia"
    },
    {
      "slug": "bromelain",
      "name": "Bromelain",
      "commonNames": [
        "pineapple enzyme",
        "Ananas comosus extract"
      ],
      "conditions": [
        "muscle-joint-pain",
        "nasal-congestion"
      ],
      "uses": "Bromelain, an enzyme from pineapple, is promoted for reducing postoperative pain and swelling, sinusitis, osteoarthritis, and exercise-induced muscle soreness. A topical formulation has FDA approval for debridement of severe burns.",
      "evidence": "Some studies suggest oral bromelain may reduce certain symptoms after wisdom tooth surgery; evidence for sinusitis is insufficient. The topical formulation has been successfully used by health professionals for burn debridement as an alternative to surgical debridement.",
      "cautions": "Oral bromelain is generally well tolerated; the most common side effects are stomach upset and diarrhea. Talk with a healthcare provider before use alongside any medications, as harmful interactions are possible. Safety during pregnancy and breastfeeding is unclear.",
      "sourceUrl": "https://www.nccih.nih.gov/health/bromelain"
    },
    {
      "slug": "butterbur",
      "name": "Butterbur",
      "commonNames": [
        "Petasites hybridus"
      ],
      "conditions": [
        "headache",
        "allergic-reaction"
      ],
      "uses": "Butterbur root extract is used for migraine prevention and for reducing symptoms of allergic rhinitis (hay fever). It has been studied for reducing migraine frequency in both adults and children.",
      "how": "Butterbur is taken orally as a root extract for migraine prevention or as a leaf extract for allergic rhinitis symptoms. Use only products that are certified and labeled as free of pyrrolizidine alkaloids (PA-free), as the untreated plant contains compounds that can damage the liver.",
      "evidence": "Studies of a butterbur root extract suggest it may reduce migraine frequency; a leaf extract may help with allergic rhinitis symptoms. However, the American Academy of Neurology withdrew its 2012 recommendation in 2015 due to safety concerns.",
      "cautions": "The plant naturally contains pyrrolizidine alkaloids (PAs) that can damage the liver and lungs and may cause cancer; only PA-free certified products should be used. Even PA-free products have been linked to rare cases of liver injury. Side effects include belching, diarrhea, drowsiness, rash, and stomach upset. Avoid during pregnancy; people allergic to ragweed or related plants are at higher risk of reactions.",
      "sourceUrl": "https://www.nccih.nih.gov/health/butterbur"
    },
    {
      "slug": "chamomile",
      "name": "Chamomile",
      "commonNames": [
        "Matricaria chamomilla",
        "German chamomile",
        "Chamaemelum nobile"
      ],
      "conditions": [
        "sleeplessness",
        "indigestion",
        "common-cold",
        "sore-throat",
        "skin-rash-itch"
      ],
      "uses": "Chamomile is promoted for insomnia, indigestion, anxiety, the common cold, and infant colic; it is also used topically for skin conditions and as a mouthwash for oral inflammation.",
      "how": "Chamomile is commonly consumed as a tea (considered safe in amounts typically found in teas), taken as an oral supplement, applied topically to skin, or used as a mouthwash for oral inflammation. Consult a healthcare provider before using oral supplements rather than tea.",
      "evidence": "Evidence is limited: some preliminary studies suggest chamomile supplements may help with anxiety, and combination products may help childhood diarrhea and infant colic. A 2019 review found minimal evidence for insomnia, with one study showing no benefit. Evidence for cold, sore throat, and skin uses in people is insufficient.",
      "cautions": "Generally considered safe in typical amounts. Side effects may include nausea, dizziness, and allergic reactions, including severe hypersensitivity; risk is higher for those sensitive to ragweed, chrysanthemums, marigolds, or daisies. May interact with blood thinners, birth control pills, sedatives, and liver-metabolized drugs. Safety during pregnancy and breastfeeding is unknown.",
      "sourceUrl": "https://www.nccih.nih.gov/health/chamomile"
    },
    {
      "slug": "echinacea",
      "name": "Echinacea",
      "commonNames": [
        "Echinacea purpurea",
        "purple coneflower"
      ],
      "conditions": [
        "common-cold"
      ],
      "uses": "Echinacea is primarily marketed for the common cold and upper respiratory tract infections, based on the idea that it may support immune system function.",
      "evidence": "Studies indicate that taking echinacea may slightly reduce the chances of catching a cold, though evidence regarding whether it shortens cold duration is inconclusive. Evidence for other conditions, including eczema, is unclear.",
      "cautions": "E. purpurea extracts appear likely safe for short periods in adults; allergic reactions can occur, with digestive symptoms most common. Children may experience rashes potentially linked to allergic reactions. Theoretical interactions with immunosuppressants and certain other drugs; consult a healthcare provider before use with medications. Limited data on safety in early pregnancy.",
      "sourceUrl": "https://www.nccih.nih.gov/health/echinacea"
    },
    {
      "slug": "elderberry",
      "name": "Elderberry",
      "commonNames": [
        "Sambucus nigra"
      ],
      "conditions": [
        "common-cold"
      ],
      "uses": "Elderberry has been used in folk medicine to treat colds and flu, and is promoted as a dietary supplement for colds, flu, and other upper respiratory infections.",
      "how": "Elderberry is used as a prepared dietary supplement (syrup, capsule, or lozenge) rather than raw fruit. Never eat raw or unripe elderberries — they contain cyanide-producing substances that cause nausea, vomiting, and severe diarrhea; cooking the berries eliminates this toxin.",
      "evidence": "A small number of studies suggest elderberry may relieve symptoms of flu, colds, or other upper respiratory infections; however, the overall evidence is limited and insufficient for most other claimed benefits.",
      "cautions": "Raw or unripe elderberries contain cyanide-producing substances that can cause nausea, vomiting, and severe diarrhea; cooking eliminates this toxin. Consult a healthcare provider before use, especially if taking medications. Limited safety data for pregnancy and breastfeeding.",
      "sourceUrl": "https://www.nccih.nih.gov/health/elderberry"
    },
    {
      "slug": "feverfew",
      "name": "Feverfew",
      "commonNames": [
        "Tanacetum parthenium"
      ],
      "conditions": [
        "headache"
      ],
      "uses": "Feverfew is promoted for migraine headache prevention and for minor head and tension pain. Topically, it is marketed for itching and skin irritation.",
      "evidence": "A 2020 systematic review of seven migraine studies found inconsistent results, though some evidence suggests feverfew may reduce migraine frequency and associated symptoms such as nausea and light sensitivity. There is little or no evidence supporting feverfew for other health conditions.",
      "cautions": "Side effects include nausea, digestive issues, bloating, and mouth sores from chewing fresh leaves. People sensitive to ragweed may experience allergic reactions. Feverfew may slow blood clotting and should be stopped at least 2 weeks before scheduled surgery. It may interact with migraine medications and should be avoided during pregnancy due to potential effects on uterine contractions.",
      "sourceUrl": "https://www.nccih.nih.gov/health/feverfew"
    },
    {
      "slug": "ginger",
      "name": "Ginger",
      "commonNames": [
        "Zingiber officinale"
      ],
      "conditions": [
        "nausea-vomiting",
        "menstrual-cramps",
        "indigestion"
      ],
      "uses": "Ginger is traditionally and commonly used for nausea and vomiting, indigestion, menstrual cramps, and osteoarthritis.",
      "evidence": "Research shows ginger may be helpful for nausea and vomiting associated with pregnancy. Studies suggest ginger dietary supplements might be helpful for reducing the severity of menstrual cramps and for knee osteoarthritis symptoms; effectiveness for chemotherapy- or post-surgery-related nausea remains uncertain.",
      "cautions": "Side effects when taken orally include abdominal discomfort, heartburn, diarrhea, and mouth and throat irritation. Possible interactions with medications exist; consult a healthcare provider before use, especially during pregnancy or breastfeeding.",
      "sourceUrl": "https://www.nccih.nih.gov/health/ginger"
    },
    {
      "slug": "goldenseal",
      "name": "Goldenseal",
      "commonNames": [
        "Hydrastis canadensis"
      ],
      "conditions": [
        "common-cold",
        "diarrhea",
        "wounds-cuts"
      ],
      "uses": "Goldenseal is marketed for the common cold and upper respiratory infections, diarrhea, constipation, and other digestive conditions. Historically, Native Americans used it for digestive disorders, wounds, and skin conditions.",
      "evidence": "There is not enough evidence to determine whether goldenseal is useful for any health condition; no rigorous studies have been done in people. Very little of the active compound berberine is absorbed when goldenseal is taken orally, making findings from berberine studies potentially inapplicable.",
      "cautions": "Short-term use at approximately 3 grams daily appears to be without serious adverse effects, though longer-term safety is uncertain. Goldenseal significantly affects drug metabolism; one study found it decreased metformin levels by about 25%. Some commercial products contain unlisted ingredients or substitute herbs. Avoid during pregnancy, breastfeeding, and in infants due to potential harm from berberine.",
      "sourceUrl": "https://www.nccih.nih.gov/health/goldenseal"
    },
    {
      "slug": "horse-chestnut",
      "name": "Horse Chestnut",
      "commonNames": [
        "Aesculus hippocastanum"
      ],
      "conditions": [
        "hemorrhoids"
      ],
      "uses": "Horse chestnut seed extract is promoted for chronic venous insufficiency (leg pain and swelling from poor circulation) and has historically been used for hemorrhoids, arthritis, and menstrual cramps.",
      "how": "Use only standardized horse chestnut seed extract (a dietary supplement) — never raw seeds, bark, flowers, or leaves, which are toxic when taken orally. Research has used standardized extract for up to 12 weeks; follow the product label and consult a healthcare provider before use.",
      "evidence": "A 2012 systematic review found horse chestnut seed extract can improve symptoms of chronic venous insufficiency, comparable to compression stockings in one study; however, more rigorous trials are needed. Evidence for other uses, including hemorrhoids, is insufficient.",
      "cautions": "Raw seeds, bark, flowers, and leaves are unsafe to take orally due to toxic components; only standardized extracts with the toxin removed should be used, and research shows safety for up to 12 weeks. Side effects include dizziness, digestive upset, headache, and itching. Safety during pregnancy and breastfeeding is unknown; consult a healthcare provider before use.",
      "sourceUrl": "https://www.nccih.nih.gov/health/horse-chestnut"
    },
    {
      "slug": "lavender",
      "name": "Lavender",
      "commonNames": [
        "Lavandula angustifolia"
      ],
      "conditions": [
        "sleeplessness"
      ],
      "uses": "Lavender is promoted as an oral supplement and aromatherapy agent for calming anxiety, stress, and sleep difficulties; it is also used topically and in aromatherapy for depression symptoms and pain.",
      "how": "Lavender is used as an oral supplement (capsule or tea), as an aromatherapy oil (inhaled or diffused), or applied topically to skin. Oral lavender oil products have been studied for anxiety; aromatherapy involves inhaling the scent from a diffuser or a few drops on a cloth.",
      "evidence": "Studies suggest oral lavender oil might be beneficial for anxiety, including anxiety with co-occurring depression; evidence for sleep improvement is insufficient, and it is unclear whether aromatherapy with lavender benefits anxiety, stress, or depression.",
      "cautions": "Oral lavender products may cause diarrhea, headache, nausea, or burping. Topical application can trigger allergic skin reactions. Potential interactions with sedative drugs warrant discussion with a healthcare provider. Safety during pregnancy and breastfeeding is unknown.",
      "sourceUrl": "https://www.nccih.nih.gov/health/lavender"
    },
    {
      "slug": "licorice-root",
      "name": "Licorice Root",
      "commonNames": [
        "Glycyrrhiza glabra"
      ],
      "conditions": [
        "canker-sores",
        "sore-throat",
        "burns",
        "skin-rash-itch"
      ],
      "uses": "Licorice root is promoted for digestive and respiratory support and is used topically for skin conditions. Research has focused on mouth rinses for canker sores, gargles or lozenges for sore throat prevention, and topical gels for eczema and burn healing.",
      "how": "For canker sores, licorice is used as a mouth rinse or gargle; for sore throat prevention around surgery, as a gargle or lozenge; for eczema or burn healing, as a topical gel applied to the skin. Avoid long-term or high-dose oral use due to serious cardiovascular risks.",
      "evidence": "Preliminary studies suggest licorice mouth rinses may reduce canker sore pain and size, and lozenges or gargles may prevent sore throat after surgical intubation. Topical gels show some promise for eczema symptoms and burn healing, though more research is needed. There is not enough high-quality evidence to support its use for most conditions.",
      "cautions": "Licorice contains glycyrrhizin, which can cause serious adverse effects including irregular heartbeat and cardiac arrest, especially with long-term or large-dose use. Even small amounts can be risky for people with high blood pressure, high salt intake, or heart or kidney conditions. Large amounts during pregnancy increase miscarriage risk and are considered unsafe. Interactions with corticosteroids have been documented; consult a healthcare provider before use.",
      "sourceUrl": "https://www.nccih.nih.gov/health/licorice-root"
    },
    {
      "slug": "passionflower",
      "name": "Passionflower",
      "commonNames": [
        "Passiflora incarnata"
      ],
      "conditions": [
        "sleeplessness"
      ],
      "uses": "Passionflower is marketed as a dietary supplement for anxiety, sleep problems, and stress.",
      "how": "Passionflower is taken as a tea (studied for up to 7 nights) or as an oral extract supplement (studied for up to 8 weeks). Follow package directions and consult a healthcare provider before use, especially if you take other medications or are scheduled for surgery.",
      "evidence": "Some research suggests oral passionflower may help reduce anxiety symptoms and improve total sleep time in adults with insomnia, though findings on falling or staying asleep are mixed and conclusions are not definite.",
      "cautions": "Taking passionflower alongside anesthesia or other surgical medications may slow the nervous system too much; avoid use before surgery. Should not be used during pregnancy as it may induce uterine contractions. Possible side effects include drowsiness, dizziness, and confusion. Discuss with a healthcare provider before combining with medications.",
      "sourceUrl": "https://www.nccih.nih.gov/health/passionflower"
    },
    {
      "slug": "peppermint-oil",
      "name": "Peppermint Oil",
      "commonNames": [
        "Mentha x piperita"
      ],
      "conditions": [
        "headache",
        "nausea-vomiting",
        "indigestion"
      ],
      "uses": "Peppermint oil is promoted for irritable bowel syndrome, indigestion, headaches, muscle tension, and nausea. Topical application is used for tension headaches, and inhaled peppermint oil is used for nausea.",
      "how": "For IBS, peppermint oil is taken as enteric-coated capsules (which reduce heartburn compared to plain capsules). For tension headaches, apply peppermint oil topically to the skin. For nausea, peppermint oil is inhaled as aromatherapy.",
      "evidence": "A 2022 review found peppermint oil was better than placebo at improving overall IBS symptoms and abdominal pain. A 2024 review concluded that inhaling peppermint oil was particularly effective at reducing nausea and vomiting in patients with cancer. Evidence suggests potential benefit for tension headaches when applied topically, though peppermint oil taken alone may worsen indigestion in some people.",
      "cautions": "Oral side effects may include heartburn, nausea, abdominal pain, and dry mouth; enteric-coated capsules reduce heartburn risk. Never apply topically to infants' or young children's faces due to menthol's respiratory effects. Skin irritation is possible with topical use. Consult a healthcare provider before use with medications, as interactions may occur.",
      "sourceUrl": "https://www.nccih.nih.gov/health/peppermint-oil"
    },
    {
      "slug": "tea-tree-oil",
      "name": "Tea Tree Oil",
      "commonNames": [
        "Melaleuca alternifolia"
      ],
      "conditions": [
        "fungal-infection",
        "acne",
        "wounds-cuts",
        "insect-bites-stings",
        "burns"
      ],
      "uses": "Tea tree oil is marketed for external use including acne, athlete's foot, toenail fungus, and lice. Traditional Aboriginal uses included treating wounds, burns, and insect bites.",
      "how": "Apply tea tree oil topically to the affected skin area only — for acne or athlete's foot, use a product containing tea tree oil as directed on the label. Never swallow tea tree oil; oral ingestion can cause serious symptoms including confusion, unsteadiness, and coma.",
      "evidence": "A small amount of research suggests tea tree oil might be helpful for acne and athlete's foot (fungal infection), though more studies are needed. Evidence for toenail fungus, lice, eyelid inflammation, and oral health uses is insufficient or uncertain. Traditional wound, burn, and insect bite uses lack rigorous clinical research.",
      "cautions": "Critical warning: tea tree oil must not be swallowed — oral ingestion can cause serious symptoms including confusion, unsteadiness, inability to walk, and coma. Topical use is generally safe for most adults, though some experience skin redness or irritation. Older products or those exposed to heat, light, or air may increase reaction risk. Products appear safe during pregnancy and breastfeeding when used topically only.",
      "sourceUrl": "https://www.nccih.nih.gov/health/tea-tree-oil"
    },
    {
      "slug": "turmeric",
      "name": "Turmeric",
      "commonNames": [
        "Curcuma longa",
        "curcumin"
      ],
      "conditions": [
        "muscle-joint-pain",
        "indigestion",
        "skin-rash-itch"
      ],
      "uses": "Turmeric is promoted for osteoarthritis, itching, depression, allergies, and high cholesterol; topical applications target joint pain. Historically, it was used in traditional medicine for indigestion, colds, skin infections, and liver disease.",
      "how": "Turmeric is taken orally as a dietary supplement or applied topically to the skin. Conventionally formulated oral turmeric or curcumin supplements are considered likely safe in recommended amounts for up to 2 to 3 months; follow the product label and consult a healthcare provider before use.",
      "evidence": "For osteoarthritis, initial evidence is positive for knee pain relief and joint function, though higher-quality research is needed. For most other claimed uses, researchers cannot definitively confirm turmeric's effectiveness, and overall evidence remains inconclusive.",
      "cautions": "Standard turmeric formulations are likely safe in recommended amounts for up to 2–3 months. Common side effects include nausea, vomiting, acid reflux, and digestive issues. Enhanced bioavailability products have caused liver damage in some users; symptoms include fatigue, dark urine, and jaundice. Use during pregnancy may be unsafe; breastfeeding safety is unclear. Consult a healthcare provider before use with medications.",
      "sourceUrl": "https://www.nccih.nih.gov/health/turmeric"
    },
    {
      "slug": "valerian",
      "name": "Valerian",
      "commonNames": [
        "Valeriana officinalis"
      ],
      "conditions": [
        "sleeplessness"
      ],
      "uses": "Valerian is promoted for insomnia, anxiety, stress, and depression; historically, it was also used for migraine, fatigue, and stomach issues.",
      "how": "Valerian is taken orally as a dietary supplement; clinical studies have used doses of 300 to 600 mg daily for up to 6 weeks. Follow the product label and consult a healthcare provider before use, particularly if combining with sedatives or alcohol.",
      "evidence": "Evidence is limited and inconsistent. The American Academy of Sleep Medicine recommended against using valerian for chronic insomnia in adults (2017). Studies on anxiety, menstrual cramps, and other conditions show inadequate evidence of benefit.",
      "cautions": "Appears generally safe for short-term use at 300–600 mg daily for up to 6 weeks; long-term safety is unknown. Common side effects include headache, stomach upset, mental dullness, and vivid dreams. Abrupt discontinuation after long-term use may cause withdrawal symptoms including anxiety and insomnia. Avoid combining with alcohol or sedatives. Safety during pregnancy and breastfeeding is unclear.",
      "sourceUrl": "https://www.nccih.nih.gov/health/valerian"
    }
  ]
 }
--- a/admin/app/jobs/app_auto_update_job.ts
+++ b/admin/app/jobs/app_auto_update_job.ts
@ -1,78 +0,0 @@
 import { Job } from 'bullmq'
 import { QueueService } from '#services/queue_service'
 import { DockerService } from '#services/docker_service'
 import { DownloadService } from '#services/download_service'
 import { SystemService } from '#services/system_service'
 import { ContainerRegistryService } from '#services/container_registry_service'
 import { AppAutoUpdateService } from '#services/app_auto_update_service'
 import logger from '@adonisjs/core/services/logger'
 /**
 * Hourly job that evaluates whether any opted-in installed apps should auto-update
 * right now and, if so, updates them. All gating (master switch, per-app opt-in,
 * window, cool-off, pre-flight, per-app backoff) lives in {@link AppAutoUpdateService};
 * this job is just the scheduled trigger. Runs hourly so it can act anywhere inside a
 * user's window regardless of the window's length. Mirrors {@link AutoUpdateJob}.
 */
 export class AppAutoUpdateJob {
  static get queue() {
    return 'system'
  }
  static get key() {
    return 'app-auto-update'
  }
  async handle(_job: Job) {
    logger.info('[AppAutoUpdateJob] Evaluating app auto-updates...')
    const dockerService = new DockerService()
    const appAutoUpdateService = new AppAutoUpdateService(
      dockerService,
      new DownloadService(QueueService.getInstance()),
      new SystemService(dockerService),
      new ContainerRegistryService()
    )
    const result = await appAutoUpdateService.attempt()
    logger.info(`[AppAutoUpdateJob] ${result.updated} updated: ${result.reason}`)
    return result
  }
  static async schedule() {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    await queue.upsertJobScheduler(
      'hourly-app-auto-update',
      { pattern: '0 * * * *' }, // Top of every hour; attempt() gates on the window
      {
        name: this.key,
        opts: {
          removeOnComplete: { count: 12 },
          removeOnFail: { count: 5 },
        },
      }
    )
    logger.info('[AppAutoUpdateJob] App auto-update evaluation scheduled with cron: 0 * * * *')
  }
  static async dispatch() {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    const job = await queue.add(
      this.key,
      {},
      {
        attempts: 1,
        removeOnComplete: { count: 12 },
        removeOnFail: { count: 5 },
      }
    )
    logger.info(`[AppAutoUpdateJob] Dispatched ad-hoc app auto-update evaluation job ${job.id}`)
    return job
  }
 }
--- a/admin/app/jobs/auto_update_job.ts
+++ b/admin/app/jobs/auto_update_job.ts
@ -1,80 +0,0 @@
 import { Job } from 'bullmq'
 import { QueueService } from '#services/queue_service'
 import { DockerService } from '#services/docker_service'
 import { DownloadService } from '#services/download_service'
 import { SystemService } from '#services/system_service'
 import { SystemUpdateService } from '#services/system_update_service'
 import { ContainerRegistryService } from '#services/container_registry_service'
 import { AutoUpdateService } from '#services/auto_update_service'
 import logger from '@adonisjs/core/services/logger'
 /**
 * Hourly job that evaluates whether the NOMAD application should auto-update right
 * now and, if so, requests it. All gating (opt-in, window, eligibility, cool-off,
 * pre-flight, backoff) lives in {@link AutoUpdateService}; this job is just the
 * scheduled trigger. Runs hourly so it can act anywhere inside a user's window
 * regardless of the window's length.
 */
 export class AutoUpdateJob {
  static get queue() {
    return 'system'
  }
  static get key() {
    return 'auto-update'
  }
  async handle(_job: Job) {
    logger.info('[AutoUpdateJob] Evaluating auto-update...')
    const dockerService = new DockerService()
    const autoUpdateService = new AutoUpdateService(
      dockerService,
      new DownloadService(QueueService.getInstance()),
      new SystemService(dockerService),
      new SystemUpdateService(),
      new ContainerRegistryService()
    )
    const result = await autoUpdateService.attempt()
    logger.info(`[AutoUpdateJob] ${result.updated ? 'Updating' : 'No update'}: ${result.reason}`)
    return result
  }
  static async schedule() {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    await queue.upsertJobScheduler(
      'hourly-auto-update',
      { pattern: '0 * * * *' }, // Top of every hour; attempt() gates on the window
      {
        name: this.key,
        opts: {
          removeOnComplete: { count: 12 },
          removeOnFail: { count: 5 },
        },
      }
    )
    logger.info('[AutoUpdateJob] Auto-update evaluation scheduled with cron: 0 * * * *')
  }
  static async dispatch() {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    const job = await queue.add(
      this.key,
      {},
      {
        attempts: 1,
        removeOnComplete: { count: 12 },
        removeOnFail: { count: 5 },
      }
    )
    logger.info(`[AutoUpdateJob] Dispatched ad-hoc auto-update evaluation job ${job.id}`)
    return job
  }
 }
--- a/admin/app/jobs/check_service_updates_job.ts
+++ b/admin/app/jobs/check_service_updates_job.ts
@ -1,142 +0,0 @@
 import { Job } from 'bullmq'
 import { QueueService } from '#services/queue_service'
 import { DockerService } from '#services/docker_service'
 import { ContainerRegistryService } from '#services/container_registry_service'
 import Service from '#models/service'
 import logger from '@adonisjs/core/services/logger'
 import transmit from '@adonisjs/transmit/services/main'
 import { BROADCAST_CHANNELS } from '../../constants/broadcast.js'
 import { DateTime } from 'luxon'
 export class CheckServiceUpdatesJob {
  static get queue() {
    return 'service-updates'
  }
  static get key() {
    return 'check-service-updates'
  }
  async handle(_job: Job) {
    logger.info('[CheckServiceUpdatesJob] Checking for service updates...')
    const dockerService = new DockerService()
    const registryService = new ContainerRegistryService()
    // Determine host architecture
    const hostArch = await this.getHostArch(dockerService)
    const installedServices = await Service.query().where('installed', true)
    let updatesFound = 0
    for (const service of installedServices) {
      try {
        const updates = await registryService.getAvailableUpdates(
          service.container_image,
          hostArch,
          service.source_repo
        )
        const latestUpdate = updates.length > 0 ? updates[0].tag : null
        // Stamp/clear the cool-off anchor only when the available version *changes*.
        // Registry tags carry no publish date, so the auto-update cool-off is measured
        // from when a version was first detected; leaving the timestamp untouched while
        // the same version persists keeps the cool-off clock running.
        if (latestUpdate !== service.available_update_version) {
          service.available_update_first_seen_at = latestUpdate ? DateTime.now() : null
        }
        service.available_update_version = latestUpdate
        service.update_checked_at = DateTime.now()
        await service.save()
        if (latestUpdate) {
          updatesFound++
          logger.info(
            `[CheckServiceUpdatesJob] Update available for ${service.service_name}: ${service.container_image} → ${latestUpdate}`
          )
        }
      } catch (error) {
        logger.error(
          `[CheckServiceUpdatesJob] Failed to check updates for ${service.service_name}: ${error.message}`
        )
        // Continue checking other services
      }
    }
    logger.info(
      `[CheckServiceUpdatesJob] Completed. ${updatesFound} update(s) found for ${installedServices.length} service(s).`
    )
    // Broadcast completion so the frontend can refresh
    transmit.broadcast(BROADCAST_CHANNELS.SERVICE_UPDATES, {
      status: 'completed',
      updatesFound,
      timestamp: new Date().toISOString(),
    })
    return { updatesFound }
  }
  private async getHostArch(dockerService: DockerService): Promise<string> {
    try {
      const info = await dockerService.docker.info()
      const arch = info.Architecture || ''
      // Map Docker architecture names to OCI names
      const archMap: Record<string, string> = {
        x86_64: 'amd64',
        aarch64: 'arm64',
        armv7l: 'arm',
        amd64: 'amd64',
        arm64: 'arm64',
      }
      return archMap[arch] || arch.toLowerCase()
    } catch (error) {
      logger.warn(
        `[CheckServiceUpdatesJob] Could not detect host architecture: ${error.message}. Defaulting to amd64.`
      )
      return 'amd64'
    }
  }
  static async scheduleNightly() {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    await queue.upsertJobScheduler(
      'nightly-service-update-check',
      { pattern: '0 3 * * *' },
      {
        name: this.key,
        opts: {
          removeOnComplete: { count: 7 },
          removeOnFail: { count: 5 },
        },
      }
    )
    logger.info('[CheckServiceUpdatesJob] Service update check scheduled with cron: 0 3 * * *')
  }
  static async dispatch() {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    const job = await queue.add(
      this.key,
      {},
      {
        attempts: 3,
        backoff: { type: 'exponential', delay: 60000 },
        removeOnComplete: { count: 7 },
        removeOnFail: { count: 5 },
      }
    )
    logger.info(`[CheckServiceUpdatesJob] Dispatched ad-hoc service update check job ${job.id}`)
    return job
  }
 }
--- a/admin/app/jobs/check_update_job.ts
+++ b/admin/app/jobs/check_update_job.ts
@ -1,77 +0,0 @@
 import { Job } from 'bullmq'
 import { QueueService } from '#services/queue_service'
 import { DockerService } from '#services/docker_service'
 import { SystemService } from '#services/system_service'
 import logger from '@adonisjs/core/services/logger'
 import KVStore from '#models/kv_store'
 export class CheckUpdateJob {
  static get queue() {
    return 'system'
  }
  static get key() {
    return 'check-update'
  }
  async handle(_job: Job) {
    logger.info('[CheckUpdateJob] Running update check...')
    const dockerService = new DockerService()
    const systemService = new SystemService(dockerService)
    try {
      const result = await systemService.checkLatestVersion()
      if (result.updateAvailable) {
        logger.info(
          `[CheckUpdateJob] Update available: ${result.currentVersion} → ${result.latestVersion}`
        )
      } else {
        await KVStore.setValue('system.updateAvailable', false)
        logger.info(
          `[CheckUpdateJob] System is up to date (${result.currentVersion})`
        )
      }
      return result
    } catch (error) {
      logger.error(`[CheckUpdateJob] Update check failed: ${error.message}`)
      throw error
    }
  }
  static async scheduleNightly() {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    await queue.upsertJobScheduler(
      'nightly-update-check',
      { pattern: '0 2,14 * * *' }, // Every 12 hours at 2am and 2pm
      {
        name: this.key,
        opts: {
          removeOnComplete: { count: 7 },
          removeOnFail: { count: 5 },
        },
      }
    )
    logger.info('[CheckUpdateJob] Update check scheduled with cron: 0 2,14 * * *')
  }
  static async dispatch() {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    const job = await queue.add(this.key, {}, {
      attempts: 3,
      backoff: { type: 'exponential', delay: 60000 },
      removeOnComplete: { count: 7 },
      removeOnFail: { count: 5 },
    })
    logger.info(`[CheckUpdateJob] Dispatched ad-hoc update check job ${job.id}`)
    return job
  }
 }
--- a/admin/app/jobs/content_auto_update_job.ts
+++ b/admin/app/jobs/content_auto_update_job.ts
@ -1,82 +0,0 @@
 import { Job } from 'bullmq'
 import { QueueService } from '#services/queue_service'
 import { DownloadService } from '#services/download_service'
 import { ContentAutoUpdateService } from '#services/content_auto_update_service'
 import logger from '@adonisjs/core/services/logger'
 /**
 * Hourly job that evaluates whether any installed content (ZIM/map) should
 * auto-update right now and, if so, dispatches the downloads. All gating (master
 * switch, content window, cool-off, per-window data cap, pre-flight, backoff)
 * lives in {@link ContentAutoUpdateService}; this job is just the scheduled
 * trigger. Runs hourly so it can act anywhere inside a user's window regardless
 * of the window's length. Mirrors {@link AppAutoUpdateJob}.
 */
 export class ContentAutoUpdateJob {
  static get queue() {
    return 'system'
  }
  static get key() {
    return 'content-auto-update'
  }
  async handle(_job: Job) {
    logger.info('[ContentAutoUpdateJob] Evaluating content auto-updates...')
    const contentAutoUpdateService = new ContentAutoUpdateService(
      new DownloadService(QueueService.getInstance())
    )
    const result = await contentAutoUpdateService.attempt()
    logger.info(`[ContentAutoUpdateJob] ${result.started} started: ${result.reason}`)
    // The FDA drug dataset refreshes on the same cycle and master switch as the
    // ZIM/map catalog (its apply path differs, so it's a separate step, not part
    // of attempt()). Governed by the same `contentAutoUpdate.*` settings.
    const drugResult = await contentAutoUpdateService.attemptDrugDataset()
    logger.info(`[ContentAutoUpdateJob] ${drugResult.started} started: ${drugResult.reason}`)
    return {
      started: result.started + drugResult.started,
      reason: `${result.reason}; ${drugResult.reason}`,
    }
  }
  static async schedule() {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    await queue.upsertJobScheduler(
      'hourly-content-auto-update',
      { pattern: '0 * * * *' }, // Top of every hour; attempt() gates on the window
      {
        name: this.key,
        opts: {
          removeOnComplete: { count: 12 },
          removeOnFail: { count: 5 },
        },
      }
    )
    logger.info('[ContentAutoUpdateJob] Content auto-update evaluation scheduled with cron: 0 * * * *')
  }
  static async dispatch() {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    const job = await queue.add(
      this.key,
      {},
      {
        attempts: 1,
        removeOnComplete: { count: 12 },
        removeOnFail: { count: 5 },
      }
    )
    logger.info(`[ContentAutoUpdateJob] Dispatched ad-hoc content auto-update evaluation job ${job.id}`)
    return job
  }
 }
--- a/admin/app/jobs/download_drug_data_job.ts
+++ b/admin/app/jobs/download_drug_data_job.ts
@ -1,345 +0,0 @@
 import { Job } from 'bullmq'
 import { access, mkdir, constants } from 'node:fs/promises'
 import logger from '@adonisjs/core/services/logger'
 import { QueueService } from '#services/queue_service'
 import { doResumableDownload } from '../utils/downloads.js'
 import { parseDrugLabelManifest, partZipPath, manifestBytesTotal } from '../../util/drug_labels.js'
 import type {
  DownloadDrugDataJobParams,
  DrugLabelManifest,
  DownloadStateMarker,
  DrugDatasetResourceMeta,
 } from '../../types/drug_reference.js'
 /** Where all part zips are staged on the bind-mounted storage volume. */
 export const STORAGE_BASE = '/app/storage/drug-data'
 const MANIFEST_URL = 'https://api.fda.gov/download.json'
 /**
 * Phase A — Download (network-only failure domain).
 *
 * Pass 0 fetches the openFDA manifest; each later pass downloads ONE part to
 * disk (resumable via doResumableDownload, so a worker restart resumes from the
 * .tmp rather than re-downloading). Continuations use queue.add with NO jobId —
 * the same rule as the embed/ingest chains, so BullMQ dedupe doesn't swallow the
 * next part against the lingering parent. After the last part lands we write the
 * `drugReference.downloadState` KV marker and, when auto-chaining, dispatch the
 * ingest phase. Parts are NEVER deleted here — they persist until a full ingest
 * succeeds so the manual "Ingest into search" path can re-run from disk.
 */
 export class DownloadDrugDataJob {
  static get queue() {
    return 'drug-download'
  }
  static get key() {
    return 'download-drug-data'
  }
  /** Deterministic jobId — only one download at a time, re-runnable. */
  static get jobId() {
    return 'drug-data-download'
  }
  // ─── Public API ────────────────────────────────────────────────────────────
  /**
   * Dispatch the initial download (pass 0). Idempotent on the deterministic
   * jobId. A finished/failed prior job under that id is cleared first so the
   * "Download FDA data" button can always restart (resume is handled at the
   * file level by doResumableDownload).
   *
   * @param autoChain - dispatch the ingest phase after the last part. Default true.
   * @param resourceMeta - install-state identity when the install came through a
   *   curated tier. Carried through the chain so the ingest job writes the
   *   `installed_resources` row on `ready`. Omit for a manual download (no row).
   */
  static async dispatch(autoChain = true, resourceMeta?: DrugDatasetResourceMeta) {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    const existing = await queue.getJob(this.jobId)
    if (existing) {
      const state = await existing.getState()
      if (state === 'active' || state === 'waiting' || state === 'delayed') {
        return { job: existing, created: false, message: 'Drug data download already running' }
      }
      try {
        await existing.remove()
      } catch {
        // Best-effort: fall through to add, which surfaces any genuine conflict.
      }
    }
    try {
      const job = await queue.add(
        this.key,
        { autoChain, resourceMeta } satisfies DownloadDrugDataJobParams,
        {
          jobId: this.jobId,
          attempts: 3,
          backoff: { type: 'exponential', delay: 5000 },
          removeOnComplete: { count: 5 },
          removeOnFail: { count: 5 },
        }
      )
      return { job, created: true, message: 'Drug data download dispatched' }
    } catch (error: unknown) {
      const msg = error instanceof Error ? error.message : String(error)
      if (msg.includes('job already exists')) {
        const stillThere = await queue.getJob(this.jobId)
        return { job: stillThere, created: false, message: 'Drug data download already running' }
      }
      throw error
    }
  }
  static async getJob(): Promise<Job | undefined> {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    return await queue.getJob(this.jobId)
  }
  // ─── Job handler ───────────────────────────────────────────────────────────
  async handle(job: Job) {
    const params = job.data as DownloadDrugDataJobParams
    const partIndex = params.partIndex ?? 0
    const autoChain = params.autoChain ?? true
    const startedAt = params.startedAt ?? Date.now()
    const resourceMeta = params.resourceMeta
    logger.info(`[DownloadDrugDataJob] Starting pass partIndex=${partIndex}`)
    // Pre-flight: storage drive writable.
    await this.verifyStorageAvailable(job)
    // Pass 0: fetch the manifest.
    let manifest: DrugLabelManifest
    if (partIndex === 0 || !params.manifest) {
      await job.updateData({
        ...job.data,
        phase: 'manifest',
        partIndex: 0,
        totalParts: 0,
        currentPartName: null,
        autoChain,
        startedAt,
      })
      await job.updateProgress(0)
      logger.info('[DownloadDrugDataJob] Fetching manifest from api.fda.gov/download.json')
      manifest = await this.fetchManifest()
      logger.info(
        `[DownloadDrugDataJob] Manifest: export_date=${manifest.export_date} ` +
          `total_records=${manifest.total_records} parts=${manifest.partitions.length}`
      )
    } else {
      manifest = params.manifest
    }
    const totalParts = params.totalParts ?? manifest.partitions.length
    if (partIndex >= totalParts) {
      logger.warn(
        `[DownloadDrugDataJob] partIndex ${partIndex} >= totalParts ${totalParts}, nothing to do`
      )
      return
    }
    const partition = manifest.partitions[partIndex]
    const zipPath = partZipPath(STORAGE_BASE, partition)
    const partName = partition.display_name || partition.file
    logger.info(
      `[DownloadDrugDataJob] Downloading part ${partIndex + 1}/${totalParts}: ${partName}`
    )
    await job.updateData({
      ...job.data,
      phase: 'downloading',
      partIndex,
      totalParts,
      currentPartName: partName,
      manifest,
      autoChain,
      startedAt,
      bytesDownloaded: 0,
    })
    await job.updateProgress(Math.floor((partIndex / totalParts) * 100))
    await mkdir(STORAGE_BASE, { recursive: true })
    // Aggregate-across-parts byte accounting for the Active Downloads card. The
    // drug job fans the manifest's N partitions into BullMQ continuations, but
    // the user sees ONE download — so progress is reported as bytes across the
    // whole set, not the current part. `priorPartsBytes` is the actual bytes of
    // the parts already on disk (from the running recordedParts list);
    // `manifestTotalBytes` is the sum of every partition's manifest size_mb.
    const priorRecorded =
      (params as { recordedParts?: DownloadStateMarker['parts'] }).recordedParts ?? []
    const priorPartsBytes = priorRecorded.reduce((acc, p) => acc + (p.bytes || 0), 0)
    const manifestTotalBytes = manifestBytesTotal(manifest)
    logger.info(`[DownloadDrugDataJob] ${partition.file} → ${zipPath}`)
    let partBytes = 0
    await doResumableDownload({
      url: partition.file,
      filepath: zipPath,
      timeout: 300_000, // 5-minute per-chunk timeout
      allowedMimeTypes: [], // skip MIME check — zip content-type varies across CDNs
      onProgress: (progress) => {
        partBytes = progress.downloadedBytes
        const downloadFraction = progress.downloadedBytes / (progress.totalBytes || 1)
        const pct = Math.floor(((partIndex + downloadFraction) / totalParts) * 100)
        const downloadedBytes = priorPartsBytes + progress.downloadedBytes
        // Emit the canonical {percent, downloadedBytes, totalBytes} object the
        // Active Downloads aggregator + the byte/speed readout expect (the old
        // bare-int progress couldn't carry bytes). parseProgress in
        // DownloadService still tolerates a bare int for back-compat, but only
        // the object surfaces a live byte/speed readout. Fire-and-forget; swallow
        // transient reject so it can't crash the worker.
        void job
          .updateProgress({
            percent: pct,
            downloadedBytes,
            totalBytes: manifestTotalBytes,
            lastProgressTime: Date.now(),
          })
          .catch(() => {})
        void job.updateData({ ...job.data, bytesDownloaded: progress.downloadedBytes }).catch(() => {})
      },
    })
    logger.info(`[DownloadDrugDataJob] Download complete: ${zipPath} (${partBytes} bytes)`)
    // Record this part for the download-state marker (written after the last one).
    const recordedParts =
      (params as { recordedParts?: DownloadStateMarker['parts'] }).recordedParts ?? []
    recordedParts.push({ index: partIndex, name: partName, path: zipPath, bytes: partBytes })
    const nextIndex = partIndex + 1
    if (nextIndex < totalParts) {
      // Continuation — NO jobId (let BullMQ auto-generate). The critical rule.
      const queueService = QueueService.getInstance()
      const queue = queueService.getQueue(DownloadDrugDataJob.queue)
      const continuationParams: DownloadDrugDataJobParams & {
        recordedParts: DownloadStateMarker['parts']
      } = {
        partIndex: nextIndex,
        manifest,
        totalParts,
        autoChain,
        startedAt,
        recordedParts,
        resourceMeta,
      }
      await queue.add(DownloadDrugDataJob.key, continuationParams, {
        attempts: 3,
        backoff: { type: 'exponential', delay: 5000 },
        removeOnComplete: { count: 5 },
        removeOnFail: { count: 5 },
      })
      logger.info(
        `[DownloadDrugDataJob] Dispatched continuation for part ${nextIndex + 1}/${totalParts}`
      )
    } else {
      // Last part — write the download-state marker. Parts stay on disk until a
      // full ingest succeeds; do NOT delete them here.
      await this.writeDownloadState(manifest, totalParts, recordedParts)
      await job.updateData({
        ...job.data,
        phase: 'downloaded',
        partIndex,
        totalParts,
        currentPartName: null,
      })
      await job.updateProgress(100)
      logger.info(
        `[DownloadDrugDataJob] All ${totalParts} parts downloaded. ` +
          `export_date=${manifest.export_date}`
      )
      if (autoChain) {
        const { IngestDrugDataJob } = await import('#jobs/ingest_drug_data_job')
        // Forward the install-state identity so the ingest job writes the
        // `installed_resources` row on `ready`. undefined for a manual download.
        await IngestDrugDataJob.dispatch(resourceMeta)
        logger.info('[DownloadDrugDataJob] Auto-chained ingest phase')
      }
    }
    return { partIndex, totalParts }
  }
  // ─── Private helpers ───────────────────────────────────────────────────────
  private async verifyStorageAvailable(job: Job): Promise<void> {
    try {
      await access(STORAGE_BASE, constants.W_OK)
    } catch {
      try {
        await mkdir(STORAGE_BASE, { recursive: true })
      } catch (mkdirErr) {
        await job.updateData({ ...job.data, phase: 'failed' })
        throw new Error(
          `Storage drive not available: cannot write to ${STORAGE_BASE} (${
            mkdirErr instanceof Error ? mkdirErr.message : String(mkdirErr)
          })`
        )
      }
    }
  }
  private async fetchManifest(): Promise<DrugLabelManifest> {
    return DownloadDrugDataJob.fetchManifest()
  }
  /**
   * Fetch + parse the openFDA download manifest. The SINGLE source of truth for
   * the openFDA manifest call (Maxim 4): the download job uses it on pass 0, and
   * the freshness check (DrugReferenceService.checkForUpdate, driven by
   * attemptAutoUpdate) reuses it so there is exactly one place that knows the URL and the
   * offline-error translation.
   */
  static async fetchManifest(): Promise<DrugLabelManifest> {
    let json: unknown
    try {
      const resp = await fetch(MANIFEST_URL)
      if (!resp.ok) {
        throw new Error(`HTTP ${resp.status} from ${MANIFEST_URL}`)
      }
      json = await resp.json()
    } catch (err) {
      const msg = err instanceof Error ? err.message : String(err)
      if (
        msg.includes('ENOTFOUND') ||
        msg.includes('ECONNREFUSED') ||
        msg.includes('ECONNRESET') ||
        msg.includes('fetch failed')
      ) {
        throw new Error(`No internet — connect to download FDA drug data. (${msg})`)
      }
      throw err
    }
    return parseDrugLabelManifest(json)
  }
  private async writeDownloadState(
    manifest: DrugLabelManifest,
    totalParts: number,
    parts: DownloadStateMarker['parts']
  ): Promise<void> {
    const KVStore = (await import('#models/kv_store')).default
    const marker: DownloadStateMarker = {
      export_date: manifest.export_date,
      totalParts,
      totalRecords: manifest.total_records,
      parts,
      completedAtMs: Date.now(),
    }
    await KVStore.setValue('drugReference.downloadState', JSON.stringify(marker))
  }
 }
--- a/admin/app/jobs/download_model_job.ts
+++ b/admin/app/jobs/download_model_job.ts
@ -1,8 +1,9 @@
-import { Job, UnrecoverableError } from 'bullmq'
+import { Job } from 'bullmq'
 import { QueueService } from '#services/queue_service'
 import { OpenWebUIService } from '#services/openwebui_service'
 import { createHash } from 'crypto'
 import logger from '@adonisjs/core/services/logger'
-import { OllamaService } from '#services/ollama_service'
+import { DockerService } from '#services/docker_service'
 export interface DownloadModelJobParams {
  modelName: string
@ -21,159 +22,86 @@ export class DownloadModelJob {
    return createHash('sha256').update(modelName).digest('hex').slice(0, 16)
  }
  /** In-memory registry of abort controllers for active model download jobs */
  static abortControllers: Map<string, AbortController> = new Map()
  /**
   * Redis key used to signal cancellation across processes. Uses a `model-cancel` prefix
   * so it cannot collide with content download cancel signals (`nomad:download:cancel:*`).
   */
  static cancelKey(jobId: string): string {
    return `nomad:download:model-cancel:${jobId}`
  }
  /** Signal cancellation via Redis so the worker process can pick it up on its next poll tick */
  static async signalCancel(jobId: string): Promise<void> {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    const client = await queue.client
    await client.set(this.cancelKey(jobId), '1', { EX: 300 }) // 5 min TTL
  }
  async handle(job: Job) {
    const { modelName } = job.data as DownloadModelJobParams
    logger.info(`[DownloadModelJob] Attempting to download model: ${modelName}`)
-    const ollamaService = new OllamaService()
+    // Check if OpenWebUI/Ollama services are ready
    const dockerService = new DockerService()
    const openWebUIService = new OpenWebUIService(dockerService)
    // Use getInstalledModels to check if the service is ready
    // Even if no models are installed, this should return an empty array if ready
-    const existingModels = await ollamaService.getModels()
+    const existingModels = await openWebUIService.getInstalledModels()
    if (!existingModels) {
      logger.warn(
-        `[DownloadModelJob] Ollama service not ready yet for model ${modelName}. Will retry...`
+        `[DownloadModelJob] OpenWebUI service not ready yet for model ${modelName}. Will retry...`
      )
-      throw new Error('Ollama service not ready yet')
+      throw new Error('OpenWebUI service not ready yet')
    }
    logger.info(
-      `[DownloadModelJob] Ollama service is ready. Initiating download for ${modelName}`
+      `[DownloadModelJob] OpenWebUI service is ready. Initiating download for ${modelName}`
    )
-    // Register abort controller for this job — used both by in-process cancels (same process
+    // Services are ready, initiate the download with progress tracking
-    // as the API server) and as the target of the Redis poll loop below.
+    const result = await openWebUIService._downloadModel(modelName, (progress) => {
-    const abortController = new AbortController()
+      // Update job progress in BullMQ
-    DownloadModelJob.abortControllers.set(job.id!, abortController)
+      const progressData = {
-
+        status: progress.status,
-    // Get Redis client for checking cancel signals from the API process
+        percent: progress.percent,
-    const queueService = QueueService.getInstance()
+        completed: progress.completed,
-    const cancelRedis = await queueService.getQueue(DownloadModelJob.queue).client
+        total: progress.total,
    // Track whether cancellation was explicitly requested by the user. Only user-initiated
    // cancels become UnrecoverableError — other failures (e.g., transient network errors)
    // should still benefit from BullMQ's retry logic.
    let userCancelled = false
    // Poll Redis for cancel signal every 2s — independent of progress events so cancellation
    // works even when the pull is mid-blob and not emitting progress updates.
    let cancelPollInterval: ReturnType<typeof setInterval> | null = setInterval(async () => {
      try {
        const val = await cancelRedis.get(DownloadModelJob.cancelKey(job.id!))
        if (val) {
          await cancelRedis.del(DownloadModelJob.cancelKey(job.id!))
          userCancelled = true
          abortController.abort('user-cancel')
        }
      } catch {
        // Redis errors are non-fatal; in-process AbortController covers same-process cancels
      }
    }, 2000)
-    try {
+      // Update the job progress (0-100 scale for BullMQ)
-      // Services are ready, initiate the download with progress tracking
+      if (progress.percent !== undefined) {
-      const result = await ollamaService.downloadModel(
+        job.updateProgress(progress.percent)
-        modelName,
+      }
        (progressPercent, bytes) => {
          if (progressPercent) {
            job.updateProgress(Math.floor(progressPercent)).catch((err) => {
              if (err?.code !== -1) throw err
            })
          }
-          // Store detailed progress in job data for clients to query
+      // Log progress with job context
-          job.updateData({
+      if (progress.percent !== undefined) {
-            ...job.data,
+        logger.info(
-            status: 'downloading',
+          `[DownloadModelJob] Model ${modelName}: ${progress.status} - ${progress.percent}% (${progress.completed}/${progress.total} bytes)`
            progress: progressPercent,
            downloadedBytes: bytes?.downloadedBytes,
            totalBytes: bytes?.totalBytes,
            progress_timestamp: new Date().toISOString(),
          }).catch((err) => {
            if (err?.code !== -1) throw err
          })
        },
        abortController.signal,
        job.id!
      )
      if (!result.success) {
        logger.error(
          `[DownloadModelJob] Failed to initiate download for model ${modelName}: ${result.message}`
        )
-        // User-initiated cancel — must be unrecoverable to avoid the 40-attempt retry storm.
+      } else {
-        // The downloadModel() catch block returns retryable: false for cancels, so this branch
+        logger.info(`[DownloadModelJob] Model ${modelName}: ${progress.status}`)
        // catches both Ollama version mismatches (existing) AND user cancels (new).
        if (result.retryable === false) {
          throw new UnrecoverableError(result.message)
        }
        throw new Error(`Failed to initiate download for model: ${result.message}`)
      }
-      logger.info(`[DownloadModelJob] Successfully completed download for model ${modelName}`)
+      // Store detailed progress in job data for clients to query
-      return {
+      job.updateData({
-        modelName,
+        ...job.data,
-        message: result.message,
+        progress: progressData,
-      }
+      })
-    } catch (error: any) {
+    })
-      // Belt-and-suspenders: if downloadModel didn't recognize the cancel (e.g., the abort
+
-      // fired after the response stream completed but before our code returned), the cancel
+    if (!result.success) {
-      // flag tells us this was a user action and should be unrecoverable.
+      logger.error(
-      if (userCancelled || abortController.signal.reason === 'user-cancel') {
+        `[DownloadModelJob] Failed to initiate download for model ${modelName}: ${result.message}`
-        if (!(error instanceof UnrecoverableError)) {
+      )
-          throw new UnrecoverableError(`Model download cancelled: ${error.message ?? error}`)
+      throw new Error(`Failed to initiate download for model: ${result.message}`)
-        }
+    }
-      }
+
-      throw error
+    logger.info(`[DownloadModelJob] Successfully completed download for model ${modelName}`)
-    } finally {
+    return {
-      if (cancelPollInterval !== null) {
+      modelName,
-        clearInterval(cancelPollInterval)
+      message: result.message,
        cancelPollInterval = null
      }
      DownloadModelJob.abortControllers.delete(job.id!)
    }
  }
  static async getByModelName(modelName: string): Promise<Job | undefined> {
-    const queueService = QueueService.getInstance()
+    const queueService = new QueueService()
    const queue = queueService.getQueue(this.queue)
    const jobId = this.getJobId(modelName)
    return await queue.getJob(jobId)
  }
  static async dispatch(params: DownloadModelJobParams) {
-    const queueService = QueueService.getInstance()
+    const queueService = new QueueService()
    const queue = queueService.getQueue(this.queue)
    const jobId = this.getJobId(params.modelName)
    // Clear any previous failed job so a fresh attempt can be dispatched
    const existing = await queue.getJob(jobId)
    if (existing) {
      const state = await existing.getState()
      if (state === 'failed') {
        await existing.remove()
      }
    }
    try {
      const job = await queue.add(this.key, params, {
        jobId,
@ -193,9 +121,9 @@ export class DownloadModelJob {
      }
    } catch (error) {
      if (error.message.includes('job already exists')) {
-        const active = await queue.getJob(jobId)
+        const existing = await queue.getJob(jobId)
        return {
-          job: active,
+          job: existing,
          created: false,
          message: `Job already exists for model ${params.modelName}`,
        }
--- a/admin/app/jobs/embed_file_job.ts
+++ b/admin/app/jobs/embed_file_job.ts
@ -1,540 +0,0 @@
 import { Job, UnrecoverableError } from 'bullmq'
 import { QueueService } from '#services/queue_service'
 import { EmbedJobWithProgress } from '../../types/rag.js'
 import { RagService } from '#services/rag_service'
 import { DockerService } from '#services/docker_service'
 import { OllamaService } from '#services/ollama_service'
 import KbIngestState from '#models/kb_ingest_state'
 import { createHash } from 'crypto'
 import logger from '@adonisjs/core/services/logger'
 import fs from 'node:fs/promises'
 import { ZIM_BATCH_SIZE } from '../../constants/zim_extraction.js'
 export interface EmbedFileJobParams {
  filePath: string
  fileName: string
  fileSize?: number
  // Batch processing for large ZIM files
  batchOffset?: number  // Current batch offset (for ZIM files)
  totalArticles?: number // Total articles in ZIM (for progress tracking)
  isFinalBatch?: boolean // Whether this is the last batch (prevents premature deletion)
  // Running total of chunks embedded across prior batches in this dispatch chain.
  // Carried forward so the final batch can persist an accurate `chunks_embedded`
  // count via KbIngestState.markIndexed (see #933 -- without this, only the last
  // batch's chunk count was stored while Qdrant held the full set).
  chunksSoFar?: number
  collection?: string
 }
 export class EmbedFileJob {
  static get queue() {
    return 'file-embeddings'
  }
  static get key() {
    return 'embed-file'
  }
  // Delay between continuation batches when embedding runs CPU-only. Gives the OS
  // scheduler a brief idle window so sshd / disk-collector / other services don't
  // starve during long multi-batch ZIM ingestions. Skipped entirely when the
  // embedding model is GPU-offloaded — see OllamaService.isEmbeddingGpuAccelerated().
  static readonly CPU_BATCH_DELAY_MS = 1000
  static getJobId(filePath: string): string {
    return createHash('sha256').update(filePath).digest('hex').slice(0, 16)
  }
  /** Calls job.updateProgress but silently ignores "Missing key" errors (code -1),
   *  which occur when the job has been removed from Redis (e.g. cancelled externally)
   *  between the time the await was issued and the Redis write completed. */
  private async safeUpdateProgress(job: Job, progress: number): Promise<void> {
    try {
      await job.updateProgress(progress)
    } catch (err: any) {
      if (err?.code !== -1) throw err
    }
  }
  async handle(job: Job) {
    const { filePath, fileName, batchOffset, totalArticles, collection } = job.data as EmbedFileJobParams
    // Only the direct KB-upload controller passes `collection` on dispatch; the other
    // six dispatch sites (download auto-index, scan/sync, re-embed, local ZIM upload,
    // replaced-file reconcile, and this job's own ZIM batch continuation) do not. Fall
    // back to whatever the file is already assigned to, so an assignment made *before*
    // the file was indexed still reaches the vectors. Resolving it here rather than at
    // each dispatch site keeps one source of truth and covers batch continuations too.
    const effectiveCollection =
      collection ?? (await KbIngestState.findBy('file_path', filePath))?.collection ?? undefined
    const isZimBatch = batchOffset !== undefined
    const batchInfo = isZimBatch ? ` (batch offset: ${batchOffset})` : ''
    logger.info(`[EmbedFileJob] Starting embedding process for: ${fileName}${batchInfo}`)
    const dockerService = new DockerService()
    const ollamaService = new OllamaService()
    const ragService = new RagService(dockerService, ollamaService)
    try {
      // Check if Ollama and Qdrant services are installed and ready
      // Use UnrecoverableError for "not installed" so BullMQ won't retry —
      // retrying 30x when the service doesn't exist just wastes Redis connections
      const ollamaUrl = await dockerService.getServiceURL('nomad_ollama')
      if (!ollamaUrl) {
        logger.warn('[EmbedFileJob] Ollama is not installed. Skipping embedding for: %s', fileName)
        throw new UnrecoverableError('Ollama service is not installed. Install AI Assistant to enable file embeddings.')
      }
      const existingModels = await ollamaService.getModels()
      if (!existingModels) {
        logger.warn('[EmbedFileJob] Ollama service not ready yet. Will retry...')
        throw new Error('Ollama service not ready yet')
      }
      const qdrantUrl = await dockerService.getServiceURL('nomad_qdrant')
      if (!qdrantUrl) {
        logger.warn('[EmbedFileJob] Qdrant is not installed. Skipping embedding for: %s', fileName)
        throw new UnrecoverableError('Qdrant service is not installed. Install AI Assistant to enable file embeddings.')
      }
      logger.info(`[EmbedFileJob] Services ready. Processing file: ${fileName}`)
      // Anchor initial progress to where we are in the overall file. For a
      // continuation batch midway through a multi-batch ZIM (e.g. offset 100k of
      // 600k), the hardcoded 5 used to make the gauge briefly flash 0→5→real,
      // which read as a backward jump. Fall back to 5 for single-batch files
      // where totalArticles isn't set.
      const initialPercent =
        totalArticles && totalArticles > 0
          ? Math.min(99, Math.round(((batchOffset || 0) / totalArticles) * 100))
          : 5
      await this.safeUpdateProgress(job, initialPercent)
      await job.updateData({
        ...job.data,
        status: 'processing',
        startedAt: job.data.startedAt || Date.now(),
      })
      logger.info(`[EmbedFileJob] Processing file: ${filePath}`)
      // Progress callback. For multi-batch ZIM ingestions, scale the service-reported
      // 0-100% (which is % through the current batch's chunks) into the overall-file
      // frame so the UI gauge climbs monotonically across the many continuation jobs
      // BullMQ creates per file. Without this, every new continuation jobId resets the
      // gauge to ~5% and the user sees ingestion progress "jumping around" between
      // each batch's local frame and the end-of-batch overall-file overwrite below.
      //
      // For single-batch files (uploaded PDFs, txts) totalArticles is undefined and
      // we fall back to the original 5-95% per-job range, which is what the UI expects
      // for a one-shot file with no continuations.
      const onProgress = async (percent: number) => {
        const useOverallFrame = totalArticles && totalArticles > 0
        if (useOverallFrame) {
          const articlesDone = (batchOffset || 0) + (percent / 100) * ZIM_BATCH_SIZE
          const overallPercent = Math.min(99, Math.round((articlesDone / totalArticles) * 100))
          await this.safeUpdateProgress(job, overallPercent)
        } else {
          await this.safeUpdateProgress(job, Math.min(95, Math.round(5 + percent * 0.9)))
        }
      }
      // Process and embed the file
      // Only allow deletion if explicitly marked as final batch
      const allowDeletion = job.data.isFinalBatch === true
      const result = await ragService.processAndEmbedFile(
        filePath,
        allowDeletion,
        batchOffset,
        onProgress,
        effectiveCollection
      )
      if (!result.success) {
        logger.error(`[EmbedFileJob] Failed to process file ${fileName}: ${result.message}`)
        throw new Error(result.message)
      }
      // For ZIM files with batching, check if more batches are needed
      if (result.hasMoreBatches) {
        const nextOffset = (batchOffset || 0) + (result.articlesProcessed || 0)
        logger.info(
          `[EmbedFileJob] Batch complete. Dispatching next batch at offset ${nextOffset}`
        )
        // Pace continuation batches when embedding is CPU-bound. Sustained 100% CPU
        // saturation across all cores during multi-batch ZIM ingestion can starve
        // other services (sshd has been seen to lose responsiveness hard enough to
        // require a power-cycle). When GPU-accelerated, embeddings stream through
        // the GPU and CPUs stay free — no pacing needed.
        const isGpuAccelerated = await ollamaService.isEmbeddingGpuAccelerated()
        if (!isGpuAccelerated) {
          logger.info(
            `[EmbedFileJob] Embedding is CPU-only — pacing ${EmbedFileJob.CPU_BATCH_DELAY_MS}ms before dispatching next batch`
          )
          await new Promise((resolve) => setTimeout(resolve, EmbedFileJob.CPU_BATCH_DELAY_MS))
        }
        // Bail before re-populating the queue if this job was cancelled mid-batch.
        // cancelAllJobs() obliterates the queue (including this active job), but a
        // worker already inside handle() would otherwise dispatch its continuation
        // afterwards and silently revive a cancelled ZIM ingestion. If our own job
        // key is gone, the cancel happened — skip the dispatch. Mirrors the
        // "tolerate external removal" handling in safeUpdateProgress above.
        const stillQueued = await QueueService.getInstance()
          .getQueue(EmbedFileJob.queue)
          .getJob(job.id!)
        if (!stillQueued) {
          logger.info(
            `[EmbedFileJob] Job ${fileName} was cancelled; skipping continuation dispatch`
          )
          return { success: false, cancelled: true, fileName, filePath }
        }
        // Dispatch next batch (not final yet). Carry forward the running
        // chunk count so the final batch can persist an accurate total (#933).
        const chunksSoFarNext = (job.data.chunksSoFar || 0) + (result.chunks || 0)
        await EmbedFileJob.dispatch({
          filePath,
          fileName,
          batchOffset: nextOffset,
          totalArticles: totalArticles || result.totalArticles,
          isFinalBatch: false, // Explicitly not final
          chunksSoFar: chunksSoFarNext,
          // Carry the collection across batches, otherwise only batch 1 of a ZIM
          // would be tagged and the rest would land uncategorized.
          ...(effectiveCollection ? { collection: effectiveCollection } : {}),
        })
        // Calculate progress based on articles processed.
        //
        // nextOffset counts entries passing our isArticleEntry() filter, but the
        // denominator (totalArticles = archive.articleCount) uses libzim's
        // narrower article definition. On ZIMs that pack one logical article as
        // several sub-pages (e.g. iFixit), nextOffset outruns articleCount and a
        // raw ratio overflows past 100%, which the UI pins at 99% for the entire
        // tail so the file looks stuck (#903). Grow the denominator once we pass
        // the reported count so the gauge keeps creeping forward monotonically,
        // and never report 100% before the genuinely-final batch (handled below).
        const progress = totalArticles
          ? Math.min(99, Math.round((nextOffset / Math.max(totalArticles, nextOffset + ZIM_BATCH_SIZE)) * 100))
          : 50
        await this.safeUpdateProgress(job, progress)
        await job.updateData({
          ...job.data,
          status: 'batch_completed',
          lastBatchAt: Date.now(),
          chunks: chunksSoFarNext,
        })
        return {
          success: true,
          fileName,
          filePath,
          chunks: result.chunks,
          hasMoreBatches: true,
          nextOffset,
          message: `Batch embedded ${result.chunks} chunks, next batch queued`,
        }
      }
      // Final batch or non-batched file - mark as complete.
      // chunksSoFar carries the accumulated count from prior dispatched batches
      // (each continuation passes it forward — see EmbedFileJobParams). For a
      // non-batched file it is undefined and we just count this single result.
      const totalChunks = (job.data.chunksSoFar || 0) + (result.chunks || 0)
      await this.safeUpdateProgress(job, 100)
      await job.updateData({
        ...job.data,
        status: 'completed',
        completedAt: Date.now(),
        chunks: totalChunks,
      })
      // Persist the post-job state so scanAndSyncStorage knows this file is done.
      // BullMQ's :completed retention (50 jobs) ages out, so the state row is
      // the only durable record of "this file finished embedding".
      try {
        await KbIngestState.markIndexed(filePath, totalChunks, effectiveCollection)
      } catch (stateErr) {
        logger.warn(
          `[EmbedFileJob] Failed to persist ingest state for ${fileName}: %s`,
          stateErr instanceof Error ? stateErr.message : String(stateErr)
        )
      }
      const batchMsg = isZimBatch ? ` (final batch, total chunks: ${totalChunks})` : ''
      logger.info(
        `[EmbedFileJob] Successfully embedded ${result.chunks} chunks from file: ${fileName}${batchMsg}`
      )
      return {
        success: true,
        fileName,
        filePath,
        chunks: result.chunks,
        message: `Successfully embedded ${result.chunks} chunks`,
      }
    } catch (error) {
      // A chunk that still exceeds the model's context after OllamaService's truncate-and-retry is
      // permanently oversized for this install (e.g. a model whose context is smaller than our safe
      // cap). Re-embedding the whole file 30x re-processes everything and can never succeed — that is
      // the "endless queue loop" / "api/embed for weeks" (#881/#944/#959). Mark it unrecoverable so
      // BullMQ stops after one pass instead of storming.
      let normalizedError = error
      if (!(error instanceof UnrecoverableError) && OllamaService.isContextLengthError(error)) {
        logger.warn(
          `[EmbedFileJob] Context-length overflow persisted for ${fileName} after truncation; not retrying.`
        )
        normalizedError = new UnrecoverableError(
          error instanceof Error ? error.message : 'Embedding input exceeds the model context length'
        )
      }
      logger.error(`[EmbedFileJob] Error embedding file ${fileName}:`, normalizedError)
      await job.updateData({
        ...job.data,
        status: 'failed',
        failedAt: Date.now(),
        error: normalizedError instanceof Error ? normalizedError.message : 'Unknown error',
      })
      // Only persist `failed` for unrecoverable errors. Retryable errors get
      // automatic BullMQ retries (30 attempts); marking state failed on every
      // transient blip would suppress the retry-driven recovery path.
      if (normalizedError instanceof UnrecoverableError) {
        try {
          await KbIngestState.markFailed(
            filePath,
            normalizedError instanceof Error ? normalizedError.message : 'Unknown error'
          )
        } catch (stateErr) {
          logger.warn(
            `[EmbedFileJob] Failed to persist failed state for ${fileName}: %s`,
            stateErr instanceof Error ? stateErr.message : String(stateErr)
          )
        }
      }
      throw normalizedError
    }
  }
  static async listActiveJobs(): Promise<EmbedJobWithProgress[]> {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    const jobs = await queue.getJobs(['waiting', 'active', 'delayed'])
    return jobs.map((job) => {
      const data = job.data as EmbedFileJobParams & {
        status?: string
        lastBatchAt?: number
        startedAt?: number
        chunks?: number
      }
      return {
        jobId: job.id!.toString(),
        fileName: data.fileName,
        filePath: data.filePath,
        progress: typeof job.progress === 'number' ? job.progress : 0,
        status: data.status ?? 'waiting',
        lastBatchAt: data.lastBatchAt,
        startedAt: data.startedAt,
        chunks: data.chunks,
      }
    })
  }
  static async getByFilePath(filePath: string): Promise<Job | undefined> {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    const jobId = this.getJobId(filePath)
    return await queue.getJob(jobId)
  }
  static async dispatch(params: EmbedFileJobParams, options?: { force?: boolean }) {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    // Continuation batches (batchOffset > 0) must NOT reuse the deterministic
    // per-file jobId. Two BullMQ dedupe paths would otherwise silently swallow them:
    //   1) The parent batch's handle() calls dispatch() before returning, so the
    //      parent job is still `active` and locked — queue.add() with the same
    //      jobId returns the locked parent rather than enqueueing the new batch.
    //   2) After the parent completes, its entry stays in `completed` (held by
    //      `removeOnComplete: { count: 50 }`), still tripping jobId dedupe.
    // Letting BullMQ auto-generate a unique jobId for continuation batches stacks
    // them as independent queue entries that each process via handle().
    // Initial dispatches keep the deterministic jobId so re-triggering an install
    // (UI re-click, sync rescan, etc.) is still idempotent.
    // `force` skips the deterministic jobId for bulk callers (reembedAll /
    // resetAndRebuild) where historical entries in :completed would otherwise
    // silently swallow the new dispatch.
    const isContinuation = !!(params.batchOffset && params.batchOffset > 0)
    const force = !!options?.force
    const initialJobId = this.getJobId(params.filePath)
    const jobOptions: Parameters<typeof queue.add>[2] = {
      attempts: 30,
      backoff: {
        type: 'fixed',
        delay: 60000, // Check every 60 seconds for service readiness
      },
      removeOnComplete: { count: 50 }, // Keep last 50 completed jobs for history
      removeOnFail: { count: 20 }, // Keep last 20 failed jobs for debugging
    }
    if (!isContinuation && !force) {
      jobOptions.jobId = initialJobId
    }
    try {
      const job = await queue.add(this.key, params, jobOptions)
      const label = isContinuation
        ? ` (continuation @ offset ${params.batchOffset})`
        : force
          ? ' (forced re-dispatch)'
          : ''
      logger.info(
        `[EmbedFileJob] Dispatched embedding job for file: ${params.fileName}${label}`
      )
      return {
        job,
        created: true,
        jobId: job.id ?? initialJobId,
        message: `File queued for embedding: ${params.fileName}`,
      }
    } catch (error) {
      if (
        !isContinuation &&
        !force &&
        error.message &&
        error.message.includes('job already exists')
      ) {
        const existing = await queue.getJob(initialJobId)
        logger.info(`[EmbedFileJob] Job already exists for file: ${params.fileName}`)
        return {
          job: existing,
          created: false,
          jobId: initialJobId,
          message: `Embedding job already exists for: ${params.fileName}`,
        }
      }
      throw error
    }
  }
  static async listFailedJobs(): Promise<EmbedJobWithProgress[]> {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    // Jobs that have failed at least once are in 'delayed' (retrying) or terminal 'failed' state.
    // We identify them by job.data.status === 'failed' set in the catch block of handle().
    const jobs = await queue.getJobs(['waiting', 'delayed', 'failed'])
    return jobs
      .filter((job) => (job.data as any).status === 'failed')
      .map((job) => ({
        jobId: job.id!.toString(),
        fileName: (job.data as EmbedFileJobParams).fileName,
        filePath: (job.data as EmbedFileJobParams).filePath,
        progress: 0,
        status: 'failed',
        error: (job.data as any).error,
      }))
  }
  static async cleanupFailedJobs(): Promise<{ cleaned: number; filesDeleted: number }> {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    const allJobs = await queue.getJobs(['waiting', 'delayed', 'failed'])
    const failedJobs = allJobs.filter((job) => (job.data as any).status === 'failed')
    let cleaned = 0
    let filesDeleted = 0
    for (const job of failedJobs) {
      const filePath = (job.data as EmbedFileJobParams).filePath
      if (filePath && filePath.includes(RagService.UPLOADS_STORAGE_PATH)) {
        try {
          await fs.unlink(filePath)
          filesDeleted++
        } catch {
          // File may already be deleted — that's fine
        }
      }
      await job.remove()
      cleaned++
    }
    logger.info(`[EmbedFileJob] Cleaned up ${cleaned} failed jobs, deleted ${filesDeleted} files`)
    return { cleaned, filesDeleted }
  }
  /** Unconditionally clear every embedding job regardless of state.
   *
   *  cleanupFailedJobs only removes jobs explicitly tagged status === 'failed',
   *  which leaves stuck jobs (waiting / active / delayed / paused that never
   *  reached 'failed') unreachable from the UI — the operator's only recourse was
   *  flushing Redis by hand. This wipes the whole queue, including a locked active
   *  job, via obliterate({ force: true }) (plain obliterate/job.remove throw on a
   *  locked job). It touches only Redis, so it is safe while Qdrant/Ollama are
   *  offline — which is exactly when jobs pile up and wedge. */
  static async cancelAllJobs(): Promise<{ cancelled: number; filesDeleted: number }> {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    const jobs = await queue.getJobs(['waiting', 'active', 'delayed', 'paused', 'failed'])
    let filesDeleted = 0
    for (const job of jobs) {
      const filePath = (job.data as EmbedFileJobParams).filePath
      // Same guard as cleanupFailedJobs: only delete user uploads, never ZIM
      // library files or Nomad docs that live outside the uploads path.
      if (filePath && filePath.includes(RagService.UPLOADS_STORAGE_PATH)) {
        try {
          await fs.unlink(filePath)
          filesDeleted++
        } catch {
          // File may already be deleted — that's fine
        }
      }
    }
    const cancelled = jobs.length
    // force: true removes the locked/active job too. An in-flight worker may keep
    // running its current batch in memory; the self-exists guard in handle()
    // prevents it from dispatching a continuation back into the cleared queue.
    await queue.obliterate({ force: true })
    logger.info(`[EmbedFileJob] Cancelled ${cancelled} jobs, deleted ${filesDeleted} files`)
    return { cancelled, filesDeleted }
  }
  static async getStatus(filePath: string): Promise<{
    exists: boolean
    status?: string
    progress?: number
    chunks?: number
    error?: string
  }> {
    const job = await this.getByFilePath(filePath)
    if (!job) {
      return { exists: false }
    }
    const state = await job.getState()
    const data = job.data
    return {
      exists: true,
      status: data.status || state,
      progress: typeof job.progress === 'number' ? job.progress : undefined,
      chunks: data.chunks,
      error: data.error,
    }
  }
 }
--- a/admin/app/jobs/ingest_drug_data_job.ts
+++ b/admin/app/jobs/ingest_drug_data_job.ts
@ -1,837 +0,0 @@
 import { Job } from 'bullmq'
 import { promises as fsPromises } from 'node:fs'
 import { access, constants } from 'node:fs/promises'
 import { Writable } from 'node:stream'
 import logger from '@adonisjs/core/services/logger'
 import { QueueService } from '#services/queue_service'
 import { mapDrugLabelRecord, parseDownloadState, partZipPath } from '../../util/drug_labels.js'
 import { STORAGE_BASE } from '#jobs/download_drug_data_job'
 import type {
  IngestDrugDataJobParams,
  DrugLabelManifest,
  DrugLabelPartition,
  DrugDatasetResourceMeta,
 } from '../../types/drug_reference.js'
 const BATCH_SIZE = 500
 /**
 * Dedupe a batch by set_id, CASE-INSENSITIVELY (last occurrence wins). MySQL's
 * uniq_drug_labels_set_id index uses a case-insensitive collation, so two ids
 * differing only in case are ONE row to the database — the dedupe key must agree
 * with the database's notion of equality or a batch can still collide with
 * itself.
 */
 function dedupeBySetId(
  rows: ReturnType<typeof mapDrugLabelRecord>[]
 ): NonNullable<ReturnType<typeof mapDrugLabelRecord>>[] {
  const bySetId = new Map<string, NonNullable<ReturnType<typeof mapDrugLabelRecord>>>()
  for (const r of rows) {
    if (r) bySetId.set(r.set_id.toLowerCase(), r)
  }
  return [...bySetId.values()]
 }
 /**
 * Upsert one batch with MySQL-native `INSERT … ON DUPLICATE KEY UPDATE`
 * (knex onConflict().merge()) instead of Lucid's updateOrCreateMany.
 *
 * WHY: updateOrCreateMany SELECTs existing rows and matches them to incoming
 * rows IN JAVASCRIPT — a case-sensitive string compare. The DB's unique key on
 * set_id is case-INsensitive, so when openFDA ships the same set_id with
 * different casing across parts (seen live: part 5's '93A0696B-…' colliding
 * with an earlier part's variant), Lucid misses the match, INSERTs, and the
 * unique key rejects it — aborting the run. A native upsert makes the unique
 * key itself the arbiter: same-key rows update, new rows insert, intra-batch
 * duplicates take the update path. No JS equality anywhere.
 *
 * `ingested_at` is stamped explicitly (raw knex bypasses Lucid's autoCreate);
 * merge() refreshes every inserted column on conflict, preserving the previous
 * "re-ingest updates the row + timestamp" behavior.
 */
 async function upsertDrugLabelBatch(
  rows: NonNullable<ReturnType<typeof mapDrugLabelRecord>>[]
 ): Promise<number> {
  if (rows.length === 0) return 0
  const { default: db } = await import('@adonisjs/lucid/services/db')
  const now = new Date()
  const withTs = rows.map((r) => ({ ...r, ingested_at: now }))
  await db.knexQuery().table('drug_labels').insert(withTs).onConflict('set_id').merge()
  return rows.length
 }
 // A single 500-row updateOrCreateMany should finish in seconds even on the
 // FULLTEXT-indexed drug_labels table. If one batch exceeds this, the DB is
 // locked/overloaded — reject loudly so the ingest FAILS VISIBLY (and retries via
 // BullMQ) instead of hanging forever with the worker's lock still renewing,
 // which reads in the UI as a frozen "part 1 of 13, 0 rows". 0.2.714 removed the
 // un-awaited-update worker crash; this removes the remaining silent hang.
 const UPSERT_TIMEOUT_MS = 120_000
 // No record parsed within this window (the zip-open + JSON-parse stage, BEFORE the
 // first batch) means the part is almost certainly corrupt/truncated: yauzl's
 // inflate stream hangs with no 'end' and no 'error', so the ingest sat on
 // "part 1, 0 rows" forever with the worker's lock still renewing. Fail loud
 // instead. Once records flow, the per-batch upsert timeout governs.
 const STALL_MS = 90_000
 /**
 * updateOrCreateMany with a hard timeout. mysql2 has no default query timeout, so
 * a stuck DB call (metadata lock, exhausted connection pool, FULLTEXT stall)
 * would otherwise never settle and the part-stream would back-pressure to a halt.
 * Promise.race turns that into a rejection the caller can log + fail + retry.
 */
 async function withUpsertTimeout<T>(
  work: Promise<T>,
  rowCount: number,
  timeoutMs: number
 ): Promise<T> {
  let timer: ReturnType<typeof setTimeout> | undefined
  const timeout = new Promise<never>((_, reject) => {
    timer = setTimeout(
      () =>
        reject(
          new Error(
            `updateOrCreateMany timed out after ${timeoutMs}ms (batch of ${rowCount} rows) — ` +
              'the database may be locked or overloaded'
          )
        ),
      timeoutMs
    )
  })
  try {
    return await Promise.race([work, timeout])
  } finally {
    if (timer) clearTimeout(timer)
  }
 }
 // ─── Local type aliases for yauzl callbacks ───────────────────────────────────
 // yauzl/stream-json are loaded via dynamic import() with @ts-ignore (see
 // streamIngestPart). The real @types ship in devDependencies and resolve on the
 // target machine; loading them lazily keeps the pure util/ helpers importable in
 // tests without the streaming deps. These local interfaces give the callback
 // parameters explicit types without importing yauzl's own types (which aren't
 // resolvable in the inertia tsconfig context).
 import type { Readable } from 'node:stream'
 interface YauzlEntry { fileName: string }
 interface YauzlZipFile {
  readEntry(): void
  openReadStream(entry: YauzlEntry, cb: (err: Error | null, stream: Readable | null) => void): void
  on(event: 'entry', listener: (entry: YauzlEntry) => void): this
  on(event: 'end', listener: () => void): this
  on(event: 'error', listener: (err: Error) => void): this
 }
 /**
 * Phase B — Ingest (parse/DB-only failure domain, ZERO network I/O).
 *
 * Each pass reads ONE on-disk part and streams it into drug_labels via the
 * memory-safe streamIngestPart pipeline (yauzl → stream-json → batched
 * updateOrCreateMany). The part list comes from the manifest in job data OR, for
 * a manual "Ingest into search" run with no manifest, is rebuilt from the
 * `drugReference.downloadState` KV marker. A missing on-disk part fails loudly
 * ("run Download first") rather than silently under-ingesting. Continuations use
 * queue.add with NO jobId. After the LAST part: write the final KV status, then
 * delete the downloaded parts and clear the download-state marker (the per-part
 * unlink that used to run during download moves here — parts persist until a
 * full ingest succeeds).
 */
 export class IngestDrugDataJob {
  static get queue() {
    return 'drug-ingest'
  }
  static get key() {
    return 'ingest-drug-data'
  }
  /** Deterministic jobId — only one ingest at a time, re-runnable. */
  static get jobId() {
    return 'drug-labels-ingest'
  }
  // ─── Public API ────────────────────────────────────────────────────────────
  /**
   * Dispatch the initial ingest (pass 0). Idempotent on the deterministic jobId.
   * A finished/failed prior job under that id is cleared first so a re-ingest can
   * always restart (upserts are idempotent on set_id).
   *
   * @param resourceMeta - install-state identity, present only when the install
   *   came through a curated tier. Carried through the chain so the final pass
   *   writes the `installed_resources` row on `ready`. Absent on a manual ingest.
   */
  static async dispatch(resourceMeta?: DrugDatasetResourceMeta) {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    const existing = await queue.getJob(this.jobId)
    if (existing) {
      const state = await existing.getState()
      if (state === 'active' || state === 'waiting' || state === 'delayed') {
        return { job: existing, created: false, message: 'Drug label ingest already running' }
      }
      try {
        await existing.remove()
      } catch {
        // Best-effort: fall through to add.
      }
    }
    try {
      const job = await queue.add(
        this.key,
        { resourceMeta } satisfies IngestDrugDataJobParams,
        {
          jobId: this.jobId,
          attempts: 3,
          backoff: { type: 'exponential', delay: 5000 },
          removeOnComplete: { count: 5 },
          removeOnFail: { count: 5 },
        }
      )
      return { job, created: true, message: 'Drug label ingest dispatched' }
    } catch (error: unknown) {
      const msg = error instanceof Error ? error.message : String(error)
      if (msg.includes('job already exists')) {
        const stillThere = await queue.getJob(this.jobId)
        return { job: stillThere, created: false, message: 'Drug label ingest already running' }
      }
      throw error
    }
  }
  static async getJob(): Promise<Job | undefined> {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    return await queue.getJob(this.jobId)
  }
  // ─── Job handler ───────────────────────────────────────────────────────────
  async handle(job: Job) {
    const params = job.data as IngestDrugDataJobParams
    const partIndex = params.partIndex ?? 0
    const runningIngested = params.recordsIngested ?? 0
    const runningSkipped = params.recordsSkipped ?? 0
    const startedAt = params.startedAt ?? Date.now()
    const resourceMeta = params.resourceMeta
    // Progress baseline (pass 0 only): the table's row count BEFORE this run.
    // Without it, a re-ingest into a populated table shows ~100% from second
    // zero because the live row count dominates the shown counter.
    let startRowCount = params.startRowCount
    if (startRowCount === undefined) {
      try {
        const { default: db } = await import('@adonisjs/lucid/services/db')
        const result = await db.rawQuery('SELECT COUNT(*) AS cnt FROM drug_labels')
        const rows = result[0] as Array<{ cnt: number | string }>
        startRowCount = Number(rows[0]?.cnt ?? 0)
      } catch {
        startRowCount = 0
      }
    }
    logger.info(`[IngestDrugDataJob] Starting pass partIndex=${partIndex}`)
    // Resolve the part list: manifest in job data, else the KV download marker.
    const { manifest, exportDate } = await this.resolvePartSource(params)
    const totalParts = params.totalParts ?? manifest.partitions.length
    if (partIndex >= totalParts) {
      logger.warn(
        `[IngestDrugDataJob] partIndex ${partIndex} >= totalParts ${totalParts}, nothing to do`
      )
      return
    }
    const partition = manifest.partitions[partIndex]
    const zipPath = partZipPath(STORAGE_BASE, partition)
    const partName = partition.display_name || partition.file
    // Guard: the part MUST already be on disk. No re-download here — fail loud so
    // a missing part can't silently produce a "ready" status with fewer rows.
    try {
      await access(zipPath, constants.R_OK)
    } catch {
      await job.updateData({ ...job.data, phase: 'failed' })
      throw new Error(
        `Part ${partIndex + 1}/${totalParts} not downloaded (${zipPath}). ` +
          'Run Download FDA data first.'
      )
    }
    logger.info(
      `[IngestDrugDataJob] Ingesting part ${partIndex + 1}/${totalParts}: ${partName}`
    )
    await job.updateData({
      ...job.data,
      phase: 'ingesting',
      partIndex,
      totalParts,
      currentPartName: partName,
      recordsIngested: runningIngested,
      recordsSkipped: runningSkipped,
      manifest,
      startedAt,
      startRowCount,
    })
    await job.updateProgress(Math.floor((partIndex / totalParts) * 100))
    const { recordsIngested: partIngested, recordsSkipped: partSkipped } =
      await this.streamIngestPart(
        job,
        zipPath,
        partIndex,
        totalParts,
        runningIngested,
        runningSkipped
      )
    const totalIngested = runningIngested + partIngested
    const totalSkipped = runningSkipped + partSkipped
    logger.info(
      `[IngestDrugDataJob] Part ${partIndex + 1} done: ` +
        `ingested=${partIngested} skipped=${partSkipped} running_total=${totalIngested}`
    )
    const nextIndex = partIndex + 1
    if (nextIndex < totalParts) {
      // Continuation — NO jobId. The critical rule.
      const queueService = QueueService.getInstance()
      const queue = queueService.getQueue(IngestDrugDataJob.queue)
      const continuationParams: IngestDrugDataJobParams = {
        partIndex: nextIndex,
        manifest,
        totalParts,
        recordsIngested: totalIngested,
        recordsSkipped: totalSkipped,
        startedAt,
        startRowCount,
        resourceMeta,
      }
      await queue.add(IngestDrugDataJob.key, continuationParams, {
        attempts: 3,
        backoff: { type: 'exponential', delay: 5000 },
        removeOnComplete: { count: 5 },
        removeOnFail: { count: 5 },
      })
      logger.info(`[IngestDrugDataJob] Dispatched continuation for part ${nextIndex + 1}/${totalParts}`)
      await job.updateData({
        ...job.data,
        phase: 'ingesting',
        partIndex,
        totalParts,
        recordsIngested: totalIngested,
        recordsSkipped: totalSkipped,
      })
    } else {
      // Final part — write KV status, mark ready, THEN reclaim disk.
      await this.writeFinalStatus(exportDate, resourceMeta)
      await job.updateData({
        ...job.data,
        phase: 'ready',
        partIndex,
        totalParts,
        currentPartName: null,
        recordsIngested: totalIngested,
        recordsSkipped: totalSkipped,
      })
      await job.updateProgress(100)
      logger.info(
        `[IngestDrugDataJob] Ingest complete. ` +
          `total_ingested=${totalIngested} total_skipped=${totalSkipped} ` +
          `export_date=${exportDate}`
      )
      // Reclaim disk only after a FULL ingest succeeds.
      await this.deleteDownloadedParts(manifest, totalParts)
    }
    return { partIndex, totalIngested, totalSkipped }
  }
  // ─── Private helpers ───────────────────────────────────────────────────────
  /**
   * Resolve the ordered partition list + export_date for this ingest run.
   *
   * Prefers the manifest carried in job data (auto-chained from the download job,
   * or a continuation pass). Falls back to the KV download-state marker so a
   * manual "Ingest into search" with no manifest still works — the marker stores
   * each part's on-disk path and manifest index, which is enough to drive
   * streamIngestPart without re-fetching the manifest from the network. Fails
   * loudly if neither source is present.
   */
  private async resolvePartSource(
    params: IngestDrugDataJobParams
  ): Promise<{ manifest: DrugLabelManifest; exportDate: string }> {
    if (params.manifest) {
      return { manifest: params.manifest, exportDate: params.manifest.export_date }
    }
    const KVStore = (await import('#models/kv_store')).default
    const marker = parseDownloadState(await KVStore.getValue('drugReference.downloadState'))
    if (!marker) {
      throw new Error('Nothing downloaded — run Download FDA data first.')
    }
    // Rebuild a manifest-shaped partition list from the marker. partZipPath uses
    // path.basename(partition.file), so feeding the recorded path as `file`
    // resolves back to the same on-disk path.
    const ordered = [...marker.parts].sort((a, b) => a.index - b.index)
    const partitions: DrugLabelPartition[] = ordered.map((p) => ({
      display_name: p.name,
      file: p.path,
      size_mb: '0',
      records: 0,
    }))
    const manifest: DrugLabelManifest = {
      export_date: marker.export_date,
      // The marker persists the real manifest total (~259k) so a rebuilt
      // manifest carries the same label-count denominator the auto-chained run
      // would have had — keeping the "X of ~259k" counter, the records-based
      // progress %, and the ETA alive on the manual-ingest path. Pre-totalRecords
      // markers parse back as 0; the service treats 0 as unknown and falls back.
      total_records: marker.totalRecords,
      partitions,
    }
    return { manifest, exportDate: marker.export_date }
  }
  /**
   * Stream-unzip the part, stream-parse the JSON, batch-upsert into drug_labels.
   *
   * Memory-safe: never loads the full JSON into memory.
   * Pipeline: yauzl entry read-stream → stream-json Pick+StreamArray → Writable batching.
   * Back-pressure: the Writable's `write()` method calls `callback()` only after
   * the DB upsert resolves, so Node's stream machinery naturally pauses the upstream
   * pipe chain when BATCH_SIZE is reached and an upsert is in flight.
   */
  private async streamIngestPart(
    job: Job,
    zipPath: string,
    partIndex: number,
    totalParts: number,
    runningIngested: number,
    runningSkipped: number
  ): Promise<{ recordsIngested: number; recordsSkipped: number }> {
    // Dynamic imports for the streaming deps (yauzl, stream-json). @ts-ignore
    // covers local dev where node_modules hasn't been refreshed with the new
    // deps yet; it is a no-op once the real @types are installed on the target
    // machine, and it also covers stream-json's deep `.js` subpaths that the
    // @types package doesn't map. Importing here (not at module top) keeps the
    // pure util/ helpers loadable in tests without these deps.
    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
    // @ts-ignore — yauzl resolved at runtime from dependencies
    const yauzl = await import('yauzl')
    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
    // @ts-ignore — stream-json resolved at runtime from dependencies
    const createParser = (await import('stream-json')).default.parser
    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
    // @ts-ignore — stream-json deep subpath resolved at runtime
    const createPick = (await import('stream-json/filters/Pick.js')).default.pick
    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
    // @ts-ignore — stream-json deep subpath resolved at runtime
    const createStreamArray = (await import('stream-json/streamers/StreamArray.js')).default.streamArray
    // Defensive: stream-json 1.9.1 (CJS) exposes its factories only as
    // `.default.parser` / `.default.pick` / `.default.streamArray` under ESM
    // dynamic import — there is NO `parser`/`pick`/`streamArray` named export, so
    // destructuring those silently yielded `undefined`. Calling an undefined
    // "factory" then threw inside the yauzl openReadStream callback (not a
    // promise), the process-level backstop swallowed the uncaughtException, the
    // streamIngestPart promise never settled, and the 90s watchdog fired in a
    // retry loop with zero records. THIS was the real ingest hang. Validate the
    // imports here so any future interop slip fails loud in the async body.
    if (
      typeof createParser !== 'function' ||
      typeof createPick !== 'function' ||
      typeof createStreamArray !== 'function'
    ) {
      throw new Error(
        'stream-json factory imports did not resolve to functions ' +
          `(parser=${typeof createParser}, pick=${typeof createPick}, streamArray=${typeof createStreamArray})`
      )
    }
    let recordsIngested = 0
    let recordsSkipped = 0
    let batch: ReturnType<typeof mapDrugLabelRecord>[] = []
    let batchNum = 0
    let firstRecordSeen = false
    // Cast to `any` so callback parameters get explicit annotations below rather
    // than triggering implicit-any in tsconfigs that don't find the yauzl types.
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    const yauzlOpen = (yauzl as any).open as (
      path: string,
      opts: { lazyEntries: boolean; autoClose: boolean },
      cb: (err: Error | null, zipFile: YauzlZipFile | null) => void
    ) => void
    return new Promise<{ recordsIngested: number; recordsSkipped: number }>((resolveRaw, rejectRaw) => {
      // Stall watchdog. The per-batch upsert has a timeout, but the stage BEFORE
      // the first record (zip-open + JSON parse) had none — a corrupt/truncated
      // part makes yauzl's inflate stream hang with no 'end' and no 'error', so
      // the ingest froze on "part 1, 0 rows". If no record is parsed within
      // STALL_MS, fail the part loudly (the reason lands in the job's failedReason
      // → the status panel) and destroy the stuck stream. resolve/reject are
      // wrapped (shadowing the raw executor params) so every existing handler
      // routes through the once-guard + watchdog cleanup.
      let settled = false
      let activeReadStream: Readable | null = null
      let watchdog: ReturnType<typeof setInterval> | undefined
      const watchStart = Date.now()
      const settle = () => {
        settled = true
        if (watchdog) clearInterval(watchdog)
        try {
          activeReadStream?.destroy()
        } catch {
          // best-effort stream teardown
        }
      }
      const resolve = (v: { recordsIngested: number; recordsSkipped: number }) => {
        if (settled) return
        settle()
        resolveRaw(v)
      }
      const reject = (e: Error) => {
        if (settled) return
        settle()
        rejectRaw(e)
      }
      watchdog = setInterval(() => {
        if (!settled && !firstRecordSeen && Date.now() - watchStart > STALL_MS) {
          reject(
            new Error(
              `Ingest stalled: no records parsed from part ${partIndex + 1}/${totalParts} ` +
                `within ${Math.round(STALL_MS / 1000)}s. The downloaded part is likely corrupt ` +
                'or truncated — re-download FDA data, then ingest again.'
            )
          )
        }
      }, 10_000)
      yauzlOpen(zipPath, { lazyEntries: true, autoClose: true }, (err, zipFile) => {
        if (err || !zipFile) {
          reject(err ?? new Error(`Failed to open zip: ${zipPath}`))
          return
        }
        zipFile.on('error', reject)
        zipFile.readEntry()
        zipFile.on('entry', (entry) => {
          // Skip directory entries
          if (/\/$/.test(entry.fileName)) {
            zipFile.readEntry()
            return
          }
          // Open the single JSON entry as a read stream — never buffer it
          zipFile.openReadStream(entry, (streamErr, readStream) => {
            if (streamErr || !readStream) {
              reject(streamErr ?? new Error(`Could not open zip entry ${entry.fileName}`))
              return
            }
            logger.info(
              `[IngestDrugDataJob] part ${partIndex + 1}/${totalParts}: reading zip entry ${entry.fileName}`
            )
            activeReadStream = readStream
            // The JSON envelope is { meta, results: [...] }
            // Pick the `results` path → StreamArray emits one record at a time
            const jsonParser = createParser({ jsonStreaming: false })
            const pick = createPick({ filter: 'results' })
            const streamArray = createStreamArray()
            // Writable that accumulates batches and flushes with back-pressure.
            // The callback is called only after the async upsert completes, which
            // naturally applies back-pressure via the pipe chain.
            const batchWriter = new Writable({
              objectMode: true,
              write(chunk: { value: unknown }, _encoding: BufferEncoding, callback: (err?: Error | null) => void) {
                const record = chunk.value
                if (!firstRecordSeen) {
                  firstRecordSeen = true
                  logger.info(
                    `[IngestDrugDataJob] part ${partIndex + 1}/${totalParts}: first record received from parser`
                  )
                }
                // Map the record
                let row: ReturnType<typeof mapDrugLabelRecord>
                try {
                  row = mapDrugLabelRecord(record as Parameters<typeof mapDrugLabelRecord>[0])
                } catch (mapErr) {
                  logger.warn(
                    `[IngestDrugDataJob] mapDrugLabelRecord threw: ${mapErr instanceof Error ? mapErr.message : String(mapErr)}`
                  )
                  recordsSkipped++
                  callback()
                  return
                }
                if (!row) {
                  recordsSkipped++
                  callback()
                  return
                }
                batch.push(row)
                if (batch.length < BATCH_SIZE) {
                  // Not full yet — don't block the stream
                  callback()
                  return
                }
                // Batch full — flush and hold the callback until the upsert
                // resolves (this is the back-pressure point).
                const currentBatch = dedupeBySetId(batch)
                batch = []
                const myBatch = ++batchNum
                const t0 = Date.now()
                logger.info(
                  `[IngestDrugDataJob] part ${partIndex + 1}/${totalParts}: upserting batch ${myBatch} (${currentBatch.length} rows)…`
                )
                withUpsertTimeout(
                  upsertDrugLabelBatch(currentBatch),
                  currentBatch.length,
                  UPSERT_TIMEOUT_MS
                )
                  .then((rowCount) => {
                    recordsIngested += rowCount
                    logger.info(
                      `[IngestDrugDataJob] part ${partIndex + 1}/${totalParts}: batch ${myBatch} ok in ${Date.now() - t0}ms ` +
                        `(${rowCount} rows; part running ${recordsIngested})`
                    )
                    // Update progress: parts-done fraction + within-part fraction
                    const withinFraction = recordsIngested / Math.max(1, 20000)
                    const pct = Math.floor(
                      ((partIndex + withinFraction) / totalParts) * 100
                    )
                    // Fire-and-forget progress writes. Swallow transient Redis/
                    // job-update rejections: an un-awaited reject would otherwise
                    // bubble to an unhandledRejection and crash the worker, which
                    // BullMQ then reports as "job stalled more than allowable
                    // limit" (a dead worker stops renewing its lock).
                    void job
                      .updateProgress(
                        Math.min(pct, Math.floor(((partIndex + 1) / totalParts) * 100) - 1)
                      )
                      .catch(() => {})
                    void job
                      .updateData({
                        ...job.data,
                        recordsIngested: runningIngested + recordsIngested,
                        recordsSkipped: runningSkipped + recordsSkipped,
                      })
                      .catch(() => {})
                    callback()
                  })
                  .catch((upsertErr: unknown) => {
                    const msg = upsertErr instanceof Error ? upsertErr.message : String(upsertErr)
                    if (/timed out/i.test(msg)) {
                      // Systemic DB hang — fail the part loudly so BullMQ retries.
                      logger.error(
                        `[IngestDrugDataJob] part ${partIndex + 1}/${totalParts}: batch ${myBatch} TIMED OUT after ${Date.now() - t0}ms: ${msg}`
                      )
                      callback(upsertErr instanceof Error ? upsertErr : new Error(msg))
                      return
                    }
                    // Per-batch data error (a row the schema rejects, etc.) — log,
                    // count as skipped, and CONTINUE. One bad batch must not abort
                    // the whole ~259k ingest (over-correcting to fail-loud here is
                    // what let a single duplicate set_id kill the run at part 5).
                    logger.error(
                      `[IngestDrugDataJob] part ${partIndex + 1}/${totalParts}: batch ${myBatch} skipped after error (${Date.now() - t0}ms): ${msg}`
                    )
                    recordsSkipped += currentBatch.length
                    callback()
                  })
              },
              final(callback: (err?: Error | null) => void) {
                // Flush the last partial batch
                if (batch.length === 0) {
                  callback()
                  return
                }
                const remainingBatch = dedupeBySetId(batch)
                batch = []
                const t0 = Date.now()
                logger.info(
                  `[IngestDrugDataJob] part ${partIndex + 1}/${totalParts}: upserting final batch (${remainingBatch.length} rows)…`
                )
                withUpsertTimeout(
                  upsertDrugLabelBatch(remainingBatch),
                  remainingBatch.length,
                  UPSERT_TIMEOUT_MS
                )
                  .then((rowCount) => {
                    recordsIngested += rowCount
                    logger.info(
                      `[IngestDrugDataJob] part ${partIndex + 1}/${totalParts}: final batch ok in ${Date.now() - t0}ms (${rowCount} rows)`
                    )
                    callback()
                  })
                  .catch((upsertErr: unknown) => {
                    const msg = upsertErr instanceof Error ? upsertErr.message : String(upsertErr)
                    if (/timed out/i.test(msg)) {
                      logger.error(
                        `[IngestDrugDataJob] part ${partIndex + 1}/${totalParts}: final batch TIMED OUT after ${Date.now() - t0}ms: ${msg}`
                      )
                      callback(upsertErr instanceof Error ? upsertErr : new Error(msg))
                      return
                    }
                    logger.error(
                      `[IngestDrugDataJob] part ${partIndex + 1}/${totalParts}: final batch skipped after error (${Date.now() - t0}ms): ${msg}`
                    )
                    recordsSkipped += remainingBatch.length
                    callback()
                  })
              },
            })
            batchWriter.on('finish', () => {
              logger.info(
                `[IngestDrugDataJob] part ${partIndex + 1}/${totalParts}: stream finished — ` +
                  `ingested=${recordsIngested} skipped=${recordsSkipped} batches=${batchNum}`
              )
              resolve({ recordsIngested, recordsSkipped })
            })
            batchWriter.on('error', reject)
            readStream.on('error', reject)
            jsonParser.on('error', reject)
            pick.on('error', reject)
            streamArray.on('error', reject)
            // Pipeline: readStream → jsonParser → pick → streamArray → batchWriter
            readStream.pipe(jsonParser).pipe(pick).pipe(streamArray).pipe(batchWriter)
          })
        })
        zipFile.on('end', () => {
          // All zip entries enumerated. The batchWriter 'finish' event resolves
          // the promise once the last batch flushes.
        })
      })
    })
  }
  private async writeFinalStatus(
    exportDate: string,
    resourceMeta?: DrugDatasetResourceMeta
  ): Promise<void> {
    // Lazy import to keep module top level free of Lucid
    const KVStore = (await import('#models/kv_store')).default
    await KVStore.setValue('drugReference.lastUpdatedExportDate', exportDate)
    // Install-state write-back: when this ingest was kicked off by a curated-tier
    // install, record an `installed_resources` row so the tier-status math (which
    // is row-driven for ZIM/map) recognizes the dataset uniformly — the home-tile
    // gate and the tier "installed" badge both read these rows. resource_type
    // 'dataset' was widened onto the model in the foundational slice. The row's
    // `version` is the openFDA export_date (the real freshness key), not the
    // manifest placeholder. Manual downloads pass no resourceMeta → no row, which
    // is correct: install-state belongs to the curated-tier path only.
    if (!resourceMeta) return
    try {
      const { default: InstalledResource } = await import('#models/installed_resource')
      const { DateTime } = await import('luxon')
      const totalBytes = await this.totalDownloadedBytes()
      await InstalledResource.updateOrCreate(
        { resource_id: resourceMeta.resourceId, resource_type: 'dataset' },
        {
          version: exportDate,
          collection_ref: resourceMeta.collectionRef,
          url: 'https://api.fda.gov/download.json',
          // No single on-disk file — the parts are deleted after ingest; the
          // installed artifact is the DB table. Record the staging dir for
          // provenance; uninstall keys off resource_id, never this path.
          file_path: STORAGE_BASE,
          file_size_bytes: totalBytes,
          installed_at: DateTime.now(),
        }
      )
      logger.info(
        `[IngestDrugDataJob] Wrote installed_resources row for ${resourceMeta.resourceId} (export_date=${exportDate})`
      )
    } catch (err) {
      // A failed row write must NOT abort a completed ingest — the data is
      // already searchable. Log loud; the tier badge will simply read
      // not-installed until the next install reconcile.
      logger.error(
        `[IngestDrugDataJob] Failed to write installed_resources row for ${resourceMeta.resourceId}: ${
          err instanceof Error ? err.message : String(err)
        }`
      )
    }
  }
  /**
   * Sum the recorded part sizes from the download-state KV marker, for the
   * `installed_resources.file_size_bytes` column. Best-effort: the parts are
   * deleted right after a full ingest, so the marker (written before deletion) is
   * the only durable size source. Returns null when the marker is absent/empty so
   * the column stays NULL rather than reporting a wrong 0.
   */
  private async totalDownloadedBytes(): Promise<number | null> {
    try {
      const KVStore = (await import('#models/kv_store')).default
      const { parseDownloadState } = await import('../../util/drug_labels.js')
      const marker = parseDownloadState(await KVStore.getValue('drugReference.downloadState'))
      if (!marker || marker.parts.length === 0) return null
      const sum = marker.parts.reduce((acc, p) => acc + (p.bytes || 0), 0)
      return sum > 0 ? sum : null
    } catch {
      return null
    }
  }
  /**
   * Delete the downloaded part zips and clear the download-state marker, run once
   * after a full ingest succeeds (reclaims ~1.7 GB). A failed unlink is logged
   * but never aborts a completed ingest.
   */
  private async deleteDownloadedParts(
    manifest: DrugLabelManifest,
    totalParts: number
  ): Promise<void> {
    for (let i = 0; i < totalParts; i++) {
      const partition = manifest.partitions[i]
      if (!partition) continue
      const zipPath = partZipPath(STORAGE_BASE, partition)
      try {
        await fsPromises.unlink(zipPath)
        logger.info(`[IngestDrugDataJob] Deleted zip: ${zipPath}`)
      } catch (err) {
        logger.warn(
          `[IngestDrugDataJob] Could not delete zip ${zipPath}: ${err instanceof Error ? err.message : String(err)}`
        )
      }
    }
    const KVStore = (await import('#models/kv_store')).default
    await KVStore.clearValue('drugReference.downloadState')
  }
 }
--- a/admin/app/jobs/run_benchmark_job.ts
+++ b/admin/app/jobs/run_benchmark_job.ts
@ -1,101 +0,0 @@
 import { Job } from 'bullmq'
 import { QueueService } from '#services/queue_service'
 import { BenchmarkService } from '#services/benchmark_service'
 import type { RunBenchmarkJobParams } from '../../types/benchmark.js'
 import logger from '@adonisjs/core/services/logger'
 import { DockerService } from '#services/docker_service'
 export class RunBenchmarkJob {
  static get queue() {
    return 'benchmarks'
  }
  static get key() {
    return 'run-benchmark'
  }
  async handle(job: Job) {
    const { benchmark_id, benchmark_type } = job.data as RunBenchmarkJobParams
    logger.info(`[RunBenchmarkJob] Starting benchmark ${benchmark_id} of type ${benchmark_type}`)
    const dockerService = new DockerService()
    const benchmarkService = new BenchmarkService(dockerService)
    try {
      let result
      switch (benchmark_type) {
        case 'full':
          result = await benchmarkService.runFullBenchmark()
          break
        case 'system':
          result = await benchmarkService.runSystemBenchmarks()
          break
        case 'ai':
          result = await benchmarkService.runAIBenchmark()
          break
        default:
          throw new Error(`Unknown benchmark type: ${benchmark_type}`)
      }
      logger.info(`[RunBenchmarkJob] Benchmark ${benchmark_id} completed with NOMAD score: ${result.nomad_score}`)
      return {
        success: true,
        benchmark_id: result.benchmark_id,
        nomad_score: result.nomad_score,
      }
    } catch (error) {
      logger.error(`[RunBenchmarkJob] Benchmark ${benchmark_id} failed: ${error.message}`)
      throw error
    }
  }
  static async dispatch(params: RunBenchmarkJobParams) {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    try {
      const job = await queue.add(this.key, params, {
        jobId: params.benchmark_id,
        attempts: 1, // Benchmarks shouldn't be retried automatically
        removeOnComplete: {
          count: 10, // Keep last 10 completed jobs
        },
        removeOnFail: {
          count: 5, // Keep last 5 failed jobs
        },
      })
      logger.info(`[RunBenchmarkJob] Dispatched benchmark job ${params.benchmark_id}`)
      return {
        job,
        created: true,
        message: `Benchmark job ${params.benchmark_id} dispatched successfully`,
      }
    } catch (error) {
      if (error.message.includes('job already exists')) {
        const existing = await queue.getJob(params.benchmark_id)
        return {
          job: existing,
          created: false,
          message: `Benchmark job ${params.benchmark_id} already exists`,
        }
      }
      throw error
    }
  }
  static async getJob(benchmarkId: string): Promise<Job | undefined> {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    return await queue.getJob(benchmarkId)
  }
  static async getJobState(benchmarkId: string): Promise<string | undefined> {
    const job = await this.getJob(benchmarkId)
    return job ? await job.getState() : undefined
  }
 }
--- a/admin/app/jobs/run_download_job.ts
+++ b/admin/app/jobs/run_download_job.ts
@ -1,41 +1,11 @@
-import { Job, UnrecoverableError } from 'bullmq'
+import { Job } from 'bullmq'
-import { RunDownloadJobParams, DownloadProgressData } from '../../types/downloads.js'
+import { RunDownloadJobParams } from '../../types/downloads.js'
 import { QueueService } from '#services/queue_service'
-import { doResumableDownload, GatedContentAuthError } from '../utils/downloads.js'
+import { doResumableDownload } from '../utils/downloads.js'
 import { createHash } from 'crypto'
 import { DockerService } from '#services/docker_service'
 import { ZimService } from '#services/zim_service'
 import { MapService } from '#services/map_service'
 import { RagService } from '#services/rag_service'
 import { OllamaService } from '#services/ollama_service'
 import { EmbedFileJob } from './embed_file_job.js'
 import { basename, join, resolve, sep } from 'node:path'
 import { ZIM_STORAGE_PATH } from '../utils/fs.js'
 /** Maps live under `<cwd>/storage/maps/pmtiles`; no shared constant exists. */
 const MAP_STORAGE_PATH = '/storage/maps'
 /**
 * Guard for the outdated-file deletion in {@link RunDownloadJob} `onComplete`:
 * returns true only when `oldFilePath` sits under the expected content storage
 * root for its type AND its filename carries this resource's id prefix. This
 * makes the delete explicit and bounded — we only ever remove the replaced
 * resource's own previous file, never another file, even if the
 * InstalledResource row is stale or malformed.
 */
 function isSafeOldContentPath(
  oldFilePath: string,
  resourceId: string,
  filetype: string
 ): boolean {
  const root =
    filetype === 'zim'
      ? join(process.cwd(), ZIM_STORAGE_PATH)
      : join(process.cwd(), MAP_STORAGE_PATH)
  const resolved = resolve(oldFilePath)
  if (!resolved.startsWith(root + sep)) return false
  return basename(resolved).startsWith(`${resourceId}_`)
 }
 export class RunDownloadJob {
  static get queue() {
@ -46,333 +16,78 @@ export class RunDownloadJob {
    return 'run-download'
  }
  /** In-memory registry of abort controllers for active download jobs */
  static abortControllers: Map<string, AbortController> = new Map()
  static getJobId(url: string): string {
    return createHash('sha256').update(url).digest('hex').slice(0, 16)
  }
  /** Redis key used to signal cancellation across processes */
  static cancelKey(jobId: string): string {
    return `nomad:download:cancel:${jobId}`
  }
  /** Signal cancellation via Redis so the worker process can pick it up */
  static async signalCancel(jobId: string): Promise<void> {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    const client = await queue.client
    await client.set(this.cancelKey(jobId), '1', { EX: 300 }) // 5 min TTL
  }
  async handle(job: Job) {
-    const { url, filepath, timeout, allowedMimeTypes, forceNew, filetype, resourceMetadata, requestHeaders } =
+    const { url, filepath, timeout, allowedMimeTypes, forceNew, filetype } =
      job.data as RunDownloadJobParams
-    // Register abort controller for this job
+    //    console.log("Simulating delay for job for URL:", url)
-    const abortController = new AbortController()
+    //  await new Promise((resolve) => setTimeout(resolve, 30000)) // Simulate initial delay
-    RunDownloadJob.abortControllers.set(job.id!, abortController)
+    //  console.log("Starting download for URL:", url)
-    // Get Redis client for checking cancel signals from the API process
+    // // simulate progress updates for demonstration
-    const queueService = QueueService.getInstance()
+    // for (let progress = 0; progress <= 100; progress += 10) {
-    const cancelRedis = await queueService.getQueue(RunDownloadJob.queue).client
+    //   await new Promise((resolve) => setTimeout(resolve, 20000)) // Simulate time taken for each progress step
    //   job.updateProgress(progress)
    //   console.log(`Job progress for URL ${url}: ${progress}%`)
    // }
-    let lastKnownProgress: Pick<DownloadProgressData, 'downloadedBytes' | 'totalBytes'> = {
+    await doResumableDownload({
-      downloadedBytes: 0,
+      url,
-      totalBytes: 0,
+      filepath,
-    }
+      timeout,
-
+      allowedMimeTypes,
-    // Track whether cancellation was explicitly requested by the user (via Redis signal
+      forceNew,
-    // or in-process AbortController). BullMQ lock mismatches can also abort the download
+      onProgress(progress) {
-    // stream, but those should be retried — only user-initiated cancels are unrecoverable.
+        const progressPercent = (progress.downloadedBytes / (progress.totalBytes || 1)) * 100
-    let userCancelled = false
+        job.updateProgress(Math.floor(progressPercent))
-
+      },
-    // Poll Redis for cancel signal every 2s — independent of progress events so cancellation
+      async onComplete(url) {
-    // works even when the stream is stalled and no onProgress ticks are firing.
+        try {
-    let cancelPollInterval: ReturnType<typeof setInterval> | null = setInterval(async () => {
+          if (filetype === 'zim') {
-      try {
+            const dockerService = new DockerService()
-        const val = await cancelRedis.get(RunDownloadJob.cancelKey(job.id!))
+            const zimService = new ZimService(dockerService)
-        if (val) {
+            await zimService.downloadRemoteSuccessCallback([url], true)
-          await cancelRedis.del(RunDownloadJob.cancelKey(job.id!))
+          } else if (filetype === 'map') {
-          userCancelled = true
+            const mapsService = new MapService()
-          abortController.abort('user-cancel')
+            await mapsService.downloadRemoteSuccessCallback([url], false)
          }
        } catch (error) {
          console.error(
            `[RunDownloadJob] Error in ZIM download success callback for URL ${url}:`,
            error
          )
        }
-      } catch {
+        job.updateProgress(100)
-        // Redis errors are non-fatal; in-process AbortController covers same-process cancels
+      },
-      }
+    })
    }, 2000)
-    try {
+    return {
-      await doResumableDownload({
+      url,
-        url,
+      filepath,
        filepath,
        timeout,
        allowedMimeTypes,
        forceNew,
        requestHeaders,
        signal: abortController.signal,
        onProgress(progress) {
          const progressPercent = (progress.downloadedBytes / (progress.totalBytes || 1)) * 100
          const progressData: DownloadProgressData = {
            percent: Math.floor(progressPercent),
            downloadedBytes: progress.downloadedBytes,
            totalBytes: progress.totalBytes,
            lastProgressTime: Date.now(),
          }
          job.updateProgress(progressData).catch((err) => {
            // Job was removed from Redis (e.g. cancelled) between the callback firing
            // and the Redis write completing — this is expected and safe to ignore.
            if (err?.code !== -1) throw err
          })
          lastKnownProgress = { downloadedBytes: progress.downloadedBytes, totalBytes: progress.totalBytes }
        },
        async onComplete(url) {
          // The previous file recorded for this resource (if any). Hoisted out of
          // the metadata block below so the ZIM branch can decide whether this
          // download is a content UPDATE (replacing a prior file) vs a fresh
          // install, which changes how we reconcile the knowledge base.
          let oldFilePath: string | null = null
          try {
            // Create InstalledResource entry if metadata was provided
            if (resourceMetadata) {
              const { default: InstalledResource } = await import('#models/installed_resource')
              const { DateTime } = await import('luxon')
              const { getFileStatsIfExists, deleteFileIfExists } = await import('../utils/fs.js')
              const stats = await getFileStatsIfExists(filepath)
              // Look up the old entry so we can clean up the previous file after updating
              const oldEntry = await InstalledResource.query()
                .where('resource_id', resourceMetadata.resource_id)
                .where('resource_type', filetype as 'zim' | 'map')
                .first()
              oldFilePath = oldEntry?.file_path ?? null
              const installed = await InstalledResource.updateOrCreate(
                { resource_id: resourceMetadata.resource_id, resource_type: filetype as 'zim' | 'map' },
                {
                  version: resourceMetadata.version,
                  collection_ref: resourceMetadata.collection_ref,
                  url: url,
                  file_path: filepath,
                  file_size_bytes: stats ? Number(stats.size) : null,
                  installed_at: DateTime.now(),
                }
              )
              // A completed auto-update is the authoritative success signal for the
              // per-resource backoff — clear it here (NOT at dispatch time, which
              // would reset the counter every window and defeat self-disable). The
              // matching terminal-failure increment lives in the worker `failed`
              // handler (commands/queue/work.ts). Manual downloads (auto !== true)
              // never touch the counter.
              if (resourceMetadata.auto === true) {
                try {
                  const { recordResourceUpdateSuccess } = await import(
                    '../utils/content_auto_update_backoff.js'
                  )
                  await recordResourceUpdateSuccess(installed)
                } catch (error) {
                  console.error(
                    `[RunDownloadJob] Error clearing auto-update backoff for ${resourceMetadata.resource_id}:`,
                    error
                  )
                }
              }
              // Step 1: delete the OUTDATED file if it differs from the new one.
              // Guarded by isSafeOldContentPath so we can ONLY ever delete the
              // replaced resource's own previous file — never another resource's
              // file, even if the InstalledResource row is stale/malformed.
              if (oldFilePath && oldFilePath !== filepath) {
                if (isSafeOldContentPath(oldFilePath, resourceMetadata.resource_id, filetype)) {
                  try {
                    await deleteFileIfExists(oldFilePath)
                    console.log(`[RunDownloadJob] Deleted old file: ${oldFilePath}`)
                  } catch (deleteError) {
                    console.warn(
                      `[RunDownloadJob] Failed to delete old file ${oldFilePath}:`,
                      deleteError
                    )
                  }
                } else {
                  console.warn(
                    `[RunDownloadJob] Refusing to delete unexpected old file path for ` +
                      `${resourceMetadata.resource_id} (${filetype}): ${oldFilePath}`
                  )
                }
              }
            }
            if (filetype === 'zim') {
              const dockerService = new DockerService()
              const zimService = new ZimService(dockerService)
              await zimService.downloadRemoteSuccessCallback([url], true)
              // Only touch the knowledge base if AI Assistant (Ollama) is installed.
              // skip_embedding opts a ZIM out entirely — Creator Pack video ZIMs are
              // media galleries, not text, so they must never be embedded or KB-reconciled.
              const ollamaUrl = await dockerService.getServiceURL('nomad_ollama')
              if (ollamaUrl && !resourceMetadata?.skip_embedding) {
                // A content UPDATE replaces a prior file at a DIFFERENT path
                // (version is in the filename). A fresh install has no prior row;
                // a same-version re-download keeps the same path. The two cases
                // reconcile the KB differently.
                const isReplacement = !!oldFilePath && oldFilePath !== filepath
                if (isReplacement) {
                  // CONTENT UPDATE: mirror the REPLACED file's prior indexed state
                  // rather than the global Always/Manual policy. reconcileReplaced-
                  // ContentFile removes the old file's points and re-queues the new
                  // file IFF the old one was indexed and Qdrant is running; it is a
                  // no-op otherwise (not installed / old not indexed / Qdrant down).
                  // The user already chose whether this content is in the KB, so we
                  // honor that choice in both directions. See the method for the
                  // full 5-step contract.
                  try {
                    const ragService = new RagService(dockerService, new OllamaService())
                    const outcome = await ragService.reconcileReplacedContentFile({
                      oldFilePath: oldFilePath!,
                      newFilePath: filepath,
                      fileName: url.split('/').pop() || '',
                    })
                    console.log(
                      `[RunDownloadJob] KB reconciliation for replaced ${filepath}: ${outcome}`
                    )
                  } catch (error) {
                    console.error(
                      `[RunDownloadJob] Error reconciling knowledge base for replaced file ${filepath}:`,
                      error
                    )
                  }
                } else {
                  // FRESH INSTALL (or same-version re-download): respect the global
                  // ingest policy. Under Manual, record the file as pending_decision
                  // so the KB panel surfaces the per-file Index affordance (PR #909)
                  // instead of silently auto-embedding behind the user's back. Unset
                  // is treated as Always to preserve legacy behavior — mirrors
                  // rag_service.ts:1587-1588.
                  const { default: KVStore } = await import('#models/kv_store')
                  const { default: KbIngestState } = await import('#models/kb_ingest_state')
                  const policyRaw = await KVStore.getValue('rag.defaultIngestPolicy')
                  const policy: 'Always' | 'Manual' = policyRaw === 'Manual' ? 'Manual' : 'Always'
                  if (policy === 'Manual') {
                    try {
                      // firstOrCreate so a re-download doesn't demote an existing
                      // indexed/failed row — user keeps prior state and can re-index
                      // explicitly from the KB panel if they want fresh content.
                      await KbIngestState.firstOrCreate(
                        { file_path: filepath },
                        { file_path: filepath, state: 'pending_decision', chunks_embedded: 0 }
                      )
                    } catch (error) {
                      console.error(
                        `[RunDownloadJob] Error recording pending_decision state for ${filepath}:`,
                        error
                      )
                    }
                  } else {
                    try {
                      await EmbedFileJob.dispatch({
                        fileName: url.split('/').pop() || '',
                        filePath: filepath,
                      })
                    } catch (error) {
                      console.error(`[RunDownloadJob] Error dispatching EmbedFileJob for URL ${url}:`, error)
                    }
                  }
                }
              }
            } else if (filetype === 'map') {
              const mapsService = new MapService()
              await mapsService.downloadRemoteSuccessCallback([url], false)
            }
          } catch (error) {
            console.error(
              `[RunDownloadJob] Error in download success callback for URL ${url}:`,
              error
            )
          }
          job.updateProgress({
            percent: 100,
            downloadedBytes: lastKnownProgress.downloadedBytes,
            totalBytes: lastKnownProgress.totalBytes,
            lastProgressTime: Date.now(),
          } as DownloadProgressData).catch((err) => {
            if (err?.code !== -1) throw err
          })
        },
      })
      return {
        url,
        filepath,
      }
    } catch (error: any) {
      // Only prevent retries for user-initiated cancellations. BullMQ lock mismatches
      // can also abort the stream, and those should be retried with backoff.
      // Check both the flag (Redis poll) and abort reason (in-process cancel).
      if (userCancelled || abortController.signal.reason === 'user-cancel') {
        throw new UnrecoverableError(`Download cancelled: ${error.message}`)
      }
      // A rejected entitlement is permanent - this build either has the key or it
      // doesn't. Left as a plain Error it consumes all 10 attempts with exponential
      // backoff from 30s, so the job reads `delayed` for ~4h15m and the user sees a
      // stuck download instead of the message above.
      if (error instanceof GatedContentAuthError) {
        throw new UnrecoverableError(error.message)
      }
      throw error
    } finally {
      if (cancelPollInterval !== null) {
        clearInterval(cancelPollInterval)
        cancelPollInterval = null
      }
      RunDownloadJob.abortControllers.delete(job.id!)
    }
  }
  static async getByUrl(url: string): Promise<Job | undefined> {
-    const queueService = QueueService.getInstance()
+    const queueService = new QueueService()
    const queue = queueService.getQueue(this.queue)
    const jobId = this.getJobId(url)
    return await queue.getJob(jobId)
  }
  /**
   * Check if a download is actively in progress for the given URL.
   * Returns the job only if it's in an active state (active, waiting, delayed).
   * If the job exists in a terminal state (failed, completed), removes it and returns undefined.
   */
  static async getActiveByUrl(url: string): Promise<Job | undefined> {
    const job = await this.getByUrl(url)
    if (!job) return undefined
    const state = await job.getState()
    if (state === 'active' || state === 'waiting' || state === 'delayed') {
      return job
    }
    // Terminal state -- clean up stale job so it doesn't block re-download
    try {
      await job.remove()
    } catch {
      // May already be gone
    }
    return undefined
  }
  static async dispatch(params: RunDownloadJobParams) {
-    const queueService = QueueService.getInstance()
+    const queueService = new QueueService()
    const queue = queueService.getQueue(this.queue)
    const jobId = this.getJobId(params.url)
    try {
      const job = await queue.add(this.key, params, {
        jobId,
-        attempts: 10,
+        attempts: 3,
-        backoff: { type: 'exponential', delay: 30000 },
+        backoff: { type: 'exponential', delay: 2000 },
        removeOnComplete: true,
      })
--- a/admin/app/jobs/run_extract_pmtiles_job.ts
+++ b/admin/app/jobs/run_extract_pmtiles_job.ts
@ -1,294 +0,0 @@
 import { Job, UnrecoverableError } from 'bullmq'
 import { spawn, ChildProcess } from 'child_process'
 import { createHash } from 'crypto'
 import { readdir, stat } from 'fs/promises'
 import { basename, dirname, join } from 'path'
 import { QueueService } from '#services/queue_service'
 import logger from '@adonisjs/core/services/logger'
 import { DownloadProgressData } from '../../types/downloads.js'
 import { PMTILES_BINARY_PATH, buildPmtilesExtractArgs } from '../../constants/map_regions.js'
 import { deleteFileIfExists } from '../utils/fs.js'
 export interface RunExtractPmtilesJobParams {
  sourceUrl: string
  outputFilepath: string
  /** Path to a GeoJSON FeatureCollection file passed to `pmtiles extract --region`. */
  regionFilepath: string
  maxzoom?: number
  /** Hint for progress reporting; obtained from `pmtiles extract --dry-run` preflight */
  estimatedBytes?: number
  filetype: 'map'
  title?: string
  resourceMetadata?: {
    resource_id: string
    version: string
    collection_ref: string | null
  }
 }
 export class RunExtractPmtilesJob {
  static get queue() {
    return 'pmtiles-extract'
  }
  static get key() {
    return 'run-pmtiles-extract'
  }
  /** In-memory registry of active child processes so in-process cancels can SIGTERM them */
  static childProcesses: Map<string, ChildProcess> = new Map()
  static getJobId(sourceUrl: string, regionFilepath: string, maxzoom?: number): string {
    const payload = JSON.stringify({ sourceUrl, regionFilepath, maxzoom: maxzoom ?? null })
    return createHash('sha256').update(payload).digest('hex').slice(0, 16)
  }
  /** Redis key used to signal cancellation across processes */
  static cancelKey(jobId: string): string {
    return `nomad:download:pmtiles-cancel:${jobId}`
  }
  static async signalCancel(jobId: string): Promise<void> {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    const client = await queue.client
    await client.set(this.cancelKey(jobId), '1', { EX: 300 })
  }
  /** Awaits job.updateProgress and swallows BullMQ stale-job errors (code -1),
   *  which occur when the job was removed from Redis (e.g. cancelled) between
   *  the await being issued and the Redis write completing. Anything else
   *  re-throws so it's caught by the surrounding try rather than becoming an
   *  unhandled rejection. */
  private async safeUpdateProgress(job: Job, progress: DownloadProgressData): Promise<void> {
    try {
      await job.updateProgress(progress)
    } catch (err: any) {
      if (err?.code !== -1) throw err
    }
  }
  async handle(job: Job) {
    const params = job.data as RunExtractPmtilesJobParams
    const { sourceUrl, outputFilepath, regionFilepath, maxzoom, estimatedBytes } = params
    logger.info(
      `[RunExtractPmtilesJob] Starting extract: source=${sourceUrl} region=${regionFilepath} ` +
        `maxzoom=${maxzoom ?? 'source-max'} out=${outputFilepath}`
    )
    const queueService = QueueService.getInstance()
    const cancelRedis = await queueService.getQueue(RunExtractPmtilesJob.queue).client
    let userCancelled = false
    let proc: ChildProcess | null = null
    let lastReportedBytes = -1
    // One 2s tick polls the Redis cancel signal and reads file-size for progress. pmtiles
    // writes incrementally but rewrites directories near the end so progress isn't strictly
    // monotonic — we cap at 99% and skip emit when bytes are unchanged to avoid Redis chatter.
    const tick = setInterval(async () => {
      try {
        const val = await cancelRedis.get(RunExtractPmtilesJob.cancelKey(job.id!))
        if (val) {
          await cancelRedis.del(RunExtractPmtilesJob.cancelKey(job.id!))
          userCancelled = true
          proc?.kill('SIGTERM')
        }
      } catch {
        // Redis errors non-fatal — in-memory handle also covers same-process cancels
      }
      try {
        const fileStat = await stat(outputFilepath)
        const downloadedBytes = Number(fileStat.size)
        if (downloadedBytes === lastReportedBytes) return
        lastReportedBytes = downloadedBytes
        const totalBytes = estimatedBytes ?? 0
        const percent =
          totalBytes > 0 ? Math.min(99, Math.floor((downloadedBytes / totalBytes) * 100)) : 0
        await this.safeUpdateProgress(job, {
          percent,
          downloadedBytes,
          totalBytes,
          lastProgressTime: Date.now(),
        } as DownloadProgressData)
      } catch {
        // File doesn't exist yet (subprocess still setting up)
      }
    }, 2000)
    try {
      const args = buildPmtilesExtractArgs({
        sourceUrl,
        outputFilepath,
        regionFilepath,
        maxzoom,
        downloadThreads: 8,
        overfetch: 0.2,
      })
      proc = spawn(PMTILES_BINARY_PATH, args, { stdio: ['ignore', 'pipe', 'pipe'] })
      RunExtractPmtilesJob.childProcesses.set(job.id!, proc)
      proc.stdout?.on('data', (chunk) => {
        logger.debug(`[RunExtractPmtilesJob:${job.id}] ${chunk.toString().trimEnd()}`)
      })
      proc.stderr?.on('data', (chunk) => {
        logger.debug(`[RunExtractPmtilesJob:${job.id}] ${chunk.toString().trimEnd()}`)
      })
      const exitCode: number = await new Promise((resolve, reject) => {
        proc!.on('close', (code) => resolve(code ?? -1))
        proc!.on('error', (err) => reject(err))
      })
      if (exitCode !== 0) {
        await deleteFileIfExists(outputFilepath)
        if (userCancelled) {
          throw new UnrecoverableError(`Extract cancelled by user (exit ${exitCode})`)
        }
        throw new Error(`pmtiles extract exited with code ${exitCode}`)
      }
      // Final progress bump — tick caps at 99 so the UI doesn't flicker to 100 mid-extract
      const finalStat = await stat(outputFilepath)
      await this.safeUpdateProgress(job, {
        percent: 100,
        downloadedBytes: Number(finalStat.size),
        totalBytes: estimatedBytes ?? Number(finalStat.size),
        lastProgressTime: Date.now(),
      } as DownloadProgressData)
      // Reuse the HTTP download path's post-download hook so the file is registered and
      // the previous version (if any) is deleted
      await this.onComplete(params)
      logger.info(
        `[RunExtractPmtilesJob] Completed extract: out=${outputFilepath} size=${finalStat.size} bytes`
      )
      return { sourceUrl, outputFilepath }
    } catch (error: any) {
      if (userCancelled && !(error instanceof UnrecoverableError)) {
        throw new UnrecoverableError(`Extract cancelled: ${error.message ?? error}`)
      }
      throw error
    } finally {
      clearInterval(tick)
      RunExtractPmtilesJob.childProcesses.delete(job.id!)
    }
  }
  private async onComplete(params: RunExtractPmtilesJobParams) {
    if (!params.resourceMetadata) return
    const [{ default: InstalledResource }, { DateTime }, fsUtils] = await Promise.all([
      import('#models/installed_resource'),
      import('luxon'),
      import('../utils/fs.js'),
    ])
    const fileStat = await fsUtils.getFileStatsIfExists(params.outputFilepath)
    const existing = await InstalledResource.query()
      .where('resource_id', params.resourceMetadata.resource_id)
      .where('resource_type', 'map')
      .first()
    const oldFilePath = existing?.file_path ?? null
    await InstalledResource.updateOrCreate(
      {
        resource_id: params.resourceMetadata.resource_id,
        resource_type: 'map',
      },
      {
        version: params.resourceMetadata.version,
        collection_ref: params.resourceMetadata.collection_ref,
        url: params.sourceUrl,
        file_path: params.outputFilepath,
        file_size_bytes: fileStat ? Number(fileStat.size) : null,
        installed_at: DateTime.now(),
      }
    )
    if (oldFilePath && oldFilePath !== params.outputFilepath) {
      try {
        await fsUtils.deleteFileIfExists(oldFilePath)
      } catch (err) {
        logger.warn(`[RunExtractPmtilesJob] Failed to delete old file ${oldFilePath}: ${err}`)
      }
    }
    // Fallback: scan the pmtiles dir for orphans with the same resource_id that the DB
    // lookup above didn't catch — e.g. a prior extract crashed before writing its
    // InstalledResource row, or an earlier bug wrote a file without registering it.
    // Matches both curated (`<id>_YYYY-MM.pmtiles`) and regional (`<id>_YYYYMMDD_zN.pmtiles`)
    // naming — prefix-only so new filename formats don't silently miss.
    const dir = dirname(params.outputFilepath)
    const keepName = basename(params.outputFilepath)
    const prefix = `${params.resourceMetadata.resource_id}_`
    try {
      const entries = await readdir(dir)
      for (const entry of entries) {
        if (entry === keepName || !entry.endsWith('.pmtiles')) continue
        if (!entry.startsWith(prefix)) continue
        const orphanPath = join(dir, entry)
        if (orphanPath === oldFilePath) continue
        try {
          await fsUtils.deleteFileIfExists(orphanPath)
          logger.info(`[RunExtractPmtilesJob] Pruned orphan pmtiles ${orphanPath}`)
        } catch (err) {
          logger.warn(`[RunExtractPmtilesJob] Failed to prune orphan ${orphanPath}: ${err}`)
        }
      }
    } catch (err) {
      logger.warn(`[RunExtractPmtilesJob] Directory scan for orphans failed: ${err}`)
    }
  }
  static async getById(jobId: string): Promise<Job | undefined> {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    return await queue.getJob(jobId)
  }
  static async dispatch(params: RunExtractPmtilesJobParams) {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(this.queue)
    const jobId = this.getJobId(params.sourceUrl, params.regionFilepath, params.maxzoom)
    const existing = await queue.getJob(jobId)
    if (existing) {
      const state = await existing.getState()
      if (state === 'active' || state === 'waiting' || state === 'delayed') {
        return {
          job: existing,
          created: false,
          message: `Extract job already exists for these params`,
        }
      }
      // Stale (completed/failed) — remove so we can re-dispatch under the same deterministic id
      try {
        await existing.remove()
      } catch {
        // Already gone or locked — add() below will still report a meaningful error
      }
    }
    // Fewer attempts than HTTP downloads — a failed extract usually means the source URL
    // rotated or the CDN is throttling, and resuming mid-extract isn't supported by the CLI
    const job = await queue.add(this.key, params, {
      jobId,
      attempts: 3,
      backoff: { type: 'exponential', delay: 60000 },
      removeOnComplete: true,
    })
    return {
      job,
      created: true,
      message: `Dispatched pmtiles extract job`,
    }
  }
 }
--- a/admin/app/middleware/compression_middleware.ts
+++ b/admin/app/middleware/compression_middleware.ts
@ -1,35 +0,0 @@
 import env from '#start/env'
 import type { HttpContext } from '@adonisjs/core/http'
 import type { NextFn } from '@adonisjs/core/types/http'
 import compression from 'compression'
 // Skip compression for Server-Sent Events. The compression library buffers
 // response writes to determine encoding, which collapses per-token streaming
 // into a single block delivered after generation completes (regression in
 // v1.31.0-rc.2, reported in #781 by @toasterking).
 const compress = env.get('DISABLE_COMPRESSION')
  ? null
  : compression({
      filter: (req: any, res: any) => {
        const contentType = res.getHeader('Content-Type')
        if (typeof contentType === 'string' && contentType.includes('text/event-stream')) {
          return false
        }
        return compression.filter(req, res)
      },
    })
 export default class CompressionMiddleware {
  async handle({ request, response }: HttpContext, next: NextFn) {
    if (!compress) return await next()
    await new Promise<void>((resolve, reject) => {
      compress(request.request as any, response.response as any, (err?: any) => {
        if (err) reject(err)
        else resolve()
      })
    })
    await next()
  }
 }
--- a/admin/app/models/benchmark_result.ts
+++ b/admin/app/models/benchmark_result.ts
@ -1,158 +0,0 @@
 import { DateTime } from 'luxon'
 import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
 import type { BenchmarkType, DiskType } from '../../types/benchmark.js'
 export default class BenchmarkResult extends BaseModel {
  static namingStrategy = new SnakeCaseNamingStrategy()
  @column({ isPrimary: true })
  declare id: number
  @column()
  declare benchmark_id: string
  @column()
  declare benchmark_type: BenchmarkType
  // Hardware information
  @column()
  declare cpu_model: string
  @column()
  declare cpu_cores: number
  @column()
  declare cpu_threads: number
  @column()
  declare ram_bytes: number
  @column()
  declare disk_type: DiskType
  @column()
  declare gpu_model: string | null
  // System benchmark scores
  @column()
  declare cpu_score: number
  @column()
  declare memory_score: number
  @column()
  declare disk_read_score: number
  @column()
  declare disk_write_score: number
  // AI benchmark scores (nullable for system-only benchmarks)
  @column()
  declare ai_tokens_per_second: number | null
  @column()
  declare ai_model_used: string | null
  @column()
  declare ai_time_to_first_token: number | null
  // Harness forensic metadata (nullable — added in Score v2 Phase 1)
  @column()
  declare sysbench_digest: string | null
  @column()
  declare ollama_version: string | null
  // Platform metadata (nullable). Sourced from the Docker daemon, not
  // systeminformation: inside the admin container si.osInfo()/os.arch()
  // describe the container, not the host.
  @column()
  declare cpu_architecture: string | null
  @column()
  declare os_name: string | null
  @column()
  declare os_version: string | null
  // NOMAD Score v2 raw channels (nullable — added in Score v2 Phase 4). Populated
  // on full benchmarks under benchmark_version >= 2.0.0; the leaderboard recomputes
  // the score from these on submit. cpu_events_multi is measured at
  // cpu_benchmark_threads; memory_ops_per_sec at memory_threads. Disk figures are
  // O_DIRECT MB/s. cpu_total_events/cpu_total_time are the W6 consistency companions.
  @column()
  declare cpu_events_single: number | null
  @column()
  declare cpu_events_multi: number | null
  @column()
  declare cpu_benchmark_threads: number | null
  @column()
  declare cpu_total_events: number | null
  @column()
  declare cpu_total_time: number | null
  @column()
  declare memory_ops_per_sec: number | null
  @column()
  declare memory_threads: number | null
  @column()
  declare disk_read_mb_per_sec: number | null
  @column()
  declare disk_write_mb_per_sec: number | null
  // Uncapped NOMAD Score v2 (null for pre-v2 rows and system-only runs). The
  // legacy nomad_score below is retained in parallel for display continuity.
  // columnName + serializeAs pinned: the snake_case strategy would otherwise map
  // this property to `nomad_score_v_2` (splitting the digit) for both the DB column
  // (which the migration doesn't create) and the JSON key (which the frontend reads
  // as nomad_score_v2). Pin both so DB, API, and UI all agree.
  @column({ columnName: 'nomad_score_v2', serializeAs: 'nomad_score_v2' })
  declare nomad_score_v2: number | null
  // Best-effort run environment metadata (issue #1016)
  @column()
  declare run_environment: string | null
  @column()
  declare storage_path_type: string | null
  @column({
    // Nullable tri-state; coerce the SQLite 0/1 int to a real boolean when present.
    consume: (value: number | null) => (value === null || value === undefined ? null : Boolean(value)),
  })
  declare gpu_compute_detected: boolean | null
  // Composite NOMAD score (0-100)
  @column()
  declare nomad_score: number
  // Repository submission tracking
  @column({
    serialize(value) {
      return Boolean(value)
    },
  })
  declare submitted_to_repository: boolean
  @column.dateTime()
  declare submitted_at: DateTime | null
  @column()
  declare repository_id: string | null
  @column()
  declare builder_tag: string | null
  @column.dateTime({ autoCreate: true })
  declare created_at: DateTime
  @column.dateTime({ autoCreate: true, autoUpdate: true })
  declare updated_at: DateTime
 }
--- a/admin/app/models/benchmark_setting.ts
+++ b/admin/app/models/benchmark_setting.ts
@ -1,60 +0,0 @@
 import { DateTime } from 'luxon'
 import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
 import type { BenchmarkSettingKey } from '../../types/benchmark.js'
 export default class BenchmarkSetting extends BaseModel {
  static namingStrategy = new SnakeCaseNamingStrategy()
  @column({ isPrimary: true })
  declare id: number
  @column()
  declare key: BenchmarkSettingKey
  @column()
  declare value: string | null
  @column.dateTime({ autoCreate: true })
  declare created_at: DateTime
  @column.dateTime({ autoCreate: true, autoUpdate: true })
  declare updated_at: DateTime
  /**
   * Get a setting value by key
   */
  static async getValue(key: BenchmarkSettingKey): Promise<string | null> {
    const setting = await this.findBy('key', key)
    return setting?.value ?? null
  }
  /**
   * Set a setting value by key (creates if not exists)
   */
  static async setValue(key: BenchmarkSettingKey, value: string | null): Promise<BenchmarkSetting> {
    const setting = await this.firstOrCreate({ key }, { key, value })
    if (setting.value !== value) {
      setting.value = value
      await setting.save()
    }
    return setting
  }
  /**
   * Get all benchmark settings as a typed object
   */
  static async getAllSettings(): Promise<{
    allow_anonymous_submission: boolean
    installation_id: string | null
    last_benchmark_run: string | null
  }> {
    const settings = await this.all()
    const map = new Map(settings.map((s) => [s.key, s.value]))
    return {
      allow_anonymous_submission: map.get('allow_anonymous_submission') === 'true',
      installation_id: map.get('installation_id') ?? null,
      last_benchmark_run: map.get('last_benchmark_run') ?? null,
    }
  }
 }
--- a/admin/app/models/chat_message.ts
+++ b/admin/app/models/chat_message.ts
@ -1,29 +0,0 @@
 import { DateTime } from 'luxon'
 import { BaseModel, column, belongsTo, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
 import type { BelongsTo } from '@adonisjs/lucid/types/relations'
 import ChatSession from './chat_session.js'
 export default class ChatMessage extends BaseModel {
  static namingStrategy = new SnakeCaseNamingStrategy()
  @column({ isPrimary: true })
  declare id: number
  @column()
  declare session_id: number
  @column()
  declare role: 'system' | 'user' | 'assistant'
  @column()
  declare content: string
  @belongsTo(() => ChatSession, { foreignKey: 'session_id', localKey: 'id' })
  declare session: BelongsTo<typeof ChatSession>
  @column.dateTime({ autoCreate: true })
  declare created_at: DateTime
  @column.dateTime({ autoCreate: true, autoUpdate: true })
  declare updated_at: DateTime
 }
--- a/admin/app/models/chat_session.ts
+++ b/admin/app/models/chat_session.ts
@ -1,29 +0,0 @@
 import { DateTime } from 'luxon'
 import { BaseModel, column, hasMany, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
 import type { HasMany } from '@adonisjs/lucid/types/relations'
 import ChatMessage from './chat_message.js'
 export default class ChatSession extends BaseModel {
  static namingStrategy = new SnakeCaseNamingStrategy()
  @column({ isPrimary: true })
  declare id: number
  @column()
  declare title: string
  @column()
  declare model: string | null
  @hasMany(() => ChatMessage, {
    foreignKey: 'session_id',
    localKey: 'id',
  })
  declare messages: HasMany<typeof ChatMessage>
  @column.dateTime({ autoCreate: true })
  declare created_at: DateTime
  @column.dateTime({ autoCreate: true, autoUpdate: true })
  declare updated_at: DateTime
 }
--- a/admin/app/models/collection_manifest.ts
+++ b/admin/app/models/collection_manifest.ts
@ -1,22 +0,0 @@
 import { DateTime } from 'luxon'
 import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
 import type { ManifestType } from '../../types/collections.js'
 export default class CollectionManifest extends BaseModel {
  static namingStrategy = new SnakeCaseNamingStrategy()
  @column({ isPrimary: true })
  declare type: ManifestType
  @column()
  declare spec_version: string
  @column({
    consume: (value: string) => (typeof value === 'string' ? JSON.parse(value) : value),
    prepare: (value: any) => JSON.stringify(value),
  })
  declare spec_data: any
  @column.dateTime()
  declare fetched_at: DateTime
 }
--- a/admin/app/models/curated_collection.ts
+++ b/admin/app/models/curated_collection.ts
@ -0,0 +1,39 @@
 import { DateTime } from 'luxon'
 import { BaseModel, column, hasMany, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
 import CuratedCollectionResource from './curated_collection_resource.js'
 import type { HasMany } from '@adonisjs/lucid/types/relations'
 import type { CuratedCollectionType } from '../../types/curated_collections.js'
 export default class CuratedCollection extends BaseModel {
  static namingStrategy = new SnakeCaseNamingStrategy()
  @column({ isPrimary: true })
  declare slug: string
  @column()
  declare type: CuratedCollectionType
  @column()
  declare name: string
  @column()
  declare description: string
  @column()
  declare icon: string
  @column()
  declare language: string
  @hasMany(() => CuratedCollectionResource, {
    foreignKey: 'curated_collection_slug',
    localKey: 'slug',
  })
  declare resources: HasMany<typeof CuratedCollectionResource>
  @column.dateTime({ autoCreate: true })
  declare created_at: DateTime
  @column.dateTime({ autoCreate: true, autoUpdate: true })
  declare updated_at: DateTime
 }
--- a/admin/app/models/curated_collection_resource.ts
+++ b/admin/app/models/curated_collection_resource.ts
@ -0,0 +1,41 @@
 import { DateTime } from 'luxon'
 import { BaseModel, belongsTo, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
 import CuratedCollection from './curated_collection.js'
 import type { BelongsTo } from '@adonisjs/lucid/types/relations'
 export default class CuratedCollectionResource extends BaseModel {
  static namingStrategy = new SnakeCaseNamingStrategy()
  @column({ isPrimary: true })
  declare id: number
  @column()
  declare curated_collection_slug: string
  @belongsTo(() => CuratedCollection, {
    foreignKey: 'slug',
    localKey: 'curated_collection_slug',
  })
  declare curated_collection: BelongsTo<typeof CuratedCollection>
  @column()
  declare title: string
  @column()
  declare url: string
  @column()
  declare description: string
  @column()
  declare size_mb: number
  @column()
  declare downloaded: boolean
  @column.dateTime({ autoCreate: true })
  declare created_at: DateTime
  @column.dateTime({ autoCreate: true, autoUpdate: true })
  declare updated_at: DateTime
 }
--- a/admin/app/models/custom_library_source.ts
+++ b/admin/app/models/custom_library_source.ts
@ -1,24 +0,0 @@
 import { DateTime } from 'luxon'
 import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
 export default class CustomLibrarySource extends BaseModel {
  static namingStrategy = new SnakeCaseNamingStrategy()
  @column({ isPrimary: true })
  declare id: number
  @column()
  declare name: string
  @column()
  declare base_url: string
  @column()
  declare is_default: boolean
  @column.dateTime({ autoCreate: true })
  declare created_at: DateTime
  @column.dateTime({ autoCreate: true, autoUpdate: true })
  declare updated_at: DateTime
 }
--- a/admin/app/models/drug_label.ts
+++ b/admin/app/models/drug_label.ts
@ -1,116 +0,0 @@
 import { DateTime } from 'luxon'
 import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
 /**
 * Drug Reference v1 — openFDA drug label catalog entry.
 *
 * One row per FDA `set_id` (stable GUID for a labeling, across all revisions).
 * Re-ingesting updates existing rows idempotently via `set_id` UNIQUE key.
 *
 * Enum-ish columns (product_type) are plain varchars validated at the edge —
 * the stl_files / inventory_items convention: no native DB enums, so the
 * schema can evolve without ALTER TABLE.
 *
 * All section-text columns are optional: label records frequently omit
 * sections (e.g. OTC labels have no `boxed_warning`). The mapper returns
 * null for any absent or empty section.
 */
 export default class DrugLabel extends BaseModel {
  static table = 'drug_labels'
  static namingStrategy = new SnakeCaseNamingStrategy()
  @column({ isPrimary: true })
  declare id: number
  /** openFDA set_id — stable GUID across label revisions. Idempotent upsert key. */
  @column()
  declare set_id: string
  /** openFDA id — per-revision GUID for provenance. */
  @column()
  declare spl_id: string | null
  @column()
  declare version: string | null
  /** openfda.brand_name[0] — first element only. */
  @column()
  declare brand_name: string | null
  /** openfda.generic_name joined with ", " (labels can list several). */
  @column()
  declare generic_name: string | null
  /** openfda.manufacturer_name[0]. */
  @column()
  declare manufacturer: string | null
  /** openfda.product_ndc joined with ", ". */
  @column()
  declare product_ndc: string | null
  /** openfda.route joined with ", ". */
  @column()
  declare route: string | null
  /**
   * openfda.product_type[0]. Drives OTC vs Rx badge + filter.
   * Expected values: 'HUMAN OTC DRUG' | 'HUMAN PRESCRIPTION DRUG'.
   */
  @column()
  declare product_type: string | null
  /**
   * Normalized brand+generic blob for FULLTEXT and LIKE search.
   * Built by normalizeDrugName() at ingest time; never re-computed on read.
   * Max 768 chars — stays inside InnoDB utf8mb4 index key-length budget.
   */
  @column()
  declare searchable_name: string | null
  /** Flattened indications_and_usage sections, joined with \n\n. */
  @column()
  declare indications: string | null
  /** Flattened dosage_and_administration sections. */
  @column()
  declare dosage: string | null
  /** Flattened warnings sections. */
  @column()
  declare warnings: string | null
  /** Flattened boxed_warning sections (absent on most OTC labels). */
  @column()
  declare boxed_warning: string | null
  /**
   * Flattened drug_interactions label text.
   * Single-drug label info only — NOT a pairwise cross-drug checker.
   */
  @column()
  declare drug_interactions: string | null
  /** Flattened contraindications sections. */
  @column()
  declare contraindications: string | null
  /** Flattened when_using sections (common on OTC labels). */
  @column()
  declare when_using: string | null
  /** Flattened stop_use sections (common on OTC labels). */
  @column()
  declare stop_use: string | null
  /**
   * Parsed from effective_time (YYYYMMDD → YYYY-MM-DD).
   * Stored as a plain date string, not a Luxon DateTime, because
   * the source format is just a date (no time component).
   */
  @column()
  declare source_updated_at: string | null
  @column.dateTime({ autoCreate: true, autoUpdate: true, columnName: 'ingested_at' })
  declare ingested_at: DateTime
 }
--- a/admin/app/models/installed_resource.ts
+++ b/admin/app/models/installed_resource.ts
@ -1,54 +0,0 @@
 import { DateTime } from 'luxon'
 import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
 export default class InstalledResource extends BaseModel {
  static namingStrategy = new SnakeCaseNamingStrategy()
  @column({ isPrimary: true })
  declare id: number
  @column()
  declare resource_id: string
  @column()
  declare resource_type: 'zim' | 'map' | 'dataset'
  @column()
  declare collection_ref: string | null
  @column()
  declare version: string
  @column()
  declare url: string
  @column()
  declare file_path: string
  @column()
  declare file_size_bytes: number | null
  @column.dateTime()
  declare installed_at: DateTime
  // ── Content auto-update state (global opt-in; gated by `contentAutoUpdate.enabled`) ──
  /** Newest catalog version (YYYY-MM) detected, or null when already current. */
  @column()
  declare available_update_version: string | null
  /** Size (bytes) of the available update, captured from the catalog. */
  @column()
  declare available_update_size_bytes: number | null
  /** Cool-off anchor: when the current available update was first detected. */
  @column.dateTime()
  declare available_update_first_seen_at: DateTime | null
  /** Per-resource failure backoff so one flapping download self-disables. */
  @column()
  declare auto_update_consecutive_failures: number
  @column()
  declare auto_update_disabled_reason: string | null
 }
--- a/admin/app/models/kb_ingest_state.ts
+++ b/admin/app/models/kb_ingest_state.ts
@ -1,81 +0,0 @@
 import { DateTime } from 'luxon'
 import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
 import type { KbIngestStateValue } from '../../types/kb_ingest_state.js'
 const LAST_ERROR_MAX_LEN = 1024
 /**
 * Tracks the per-file decision and outcome of AI knowledge-base ingestion.
 *
 * The row exists for any embeddable file the scanner has seen and is independent
 * of `installed_resources` (which only covers curated downloads). Replaces the
 * earlier "any chunks in qdrant ⇒ embedded" binary check, which conflated
 * partially-stalled ingestions with fully-indexed files. See RFC #883.
 */
 export default class KbIngestState extends BaseModel {
  static table = 'kb_ingest_state'
  static namingStrategy = new SnakeCaseNamingStrategy()
  @column({ isPrimary: true })
  declare id: number
  @column()
  declare file_path: string
  @column()
  declare state: KbIngestStateValue
  @column()
  declare chunks_embedded: number
  @column()
  declare collection: string | null
  @column()
  declare last_error: string | null
  @column.dateTime({ autoCreate: true })
  declare created_at: DateTime
  @column.dateTime({ autoCreate: true, autoUpdate: true })
  declare updated_at: DateTime
  static async getOrCreate(filePath: string, collection?: string): Promise<KbIngestState> {
    return this.firstOrCreate(
      { file_path: filePath },
      { file_path: filePath, state: 'pending_decision', chunks_embedded: 0, collection: collection ?? null }
    )
  }
  static async markIndexed(filePath: string, chunksEmbedded: number, collection?: string): Promise<void> {
    const row = await this.getOrCreate(filePath, collection)
    row.state = 'indexed'
    row.chunks_embedded = chunksEmbedded
    row.last_error = null
    if (collection) row.collection = collection
    await row.save()
  }
  static async markFailed(filePath: string, errorMessage: string): Promise<void> {
    const row = await this.getOrCreate(filePath)
    row.state = 'failed'
    row.last_error = errorMessage.slice(0, LAST_ERROR_MAX_LEN)
    await row.save()
  }
  static async markBrowseOnly(filePath: string): Promise<void> {
    const row = await this.getOrCreate(filePath)
    row.state = 'browse_only'
    await row.save()
  }
  static async markStalled(filePath: string): Promise<void> {
    const row = await this.getOrCreate(filePath)
    row.state = 'stalled'
    await row.save()
  }
  static async remove(filePath: string): Promise<void> {
    await this.query().where('file_path', filePath).delete()
  }
 }
--- a/admin/app/models/kb_ratio_registry.ts
+++ b/admin/app/models/kb_ratio_registry.ts
@ -1,79 +0,0 @@
 import { DateTime } from 'luxon'
 import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
 import {
  findChunksPerMb,
  estimateChunkCount,
  estimateBatch,
  type BatchEstimate,
  type BatchEstimateInput,
 } from '../utils/kb_ratio_lookup.js'
 /**
 * Self-calibrating registry of `{filename-prefix → chunks_per_mb}` ratios used
 * for disk-footprint and time-to-embed estimates surfaced in the KB panel.
 *
 * Migration seeds the registry with heuristic defaults from the RFC #883
 * appendix; Phase 4 self-calibration will update rows in place as ZIMs finish
 * ingesting and the real ratio becomes known. Lookup is longest-prefix-match
 * (see `kb_ratio_lookup.ts`) so a specific entry (`wikipedia_en_simple_`)
 * overrides a broader one (`wikipedia_en_`).
 */
 export default class KbRatioRegistry extends BaseModel {
  static table = 'kb_ratio_registry'
  static namingStrategy = new SnakeCaseNamingStrategy()
  @column({ isPrimary: true })
  declare id: number
  @column()
  declare pattern: string
  @column()
  declare chunks_per_mb: number
  @column()
  declare sample_count: number
  @column()
  declare notes: string | null
  @column.dateTime({ autoCreate: true })
  declare created_at: DateTime
  @column.dateTime({ autoCreate: true, autoUpdate: true })
  declare updated_at: DateTime
  /** Look up chunks_per_mb for a filename by longest-prefix match. */
  static async lookup(filename: string): Promise<number | null> {
    const rows = await this.all()
    return findChunksPerMb(filename, rows)
  }
  /**
   * Estimate total chunks for a file of the given size on disk.
   *
   * `ignoreCatchAll` excludes the empty-pattern fallback, returning `null` for
   * filenames that only the catch-all would match. The partial_stall warning
   * uses this so it never flags ZIMs the registry can't specifically
   * characterize (e.g. PDF/link-out-heavy archives whose byte size wildly
   * over-predicts embeddable chunks). See #913.
   */
  static async estimateChunks(
    filename: string,
    fileSizeBytes: number,
    opts: { ignoreCatchAll?: boolean } = {}
  ): Promise<number | null> {
    const rows = await this.all()
    return estimateChunkCount(filename, fileSizeBytes, rows, opts)
  }
  /**
   * Aggregate an embedding-disk-cost estimate across a batch of files. Used by
   * the curated-tier-change UI to show "you're about to add ~X GB of
   * embeddings on top of the ZIM downloads" before the user commits.
   */
  static async estimateBatch(files: BatchEstimateInput[]): Promise<BatchEstimate> {
    const rows = await this.all()
    return estimateBatch(files, rows)
  }
 }
--- a/admin/app/models/kv_store.ts
+++ b/admin/app/models/kv_store.ts
@ -1,64 +0,0 @@
 import { DateTime } from 'luxon'
 import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
 import { KV_STORE_SCHEMA, type KVStoreKey, type KVStoreValue } from '../../types/kv_store.js'
 import { parseBoolean } from '../utils/misc.js'
 /**
 * Generic key-value store model for storing various settings
 * that don't necessitate their own dedicated models.
 */
 export default class KVStore extends BaseModel {
  static table = 'kv_store'
  static namingStrategy = new SnakeCaseNamingStrategy()
  @column({ isPrimary: true })
  declare id: number
  @column()
  declare key: KVStoreKey
  @column()
  declare value: string | null
  @column.dateTime({ autoCreate: true })
  declare created_at: DateTime
  @column.dateTime({ autoCreate: true, autoUpdate: true })
  declare updated_at: DateTime
  /**
   * Get a setting value by key, automatically deserializing to the correct type.
   */
  static async getValue<K extends KVStoreKey>(key: K): Promise<KVStoreValue<K> | null> {
    const setting = await this.findBy('key', key)
    if (!setting || setting.value === undefined || setting.value === null) {
      return null
    }
    const raw = String(setting.value)
    return (KV_STORE_SCHEMA[key] === 'boolean' ? parseBoolean(raw) : raw) as KVStoreValue<K>
  }
  /**
   * Set a setting value by key (creates if not exists), automatically serializing to string.
   */
  static async setValue<K extends KVStoreKey>(key: K, value: KVStoreValue<K>): Promise<KVStore> {
    const serialized = String(value)
    const setting = await this.firstOrCreate({ key }, { key, value: serialized })
    if (setting.value !== serialized) {
      setting.value = serialized
      await setting.save()
    }
    return setting
  }
  /**
   * Clear a setting value by key, storing null so getValue returns null.
   */
  static async clearValue<K extends KVStoreKey>(key: K): Promise<void> {
    const setting = await this.findBy('key', key)
    if (setting && setting.value !== null) {
      setting.value = null
      await setting.save()
    }
  }
 }
--- a/admin/app/models/map_marker.ts
+++ b/admin/app/models/map_marker.ts
@ -1,43 +0,0 @@
 import { DateTime } from 'luxon'
 import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
 export default class MapMarker extends BaseModel {
  static namingStrategy = new SnakeCaseNamingStrategy()
  @column({ isPrimary: true })
  declare id: number
  @column()
  declare name: string
  @column()
  declare longitude: number
  @column()
  declare latitude: number
  @column()
  declare color: string
  // 'pin' for user-placed markers, 'waypoint' for route points (future)
  @column()
  declare marker_type: string
  // Groups markers into a route (future)
  @column()
  declare route_id: string | null
  // Order within a route (future)
  @column()
  declare route_order: number | null
  // Optional user notes for a location
  @column()
  declare notes: string | null
  @column.dateTime({ autoCreate: true })
  declare created_at: DateTime
  @column.dateTime({ autoCreate: true, autoUpdate: true })
  declare updated_at: DateTime
 }
--- a/admin/app/models/service.ts
+++ b/admin/app/models/service.ts
@ -26,12 +26,6 @@ export default class Service extends BaseModel {
  @column()
  declare description: string | null
  @column()
  declare powered_by: string | null
  @column()
  declare display_order: number | null
  @column()
  declare icon: string | null // must be a TablerIcons name to be properly rendered in the UI (e.g. "IconBrandDocker")
@ -59,73 +53,9 @@ export default class Service extends BaseModel {
  @column()
  declare ui_location: string | null
  // User-set override for the launch ("Open") link (e.g. a reverse-proxy/local-DNS host like
  // https://jellyfin.myhomelab.net). When null, the default host + port link derived from
  // ui_location is used. Only affects user-facing links — never internal service-to-service URLs.
  @column()
  declare custom_url: string | null
  @column()
  declare metadata: string | null
  @column({
    serialize(value) {
      return Boolean(value)
    },
  })
  declare is_custom: boolean
  @column({
    serialize(value) {
      return Boolean(value)
    },
  })
  declare is_user_modified: boolean
  @column()
  declare category: string | null
  // When true the service is sunset: hidden from the install catalog unless it is already
  // installed (see SystemService.getServices). Lets a deprecated app stay manageable for users who
  // still run it while keeping new users from installing it.
  @column({
    serialize(value) {
      return Boolean(value)
    },
  })
  declare is_deprecated: boolean
  @column()
  declare source_repo: string | null
  @column()
  declare available_update_version: string | null
  @column.dateTime()
  declare update_checked_at: DateTime | null
  // Per-app opt-in for automatic updates. An app auto-updates only when both this
  // and the global `appAutoUpdate.enabled` master switch are on.
  @column({
    serialize(value) {
      return Boolean(value)
    },
  })
  declare auto_update_enabled: boolean
  // When the current `available_update_version` was first detected — the anchor for
  // the auto-update cool-off (registry tags carry no publish timestamp).
  @column.dateTime()
  declare available_update_first_seen_at: DateTime | null
  // Per-app auto-update failure backoff; at the threshold the app self-disables via
  // `auto_update_disabled_reason` without affecting other apps.
  @column()
  declare auto_update_consecutive_failures: number
  @column()
  declare auto_update_disabled_reason: string | null
  @column.dateTime({ autoCreate: true })
  declare created_at: DateTime
--- a/admin/app/models/wikipedia_selection.ts
+++ b/admin/app/models/wikipedia_selection.ts
@ -1,27 +0,0 @@
 import { DateTime } from 'luxon'
 import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
 export default class WikipediaSelection extends BaseModel {
  static namingStrategy = new SnakeCaseNamingStrategy()
  @column({ isPrimary: true })
  declare id: number
  @column()
  declare option_id: string
  @column()
  declare url: string | null
  @column()
  declare filename: string | null
  @column()
  declare status: 'none' | 'downloading' | 'installed' | 'failed'
  @column.dateTime({ autoCreate: true })
  declare created_at: DateTime
  @column.dateTime({ autoCreate: true, autoUpdate: true })
  declare updated_at: DateTime
 }
--- a/admin/app/services/app_auto_update_service.ts
+++ b/admin/app/services/app_auto_update_service.ts
@ -1,398 +0,0 @@
 import { inject } from '@adonisjs/core'
 import logger from '@adonisjs/core/services/logger'
 import { DateTime } from 'luxon'
 import KVStore from '#models/kv_store'
 import Service from '#models/service'
 import { DockerService } from '#services/docker_service'
 import { DownloadService } from '#services/download_service'
 import { SystemService } from '#services/system_service'
 import { ContainerRegistryService } from '#services/container_registry_service'
 import { isNewerVersion, parseMajorVersion } from '../utils/version.js'
 import { isWithinWindow } from '../utils/update_window.js'
 import {
  checkImageDiskSpace,
  type Blocker,
  type PreflightResult,
 } from '../utils/image_disk_preflight.js'
 /**
 * Defaults shared with the core auto-update. App auto-updates intentionally reuse
 * the SAME window/cool-off settings (`autoUpdate.windowStart/windowEnd/cooloffHours`);
 * only the enable flag (`appAutoUpdate.enabled`) is separate.
 */
 const DEFAULT_WINDOW_START = '02:00'
 const DEFAULT_WINDOW_END = '05:00'
 const DEFAULT_COOLOFF_HOURS = 72
 /** Per-app genuine failures before that app self-disables (others keep running). */
 const MAX_CONSECUTIVE_FAILURES = 3
 export interface AppAutoUpdateConfig {
  /** Global master switch (`appAutoUpdate.enabled`). */
  enabled: boolean
  windowStart: string
  windowEnd: string
  cooloffHours: number
 }
 /** An installed app that should be auto-updated this run. */
 export interface AppUpdateTarget {
  service: Service
  /** Exact registry tag to update to (the value in `available_update_version`). */
  targetVersion: string
 }
 /** Per-app eligibility verdict (drives both selection and the status UI). */
 export interface AppEligibility {
  eligible: boolean
  reason: string
  cooloffRemainingHours: number | null
 }
 export interface AppAutoUpdateAppStatus {
  service_name: string
  friendly_name: string | null
  auto_update_enabled: boolean
  current_version: string
  available_update_version: string | null
  first_seen_at: string | null
  eligible: boolean
  reason: string
  cooloff_remaining_hours: number | null
  consecutive_failures: number
  auto_disabled_reason: string | null
 }
 export interface AppAutoUpdateStatus extends AppAutoUpdateConfig {
  withinWindow: boolean
  lastAttemptAt: string | null
  lastResult: string | null
  apps: AppAutoUpdateAppStatus[]
 }
 /**
 * Decision + safety layer for automatic updates of installed sibling apps (the
 * containers NOMAD deploys via the Docker socket and manages in Supply Depot).
 *
 * This is the app-side counterpart to {@link AutoUpdateService} and intentionally
 * reuses its generic window/disk pre-flight helpers. Unlike the core update, an
 * app update needs no sidecar — the admin container recreates its siblings directly
 * via {@link DockerService.updateContainer} (in-process pull → rename → health-check
 * → rollback). Auto-update only decides *whether* each opted-in app should update now
 * (master switch on + per-app toggle on + in window + an eligible minor/patch past
 * its cool-off + pre-flight passes) and then drives the existing update path.
 *
 * Minor/patch-only is already guaranteed upstream by
 * {@link ContainerRegistryService.getAvailableUpdates} (same-major filter); the
 * major-version check here is defense-in-depth.
 */
@inject()
 export class AppAutoUpdateService {
  constructor(
    private dockerService: DockerService,
    private downloadService: DownloadService,
    private systemService: SystemService,
    private containerRegistryService: ContainerRegistryService
  ) {}
  /** Read the global master switch plus the shared window/cool-off settings. */
  async getConfig(): Promise<AppAutoUpdateConfig> {
    const [enabled, windowStart, windowEnd, cooloffHours] = await Promise.all([
      KVStore.getValue('appAutoUpdate.enabled'),
      KVStore.getValue('autoUpdate.windowStart'),
      KVStore.getValue('autoUpdate.windowEnd'),
      KVStore.getValue('autoUpdate.cooloffHours'),
    ])
    const parsedCooloff = Number(cooloffHours)
    return {
      enabled: enabled ?? false,
      windowStart: windowStart || DEFAULT_WINDOW_START,
      windowEnd: windowEnd || DEFAULT_WINDOW_END,
      // `Number(null) === 0`, so an unset value must fall through to the default
      // rather than silently resolving to a zero cool-off. An explicit 0 is honored.
      cooloffHours:
        cooloffHours !== null && Number.isFinite(parsedCooloff) && parsedCooloff >= 0
          ? parsedCooloff
          : DEFAULT_COOLOFF_HOURS,
    }
  }
  /**
   * Pure per-app eligibility verdict. An app is eligible when it has a detected
   * update that is the same major (defense-in-depth), strictly newer, not self-
   * disabled, and past its cool-off (measured from first-detected).
   */
  appEligibility(service: Service, cooloffHours: number, now: DateTime): AppEligibility {
    if (!service.available_update_version) {
      return { eligible: false, reason: 'Up to date', cooloffRemainingHours: null }
    }
    if (service.auto_update_disabled_reason) {
      return {
        eligible: false,
        reason: 'Auto-update disabled after repeated failures',
        cooloffRemainingHours: null,
      }
    }
    const currentTag = this.containerRegistryService.parseImageReference(
      service.container_image
    ).tag
    if (currentTag === 'latest') {
      return {
        eligible: false,
        reason: 'Pinned to :latest — cannot version-check',
        cooloffRemainingHours: null,
      }
    }
    if (parseMajorVersion(service.available_update_version) !== parseMajorVersion(currentTag)) {
      return {
        eligible: false,
        reason: 'Major version — manual update required',
        cooloffRemainingHours: null,
      }
    }
    if (!isNewerVersion(service.available_update_version, currentTag)) {
      return { eligible: false, reason: 'Up to date', cooloffRemainingHours: null }
    }
    if (!service.available_update_first_seen_at) {
      return { eligible: false, reason: 'Cool-off pending', cooloffRemainingHours: cooloffHours }
    }
    const ageHours = now.diff(service.available_update_first_seen_at, 'hours').hours
    const remaining = cooloffHours - ageHours
    if (remaining > 0) {
      const rounded = Math.ceil(remaining)
      return {
        eligible: false,
        reason: `In cool-off (${rounded}h remaining)`,
        cooloffRemainingHours: rounded,
      }
    }
    return {
      eligible: true,
      reason: `Eligible → ${service.available_update_version}`,
      cooloffRemainingHours: 0,
    }
  }
  /** Installed, opted-in apps that are eligible to update right now. */
  async getEligibleApps(config: AppAutoUpdateConfig, now: DateTime): Promise<AppUpdateTarget[]> {
    const apps = await Service.query().where('installed', true).where('auto_update_enabled', true)
    const targets: AppUpdateTarget[] = []
    for (const service of apps) {
      const verdict = this.appEligibility(service, config.cooloffHours, now)
      if (verdict.eligible) {
        targets.push({ service, targetVersion: service.available_update_version! })
      }
    }
    return targets
  }
  /**
   * Run-wide pre-flight checked once per attempt (independent of any single app):
   * never auto-update while content/model downloads are running. Transient → `skip`.
   */
  async runGlobalPreflight(): Promise<PreflightResult> {
    const blockers: Blocker[] = []
    try {
      const downloads = await this.downloadService.listDownloadJobs()
      const active = downloads.filter(
        (d) => !!d.status && ['waiting', 'active', 'delayed'].includes(d.status)
      )
      if (active.length > 0) {
        blockers.push({ reason: `${active.length} download(s) in progress`, severity: 'skip' })
      }
    } catch (error) {
      logger.warn(`[AppAutoUpdateService] Could not check active downloads: ${error.message}`)
    }
    return { ok: blockers.length === 0, blockers }
  }
  /** Per-app pre-flight: not already mid-operation (`skip`) and enough disk (`failure`). */
  async runAppPreflight(target: AppUpdateTarget): Promise<PreflightResult> {
    const blockers: Blocker[] = []
    const service = target.service
    if (service.installation_status !== 'idle') {
      blockers.push({
        reason: `App has an operation in progress (status: ${service.installation_status})`,
        severity: 'skip',
      })
    }
    const hostArch = await this.getHostArch()
    const targetImage = `${this.imageBase(service.container_image)}:${target.targetVersion}`
    const diskBlocker = await checkImageDiskSpace({
      image: targetImage,
      hostArch,
      containerRegistryService: this.containerRegistryService,
      systemService: this.systemService,
    })
    if (diskBlocker) blockers.push(diskBlocker)
    return { ok: blockers.length === 0, blockers }
  }
  /**
   * Entry point invoked by AppAutoUpdateJob. Gates on the master switch + window,
   * then runs each eligible app through pre-flight and {@link DockerService.updateContainer}.
   * A failing app self-disables after repeated failures without affecting the others.
   */
  async attempt(): Promise<{ updated: number; reason: string }> {
    const config = await this.getConfig()
    const now = DateTime.now()
    if (!config.enabled) {
      return { updated: 0, reason: 'App auto-update is disabled' }
    }
    if (!isWithinWindow(config.windowStart, config.windowEnd, now)) {
      const reason = `Outside update window (${config.windowStart}-${config.windowEnd})`
      await this.recordRun(reason)
      return { updated: 0, reason }
    }
    const eligible = await this.getEligibleApps(config, now)
    if (eligible.length === 0) {
      const reason = 'No eligible app updates (all current, in cool-off, or major-only)'
      await this.recordRun(reason)
      return { updated: 0, reason }
    }
    const global = await this.runGlobalPreflight()
    if (!global.ok) {
      const reason = `Pre-flight blocked: ${global.blockers.map((b) => b.reason).join('; ')}`
      await this.recordRun(reason)
      return { updated: 0, reason }
    }
    let updated = 0
    let failed = 0
    let skipped = 0
    for (const target of eligible) {
      const name = target.service.service_name
      const preflight = await this.runAppPreflight(target)
      if (!preflight.ok) {
        const summary = preflight.blockers.map((b) => b.reason).join('; ')
        if (preflight.blockers.some((b) => b.severity === 'failure')) {
          await this.recordAppFailure(target.service, summary)
          failed++
        } else {
          logger.info(`[AppAutoUpdateService] Skipped ${name}: ${summary}`)
          skipped++
        }
        continue
      }
      logger.info(`[AppAutoUpdateService] Updating ${name} → ${target.targetVersion}`)
      const result = await this.dockerService.updateContainer(name, target.targetVersion)
      if (result.success) {
        await this.recordAppSuccess(target.service)
        updated++
      } else {
        await this.recordAppFailure(target.service, result.message)
        failed++
      }
    }
    const reason = `${updated} updated, ${failed} failed, ${skipped} skipped`
    await this.recordRun(reason)
    logger.info(`[AppAutoUpdateService] Run complete: ${reason}`)
    return { updated, reason }
  }
  /** Clear an app's failure backoff after a successful auto-update. */
  private async recordAppSuccess(service: Service): Promise<void> {
    // updateContainer already advanced container_image and cleared
    // available_update_version on its own (fresh) row; here we only touch the
    // backoff fields, so Lucid persists just those dirty columns.
    service.auto_update_consecutive_failures = 0
    service.auto_update_disabled_reason = null
    await service.save()
  }
  /** Record an app failure and self-disable it once the threshold is reached. */
  private async recordAppFailure(service: Service, reason: string): Promise<void> {
    const failures = (service.auto_update_consecutive_failures || 0) + 1
    service.auto_update_consecutive_failures = failures
    if (failures >= MAX_CONSECUTIVE_FAILURES) {
      service.auto_update_disabled_reason = `Auto-update disabled after ${failures} consecutive failures. Last error: ${reason}`
      logger.error(
        `[AppAutoUpdateService] ${service.service_name} auto-disabled after ${failures} failures`
      )
    }
    await service.save()
    logger.error(
      `[AppAutoUpdateService] ${service.service_name} failure ${failures}/${MAX_CONSECUTIVE_FAILURES}: ${reason}`
    )
  }
  /** Record the global last-attempt summary for the settings UI. */
  private async recordRun(reason: string): Promise<void> {
    await KVStore.setValue('appAutoUpdate.lastAttemptAt', DateTime.now().toISO()!)
    await KVStore.setValue('appAutoUpdate.lastResult', reason)
  }
  /** Full state snapshot for the settings UI (opted-in apps + their eligibility). */
  async getStatus(): Promise<AppAutoUpdateStatus> {
    const config = await this.getConfig()
    const now = DateTime.now()
    const apps = await Service.query().where('installed', true).where('auto_update_enabled', true)
    const appStatuses: AppAutoUpdateAppStatus[] = apps.map((service) => {
      const verdict = this.appEligibility(service, config.cooloffHours, now)
      return {
        service_name: service.service_name,
        friendly_name: service.friendly_name,
        auto_update_enabled: service.auto_update_enabled,
        current_version: this.containerRegistryService.parseImageReference(service.container_image)
          .tag,
        available_update_version: service.available_update_version,
        first_seen_at: service.available_update_first_seen_at?.toISO() ?? null,
        eligible: verdict.eligible,
        reason: verdict.reason,
        cooloff_remaining_hours: verdict.cooloffRemainingHours,
        consecutive_failures: service.auto_update_consecutive_failures || 0,
        auto_disabled_reason: service.auto_update_disabled_reason,
      }
    })
    const [lastAttemptAt, lastResult] = await Promise.all([
      KVStore.getValue('appAutoUpdate.lastAttemptAt'),
      KVStore.getValue('appAutoUpdate.lastResult'),
    ])
    return {
      ...config,
      withinWindow: isWithinWindow(config.windowStart, config.windowEnd, now),
      lastAttemptAt: lastAttemptAt || null,
      lastResult: lastResult || null,
      apps: appStatuses,
    }
  }
  /** Strip the tag from an image reference, leaving "registry/namespace/repo". */
  private imageBase(image: string): string {
    return image.includes(':') ? image.substring(0, image.lastIndexOf(':')) : image
  }
  /** Map the Docker daemon's architecture string to OCI naming (amd64/arm64/...). */
  private async getHostArch(): Promise<string> {
    try {
      const info = await this.dockerService.docker.info()
      const arch = info.Architecture || ''
      const archMap: Record<string, string> = {
        x86_64: 'amd64',
        aarch64: 'arm64',
        armv7l: 'arm',
        amd64: 'amd64',
        arm64: 'arm64',
      }
      return archMap[arch] || arch.toLowerCase()
    } catch {
      return 'amd64'
    }
  }
 }
--- a/admin/app/services/auto_update_service.ts
+++ b/admin/app/services/auto_update_service.ts
@ -1,580 +0,0 @@
 import { inject } from '@adonisjs/core'
 import logger from '@adonisjs/core/services/logger'
 import axios from 'axios'
 import { DateTime } from 'luxon'
 import KVStore from '#models/kv_store'
 import Service from '#models/service'
 import { DockerService } from '#services/docker_service'
 import { DownloadService } from '#services/download_service'
 import { SystemService } from '#services/system_service'
 import { SystemUpdateService } from '#services/system_update_service'
 import { ContainerRegistryService } from '#services/container_registry_service'
 import { isNewerVersion, parseMajorVersion } from '../utils/version.js'
 import { isWithinWindow as isWithinWindowUtil } from '../utils/update_window.js'
 import {
  checkImageDiskSpace,
  type Blocker,
  type PreflightResult,
 } from '../utils/image_disk_preflight.js'
 /** Docker image repository for the NOMAD admin/core image (tag applied per-release). */
 const NOMAD_IMAGE_REPO = 'ghcr.io/crosstalk-solutions/project-nomad'
 const RELEASES_URL = 'https://api.github.com/repos/Crosstalk-Solutions/project-nomad/releases'
 /** Defaults for user-configurable settings (server-local time window + cool-off). */
 const DEFAULT_WINDOW_START = '02:00'
 const DEFAULT_WINDOW_END = '05:00'
 const DEFAULT_COOLOFF_HOURS = 72
 /** Genuine failures before auto-update disables itself to avoid an update loop. */
 const MAX_CONSECUTIVE_FAILURES = 3
 /**
 * Only tags matching strict semver are eligible. Defense-in-depth: the selected
 * tag becomes `target_tag`, which the sidecar interpolates into a host-side `sed`
 * (install/sidecar-updater/update-watcher.sh) — so a malformed tag must never be
 * able to reach it, even though releases come from a trusted repo.
 */
 const SEMVER_TAG = /^\d+\.\d+\.\d+$/
 /** Cache the GitHub releases feed in-process to avoid hammering the API. */
 const RELEASES_CACHE_TTL_MS = 15 * 60 * 1000
 /** Briefly remember a failed fetch so repeated calls don't each block on the timeout. */
 const RELEASES_FAILURE_TTL_MS = 60 * 1000
 export interface AutoUpdateConfig {
  enabled: boolean
  windowStart: string
  windowEnd: string
  cooloffHours: number
 }
 export interface EligibleTarget {
  version: string
  tag: string
  publishedAt: string
 }
 // Pre-flight types/primitives are shared with AppAutoUpdateService; re-exported
 // here for back-compat with existing imports of this module.
 export type { Blocker, BlockerSeverity, PreflightResult } from '../utils/image_disk_preflight.js'
 /** Minimal shape of a GitHub release entry we depend on. */
 export interface GithubRelease {
  tag_name?: string
  published_at?: string
  draft?: boolean
  prerelease?: boolean
 }
 /**
 * Inputs that can be injected to exercise the decision pipeline deterministically
 * (used by the dry-run command/tests). All are optional; when omitted the real
 * settings/clock/GitHub feed/pre-flight are used, exactly as production runs.
 */
 export interface EvaluateOverrides {
  currentVersion?: string
  releases?: GithubRelease[]
  now?: DateTime
  forceEnabled?: boolean
  windowStart?: string
  windowEnd?: string
  cooloffHours?: number
  /** Treat pre-flight as passing without touching Docker/disk/queues. */
  skipPreflight?: boolean
  /** Substitute a canned pre-flight result. */
  fakePreflight?: PreflightResult
 }
 export type DecisionOutcome =
  | 'disabled'
  | 'outside-window'
  | 'eligibility-error'
  | 'no-eligible'
  | 'blocked'
  | 'ready'
 /** Side-effect-free verdict of the decision pipeline. */
 export interface AutoUpdateDecision {
  enabled: boolean
  currentVersion: string
  config: AutoUpdateConfig
  withinWindow: boolean
  eligibleTarget: EligibleTarget | null
  preflight: PreflightResult | null
  outcome: DecisionOutcome
  reason: string
 }
 export interface AutoUpdateStatus extends AutoUpdateConfig {
  currentVersion: string
  withinWindow: boolean
  eligibleTarget: EligibleTarget | null
  lastAttemptAt: string | null
  lastResult: string | null
  lastError: string | null
  consecutiveFailures: number
  autoDisabledReason: string | null
 }
 /**
 * Decision + safety layer for automatic updates of the NOMAD application itself.
 *
 * It does NOT recreate containers — that remains the sidecar's job. This service
 * decides *whether* an update should run right now (opt-in, in-window, an eligible
 * minor/patch release exists past its cool-off, pre-flight checks pass) and, if so,
 * drives the existing {@link SystemUpdateService.requestUpdate} with an explicit,
 * eligibility-vetted image tag.
 *
 * The window/pre-flight helpers are intentionally generic so a future PR can reuse
 * them to auto-update installed apps (driving DockerService.updateContainer instead).
 */
@inject()
 export class AutoUpdateService {
  constructor(
    private dockerService: DockerService,
    private downloadService: DownloadService,
    private systemService: SystemService,
    private systemUpdateService: SystemUpdateService,
    private containerRegistryService: ContainerRegistryService
  ) {}
  /** In-process cache of the last successful releases fetch (per-process). */
  private static releasesCache: { releases: GithubRelease[]; at: number } | null = null
  /** Timestamp of the last failed fetch, for short-lived negative caching. */
  private static releasesFailureAt = 0
  /** Read user-configurable settings, applying defaults. */
  async getConfig(): Promise<AutoUpdateConfig> {
    const [enabled, windowStart, windowEnd, cooloffHours] = await Promise.all([
      KVStore.getValue('autoUpdate.enabled'),
      KVStore.getValue('autoUpdate.windowStart'),
      KVStore.getValue('autoUpdate.windowEnd'),
      KVStore.getValue('autoUpdate.cooloffHours'),
    ])
    const parsedCooloff = Number(cooloffHours)
    return {
      enabled: enabled ?? false,
      windowStart: windowStart || DEFAULT_WINDOW_START,
      windowEnd: windowEnd || DEFAULT_WINDOW_END,
      // `Number(null) === 0`, so an *unset* value must fall through to the default
      // rather than silently resolving to a zero cool-off. An explicit 0 is honored.
      cooloffHours:
        cooloffHours != null && Number.isFinite(parsedCooloff) && parsedCooloff >= 0
          ? parsedCooloff
          : DEFAULT_COOLOFF_HOURS,
    }
  }
  /**
   * Determine whether `now` falls inside the configured update window. The window
   * is interpreted in the container's local time (set via the TZ env var). Windows
   * that wrap past midnight (start > end, e.g. 22:00-02:00) are handled.
   */
  isWithinWindow(config: AutoUpdateConfig, now: DateTime = DateTime.now()): boolean {
    return isWithinWindowUtil(config.windowStart, config.windowEnd, now)
  }
  /**
   * Find the newest release that is safe to auto-apply: same major version as the
   * running build (major bumps are deliberately left for manual update), strictly
   * newer than current, and published at least `cooloffHours` ago. Prereleases and
   * drafts are ignored — auto-update never rides early access.
   *
   * Returns null when nothing qualifies (e.g. only a major bump is newer, or the
   * newest eligible release is still inside its cool-off window).
   */
  async getEligibleTarget(config: AutoUpdateConfig): Promise<EligibleTarget | null> {
    const releases = await this.fetchReleases()
    return this.selectEligibleTarget(
      releases,
      SystemService.getAppVersion(),
      config.cooloffHours,
      DateTime.now()
    )
  }
  /**
   * Fetch the published GitHub releases for the NOMAD repo, cached in-process.
   * A successful result is reused for {@link RELEASES_CACHE_TTL_MS} so repeated
   * status-page loads don't each hit (and risk rate-limiting) the unauthenticated
   * GitHub API. A recent failure is negatively cached for {@link RELEASES_FAILURE_TTL_MS}
   * so back-to-back calls while offline don't each block on the request timeout.
   */
  async fetchReleases(): Promise<GithubRelease[]> {
    const now = Date.now()
    const cached = AutoUpdateService.releasesCache
    if (cached && now - cached.at < RELEASES_CACHE_TTL_MS) {
      return cached.releases
    }
    if (now - AutoUpdateService.releasesFailureAt < RELEASES_FAILURE_TTL_MS) {
      throw new Error('GitHub releases fetch recently failed; backing off')
    }
    try {
      const response = await axios.get(RELEASES_URL, {
        headers: { Accept: 'application/vnd.github+json' },
        timeout: 5000,
      })
      if (!Array.isArray(response.data)) {
        throw new Error('Unexpected response from GitHub releases API')
      }
      AutoUpdateService.releasesCache = { releases: response.data, at: now }
      return response.data
    } catch (error) {
      AutoUpdateService.releasesFailureAt = now
      throw error
    }
  }
  /**
   * Pure selection of the newest auto-applicable release from a release list.
   * Extracted so the dry-run command and tests can drive it with fixtures.
   * Same major as `currentVersion`, strictly newer, published on/before
   * `now - cooloffHours`, prereleases/drafts excluded. Returns null for dev
   * builds or when nothing qualifies.
   */
  selectEligibleTarget(
    releases: GithubRelease[],
    currentVersion: string,
    cooloffHours: number,
    now: DateTime
  ): EligibleTarget | null {
    if (currentVersion === 'dev' || currentVersion === '0.0.0') {
      return null
    }
    const currentMajor = parseMajorVersion(currentVersion)
    const cutoff = now.minus({ hours: cooloffHours })
    const candidates = releases
      .filter((r) => r && !r.draft && !r.prerelease && r.tag_name && r.published_at)
      .map((r) => ({
        version: String(r.tag_name).replace(/^v/, '').trim(),
        publishedAt: String(r.published_at),
      }))
      .filter((r) => SEMVER_TAG.test(r.version))
      .filter((r) => parseMajorVersion(r.version) === currentMajor)
      .filter((r) => isNewerVersion(r.version, currentVersion))
      .filter((r) => DateTime.fromISO(r.publishedAt) <= cutoff)
      .sort((a, b) => (isNewerVersion(a.version, b.version) ? -1 : 1))
    const best = candidates[0]
    if (!best) return null
    return {
      version: best.version,
      tag: `v${best.version}`,
      publishedAt: best.publishedAt,
    }
  }
  /**
   * Pre-flight checks that gate an auto-update. `skip` blockers are transient
   * (retry next window, no penalty); `failure` blockers count toward the backoff
   * that eventually auto-disables auto-update.
   */
  async runPreflight(targetTag: string): Promise<PreflightResult> {
    const blockers: Blocker[] = []
    // 1. Sidecar must be present to perform the update.
    if (!this.systemUpdateService.isSidecarAvailable()) {
      blockers.push({ reason: 'Update sidecar is not available', severity: 'failure' })
    }
    // 2. No system update already running.
    const updateStatus = this.systemUpdateService.getUpdateStatus()
    if (updateStatus && !['idle', 'complete', 'error'].includes(updateStatus.stage)) {
      blockers.push({
        reason: `A system update is already in progress (stage: ${updateStatus.stage})`,
        severity: 'skip',
      })
    }
    // 3. No content/model downloads in progress.
    try {
      const downloads = await this.downloadService.listDownloadJobs()
      const active = downloads.filter(
        (d) => !!d.status && ['waiting', 'active', 'delayed'].includes(d.status)
      )
      if (active.length > 0) {
        blockers.push({
          reason: `${active.length} download(s) in progress`,
          severity: 'skip',
        })
      }
    } catch (error) {
      logger.warn(`[AutoUpdateService] Could not check active downloads: ${error.message}`)
    }
    // 4. No app (container) install/update in progress.
    try {
      const installing = await Service.query().whereNot('installation_status', 'idle')
      if (installing.length > 0) {
        blockers.push({
          reason: `${installing.length} app install/update(s) in progress`,
          severity: 'skip',
        })
      }
    } catch (error) {
      logger.warn(`[AutoUpdateService] Could not check app installations: ${error.message}`)
    }
    // 5. Sufficient host storage for the new image.
    const diskBlocker = await this.checkDiskSpace(targetTag)
    if (diskBlocker) blockers.push(diskBlocker)
    return { ok: blockers.length === 0, blockers }
  }
  /** Returns a disk blocker if free space is insufficient, otherwise null. */
  private async checkDiskSpace(targetTag: string): Promise<Blocker | null> {
    const hostArch = await this.getHostArch()
    return checkImageDiskSpace({
      image: `${NOMAD_IMAGE_REPO}:${targetTag}`,
      hostArch,
      containerRegistryService: this.containerRegistryService,
      systemService: this.systemService,
    })
  }
  /** Map the Docker daemon's architecture string to OCI naming (amd64/arm64/...). */
  private async getHostArch(): Promise<string> {
    try {
      const info = await this.dockerService.docker.info()
      const arch = info.Architecture || ''
      const archMap: Record<string, string> = {
        x86_64: 'amd64',
        aarch64: 'arm64',
        armv7l: 'arm',
        amd64: 'amd64',
        arm64: 'arm64',
      }
      return archMap[arch] || arch.toLowerCase()
    } catch {
      return 'amd64'
    }
  }
  /**
   * Side-effect-free core of the decision pipeline. Resolves the effective config
   * (settings, overridable), checks the window, finds an eligible target, and runs
   * pre-flight — returning a verdict WITHOUT requesting an update or mutating any
   * persisted state. Both {@link attempt} (production) and {@link dryRun} (testing)
   * are built on this so a dry run faithfully reflects what a real run would do.
   */
  async evaluate(overrides: EvaluateOverrides = {}): Promise<AutoUpdateDecision> {
    const baseConfig = await this.getConfig()
    const config: AutoUpdateConfig = {
      enabled: overrides.forceEnabled ?? baseConfig.enabled,
      windowStart: overrides.windowStart ?? baseConfig.windowStart,
      windowEnd: overrides.windowEnd ?? baseConfig.windowEnd,
      cooloffHours: overrides.cooloffHours ?? baseConfig.cooloffHours,
    }
    const now = overrides.now ?? DateTime.now()
    const currentVersion = overrides.currentVersion ?? SystemService.getAppVersion()
    const base = {
      enabled: config.enabled,
      currentVersion,
      config,
      withinWindow: false,
      eligibleTarget: null as EligibleTarget | null,
      preflight: null as PreflightResult | null,
    }
    if (!config.enabled) {
      return { ...base, outcome: 'disabled', reason: 'Auto-update is disabled' }
    }
    const withinWindow = this.isWithinWindow(config, now)
    if (!withinWindow) {
      return {
        ...base,
        outcome: 'outside-window',
        reason: `Outside update window (${config.windowStart}-${config.windowEnd})`,
      }
    }
    let eligibleTarget: EligibleTarget | null
    try {
      const releases = overrides.releases ?? (await this.fetchReleases())
      eligibleTarget = this.selectEligibleTarget(releases, currentVersion, config.cooloffHours, now)
    } catch (error) {
      return {
        ...base,
        withinWindow,
        outcome: 'eligibility-error',
        reason: `Failed to determine eligible version: ${error.message}`,
      }
    }
    if (!eligibleTarget) {
      return {
        ...base,
        withinWindow,
        outcome: 'no-eligible',
        reason: 'No eligible minor/patch update available (or still in cool-off)',
      }
    }
    const preflight = overrides.fakePreflight
      ? overrides.fakePreflight
      : overrides.skipPreflight
        ? { ok: true, blockers: [] }
        : await this.runPreflight(eligibleTarget.tag)
    if (!preflight.ok) {
      const summary = preflight.blockers.map((b) => b.reason).join('; ')
      return {
        ...base,
        withinWindow,
        eligibleTarget,
        preflight,
        outcome: 'blocked',
        reason: `Pre-flight blocked: ${summary}`,
      }
    }
    return {
      ...base,
      withinWindow,
      eligibleTarget,
      preflight,
      outcome: 'ready',
      reason: `Ready to update to ${eligibleTarget.tag}`,
    }
  }
  /**
   * Run the full decision pipeline WITHOUT requesting an update or recording any
   * state. Accepts the same injectable overrides as {@link evaluate}, so callers can
   * simulate any scenario (a given current version, a canned release list, a fixed
   * clock, a forced window) and see exactly what a real run would decide.
   */
  async dryRun(overrides: EvaluateOverrides = {}): Promise<AutoUpdateDecision> {
    return this.evaluate(overrides)
  }
  /**
   * The entry point invoked by AutoUpdateJob. Evaluates the decision pipeline and,
   * when everything passes, requests the update with the vetted tag — recording the
   * outcome to the KVStore (for the UI) and applying failure backoff.
   */
  async attempt(): Promise<{ updated: boolean; reason: string }> {
    const decision = await this.evaluate()
    switch (decision.outcome) {
      case 'disabled':
        return { updated: false, reason: decision.reason }
      case 'outside-window':
      case 'no-eligible':
      // A failed release lookup is transient (offline-first appliances are
      // routinely without connectivity) — treat as a skip so it never trips the
      // backoff that auto-disables the feature. Only real update-request failures
      // (the `ready` case below) count toward MAX_CONSECUTIVE_FAILURES.
      case 'eligibility-error':
        await this.recordSkip(decision.reason)
        return { updated: false, reason: decision.reason }
      case 'blocked': {
        const hasFailure = decision.preflight!.blockers.some((b) => b.severity === 'failure')
        if (hasFailure) {
          await this.recordFailure(decision.reason)
        } else {
          await this.recordSkip(decision.reason)
        }
        return { updated: false, reason: decision.reason }
      }
      case 'ready': {
        const target = decision.eligibleTarget!
        const result = await this.systemUpdateService.requestUpdate({
          targetTag: target.tag,
          requester: 'auto-update',
        })
        if (result.success) {
          await this.recordSuccess(target)
          logger.info(`[AutoUpdateService] Auto-update requested: ${target.tag}`)
          return { updated: true, reason: `Update requested: ${target.tag}` }
        }
        await this.recordFailure(`Update request failed: ${result.message}`)
        return { updated: false, reason: result.message }
      }
    }
  }
  // --- Outcome recording -----------------------------------------------------
  private async recordSuccess(target: EligibleTarget): Promise<void> {
    await KVStore.setValue('autoUpdate.lastAttemptAt', DateTime.now().toISO()!)
    await KVStore.setValue('autoUpdate.lastResult', `Update requested: ${target.tag}`)
    await KVStore.clearValue('autoUpdate.lastError')
    await KVStore.setValue('autoUpdate.consecutiveFailures', '0')
  }
  private async recordSkip(reason: string): Promise<void> {
    await KVStore.setValue('autoUpdate.lastAttemptAt', DateTime.now().toISO()!)
    await KVStore.setValue('autoUpdate.lastResult', reason)
    logger.info(`[AutoUpdateService] Skipped: ${reason}`)
  }
  private async recordFailure(reason: string): Promise<void> {
    await KVStore.setValue('autoUpdate.lastAttemptAt', DateTime.now().toISO()!)
    await KVStore.setValue('autoUpdate.lastResult', reason)
    await KVStore.setValue('autoUpdate.lastError', reason)
    const prior = Number(await KVStore.getValue('autoUpdate.consecutiveFailures')) || 0
    const failures = prior + 1
    await KVStore.setValue('autoUpdate.consecutiveFailures', String(failures))
    logger.error(`[AutoUpdateService] Failure ${failures}/${MAX_CONSECUTIVE_FAILURES}: ${reason}`)
    if (failures >= MAX_CONSECUTIVE_FAILURES) {
      await KVStore.setValue('autoUpdate.enabled', false)
      await KVStore.setValue(
        'autoUpdate.autoDisabledReason',
        `Auto-update disabled after ${failures} consecutive failures. Last error: ${reason}`
      )
      logger.error(
        `[AutoUpdateService] Auto-update auto-disabled after ${failures} consecutive failures`
      )
    }
  }
  /** Full state snapshot for the settings UI. */
  async getStatus(): Promise<AutoUpdateStatus> {
    const config = await this.getConfig()
    const currentVersion = SystemService.getAppVersion()
    let eligibleTarget: EligibleTarget | null = null
    try {
      eligibleTarget = await this.getEligibleTarget(config)
    } catch (error) {
      logger.warn(`[AutoUpdateService] getStatus eligibility lookup failed: ${error.message}`)
    }
    const [lastAttemptAt, lastResult, lastError, consecutiveFailures, autoDisabledReason] =
      await Promise.all([
        KVStore.getValue('autoUpdate.lastAttemptAt'),
        KVStore.getValue('autoUpdate.lastResult'),
        KVStore.getValue('autoUpdate.lastError'),
        KVStore.getValue('autoUpdate.consecutiveFailures'),
        KVStore.getValue('autoUpdate.autoDisabledReason'),
      ])
    return {
      ...config,
      currentVersion,
      withinWindow: this.isWithinWindow(config),
      eligibleTarget,
      lastAttemptAt: lastAttemptAt || null,
      lastResult: lastResult || null,
      lastError: lastError || null,
      consecutiveFailures: Number(consecutiveFailures) || 0,
      autoDisabledReason: autoDisabledReason || null,
    }
  }
 }
--- a/admin/app/services/benchmark_service.ts
+++ b/admin/app/services/benchmark_service.ts
--- a/admin/app/services/benchmark_telemetry.ts
+++ b/admin/app/services/benchmark_telemetry.ts
@ -1,102 +0,0 @@
 import transmit from '@adonisjs/transmit/services/main'
 import si from 'systeminformation'
 import logger from '@adonisjs/core/services/logger'
 import { BROADCAST_CHANNELS } from '../../constants/broadcast.js'
 import type { BenchmarkStatus, BenchmarkTelemetry } from '../../types/benchmark.js'
 const SAMPLE_INTERVAL_MS = 1000
 /**
 * Samples host telemetry (per-core CPU load, temperature, disk throughput) on a
 * timer and broadcasts it over SSE while a benchmark runs. Read-only: the sampler
 * runs in the orchestration process, never inside the sysbench container, so it
 * cannot influence the scored numbers.
 *
 * Signals come from `systeminformation`, which reads the host's /proc and /sys
 * (not namespaced for these counters), so this works from inside a container too.
 * Temperature legitimately isn't available on every host (VMs, some hwmon-less
 * boards) and is reported as null rather than faked.
 */
 export class BenchmarkTelemetrySampler {
  private timer: NodeJS.Timeout | null = null
  private readonly benchmarkId: string | null
  private status: BenchmarkStatus = 'starting'
  private startedAt = Date.now()
  private stageMetric: BenchmarkTelemetry['stage_metric'] | undefined
  private gpu: BenchmarkTelemetry['gpu'] | undefined
  private sampling = false
  constructor(benchmarkId: string | null) {
    this.benchmarkId = benchmarkId
  }
  start() {
    if (this.timer) return
    this.startedAt = Date.now()
    // Prime systeminformation's per-second deltas and emit an initial frame,
    // then continue on the interval.
    void this._sample()
    this.timer = setInterval(() => void this._sample(), SAMPLE_INTERVAL_MS)
  }
  /** Tag subsequent frames with the current stage; clears any prior in-test state. */
  setStage(status: BenchmarkStatus) {
    this.status = status
    this.stageMetric = undefined
    this.gpu = undefined
  }
  /** Inject an in-test metric (e.g. live AI tokens/sec) into subsequent frames. */
  setStageMetric(kind: NonNullable<BenchmarkTelemetry['stage_metric']>['kind'], value: number, ttftMs?: number) {
    this.stageMetric = { kind, value, ...(ttftMs !== undefined ? { ttft_ms: ttftMs } : {}) }
  }
  /** Inject NVIDIA GPU stats into subsequent frames (null clears them). */
  setGpuStats(gpu: NonNullable<BenchmarkTelemetry['gpu']> | null) {
    this.gpu = gpu ?? undefined
  }
  stop() {
    if (this.timer) {
      clearInterval(this.timer)
      this.timer = null
    }
  }
  private async _sample() {
    // Skip if the previous sample is still in flight (currentLoad can take a
    // couple hundred ms); never let samples pile up.
    if (this.sampling) return
    this.sampling = true
    try {
      const [load, temp, fs] = await Promise.all([
        si.currentLoad().catch(() => null),
        si.cpuTemperature().catch(() => null),
        si.fsStats().catch(() => null),
      ])
      const overall = load ? Math.max(0, Math.round(load.currentLoad)) : 0
      const perCore = load?.cpus?.map((c) => Math.max(0, Math.round(c.load))) ?? []
      const tempC = temp && typeof temp.main === 'number' && temp.main > 0 ? Math.round(temp.main) : null
      const readMb = fs && typeof fs.rx_sec === 'number' && fs.rx_sec >= 0 ? Number((fs.rx_sec / 1e6).toFixed(1)) : 0
      const writeMb = fs && typeof fs.wx_sec === 'number' && fs.wx_sec >= 0 ? Number((fs.wx_sec / 1e6).toFixed(1)) : 0
      const payload: BenchmarkTelemetry = {
        benchmark_id: this.benchmarkId,
        status: this.status,
        t: Date.now() - this.startedAt,
        cpu: { overall, per_core: perCore },
        temp_c: tempC,
        disk: { read_mb_s: readMb, write_mb_s: writeMb },
        ...(this.stageMetric ? { stage_metric: this.stageMetric } : {}),
        ...(this.gpu ? { gpu: this.gpu } : {}),
      }
      transmit.broadcast(BROADCAST_CHANNELS.BENCHMARK_TELEMETRY, payload)
    } catch (err) {
      logger.debug(`[BenchmarkTelemetry] sample failed: ${err.message}`)
    } finally {
      this.sampling = false
    }
  }
 }
--- a/admin/app/services/chat_service.ts
+++ b/admin/app/services/chat_service.ts
@ -1,289 +0,0 @@
 import ChatSession from '#models/chat_session'
 import ChatMessage from '#models/chat_message'
 import KVStore from '#models/kv_store'
 import logger from '@adonisjs/core/services/logger'
 import { DateTime } from 'luxon'
 import { inject } from '@adonisjs/core'
 import { OllamaService } from './ollama_service.js'
 import { SYSTEM_PROMPTS } from '../../constants/ollama.js'
 import { toTitleCase } from '../utils/misc.js'
@inject()
 export class ChatService {
  constructor(private ollamaService: OllamaService) {}
  async getAllSessions() {
    try {
      const sessions = await ChatSession.query().orderBy('updated_at', 'desc')
      return sessions.map((session) => ({
        id: session.id.toString(),
        title: session.title,
        model: session.model,
        timestamp: session.updated_at.toJSDate(),
        lastMessage: null, // Will be populated from messages if needed
      }))
    } catch (error) {
      logger.error(
        `[ChatService] Failed to get sessions: ${error instanceof Error ? error.message : error}`
      )
      return []
    }
  }
  async getChatSuggestions() {
    try {
      const models = await this.ollamaService.getModels()
      if (!models || models.length === 0) {
        return [] // If no models are available, return empty suggestions
      }
      // Prefer the user's selected chat model. Fall back to the smallest
      // installed model — picking the largest by file size is unsafe: if any
      // installed model exceeds available VRAM (e.g. llama3.1:405b on a 96 GB
      // GPU), Ollama spends minutes trying to load it and the request 500s.
      // Suggestions are short prompts that don't benefit from a flagship model.
      const lastModel = await KVStore.getValue('chat.lastModel')
      const preferred = lastModel ? models.find((m) => m.name === lastModel) : undefined
      const chosen =
        preferred ??
        models.reduce((prev, current) => (prev.size < current.size ? prev : current))
      if (!chosen) {
        return []
      }
      const response = await this.ollamaService.chat({
        model: chosen.name,
        messages: [
          {
            role: 'user',
            content: SYSTEM_PROMPTS.chat_suggestions,
          }
        ],
        stream: false,
      })
      if (response && response.message && response.message.content) {
        const content = response.message.content.trim()
        // Handle both comma-separated and newline-separated formats
        let suggestions: string[] = []
        // Try splitting by commas first
        if (content.includes(',')) {
          suggestions = content.split(',').map((s) => s.trim())
        } 
        // Fall back to newline separation
        else {
          suggestions = content
            .split(/\r?\n/)
            .map((s) => s.trim())
            // Remove numbered list markers (1., 2., 3., etc.) and bullet points
            .map((s) => s.replace(/^\d+\.\s*/, '').replace(/^[-*•]\s*/, ''))
            // Remove surrounding quotes if present
            .map((s) => s.replace(/^["']|["']$/g, ''))
        }
        // Filter out empty strings and limit to 3 suggestions
        const filtered =  suggestions
          .filter((s) => s.length > 0)
          .slice(0, 3)
        return filtered.map((s) => toTitleCase(s))
      } else {
        return []
      }
    } catch (error) {
      logger.error(
        `[ChatService] Failed to get chat suggestions: ${
          error instanceof Error ? error.message : error
        }`
      )
      return []
    }
  }
  async getSession(sessionId: number) {
    try {
      const session = await ChatSession.query().where('id', sessionId).preload('messages').first()
      if (!session) {
        return null
      }
      return {
        id: session.id.toString(),
        title: session.title,
        model: session.model,
        timestamp: session.updated_at.toJSDate(),
        messages: session.messages.map((msg) => ({
          id: msg.id.toString(),
          role: msg.role,
          content: msg.content,
          timestamp: msg.created_at.toJSDate(),
        })),
      }
    } catch (error) {
      logger.error(
        `[ChatService] Failed to get session ${sessionId}: ${
          error instanceof Error ? error.message : error
        }`
      )
      return null
    }
  }
  async createSession(title: string, model?: string) {
    try {
      const session = await ChatSession.create({
        title,
        model: model || null,
      })
      return {
        id: session.id.toString(),
        title: session.title,
        model: session.model,
        timestamp: session.created_at.toJSDate(),
      }
    } catch (error) {
      logger.error(
        `[ChatService] Failed to create session: ${error instanceof Error ? error.message : error}`
      )
      throw new Error('Failed to create chat session')
    }
  }
  async updateSession(sessionId: number, data: { title?: string; model?: string }) {
    try {
      const session = await ChatSession.findOrFail(sessionId)
      if (data.title) {
        session.title = data.title
      }
      if (data.model !== undefined) {
        session.model = data.model
      }
      await session.save()
      return {
        id: session.id.toString(),
        title: session.title,
        model: session.model,
        timestamp: session.updated_at.toJSDate(),
      }
    } catch (error) {
      logger.error(
        `[ChatService] Failed to update session ${sessionId}: ${
          error instanceof Error ? error.message : error
        }`
      )
      throw new Error('Failed to update chat session')
    }
  }
  async addMessage(sessionId: number, role: 'system' | 'user' | 'assistant', content: string) {
    try {
      const message = await ChatMessage.create({
        session_id: sessionId,
        role,
        content,
      })
      // Update session's updated_at timestamp
      const session = await ChatSession.findOrFail(sessionId)
      session.updated_at = DateTime.now()
      await session.save()
      return {
        id: message.id.toString(),
        role: message.role,
        content: message.content,
        timestamp: message.created_at.toJSDate(),
      }
    } catch (error) {
      logger.error(
        `[ChatService] Failed to add message to session ${sessionId}: ${
          error instanceof Error ? error.message : error
        }`
      )
      throw new Error('Failed to add message')
    }
  }
  async deleteSession(sessionId: number) {
    try {
      const session = await ChatSession.findOrFail(sessionId)
      await session.delete()
      return { success: true }
    } catch (error) {
      logger.error(
        `[ChatService] Failed to delete session ${sessionId}: ${
          error instanceof Error ? error.message : error
        }`
      )
      throw new Error('Failed to delete chat session')
    }
  }
  async getMessageCount(sessionId: number): Promise<number> {
    try {
      const count = await ChatMessage.query().where('session_id', sessionId).count('* as total')
      return Number(count[0].$extras.total)
    } catch (error) {
      logger.error(
        `[ChatService] Failed to get message count for session ${sessionId}: ${error instanceof Error ? error.message : error}`
      )
      return 0
    }
  }
  async generateTitle(sessionId: number, userMessage: string, assistantMessage: string, model: string) {
    try {
      let title: string
      const response = await this.ollamaService.chat({
        model,
        messages: [
          { role: 'system', content: SYSTEM_PROMPTS.title_generation },
          { role: 'user', content: userMessage },
          { role: 'assistant', content: assistantMessage },
        ],
      })
      title = response?.message?.content?.trim()
      if (!title) {
        title = userMessage.slice(0, 57) + (userMessage.length > 57 ? '...' : '')
      }
      await this.updateSession(sessionId, { title })
      logger.info(`[ChatService] Generated title for session ${sessionId}: "${title}"`)
    } catch (error) {
      logger.error(
        `[ChatService] Failed to generate title for session ${sessionId}: ${error instanceof Error ? error.message : error}`
      )
      // Fall back to truncated user message
      try {
        const fallbackTitle = userMessage.slice(0, 57) + (userMessage.length > 57 ? '...' : '')
        await this.updateSession(sessionId, { title: fallbackTitle })
      } catch {
        // Silently fail - session keeps "New Chat" title
      }
    }
  }
  async deleteAllSessions() {
    try {
      await ChatSession.query().delete()
      return { success: true, message: 'All chat sessions deleted' }
    } catch (error) {
      logger.error(
        `[ChatService] Failed to delete all sessions: ${
          error instanceof Error ? error.message : error
        }`
      )
      throw new Error('Failed to delete all chat sessions')
    }
  }
 }
--- a/admin/app/services/collection_manifest_service.ts
+++ b/admin/app/services/collection_manifest_service.ts
@ -1,550 +0,0 @@
 import axios from 'axios'
 import vine from '@vinejs/vine'
 import logger from '@adonisjs/core/services/logger'
 import { DateTime } from 'luxon'
 import { join } from 'path'
 import CollectionManifest from '#models/collection_manifest'
 import InstalledResource from '#models/installed_resource'
 import WikipediaSelection from '#models/wikipedia_selection'
 import { QueueService } from './queue_service.js'
 import { RunDownloadJob } from '#jobs/run_download_job'
 import { zimCategoriesSpecSchema, mapsSpecSchema, wikipediaSpecSchema, creatorPacksSpecSchema } from '#validators/curated_collections'
 import { isGatedResource } from '../utils/hosted_content.js'
 import {
  ensureDirectoryExists,
  listDirectoryContents,
  getFileStatsIfExists,
  ZIM_STORAGE_PATH,
 } from '../utils/fs.js'
 import type {
  ManifestType,
  ZimCategoriesSpec,
  MapsSpec,
  CreatorPacksSpec,
  CategoryWithStatus,
  CollectionWithStatus,
  CreatorPackWithStatus,
  SpecResource,
  SpecTier,
 } from '../../types/collections.js'
 const SPEC_URLS: Record<ManifestType, string> = {
  zim_categories: 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/kiwix-categories.json',
  maps: 'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/main/collections/maps.json',
  wikipedia: 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/wikipedia.json',
  creator_packs: 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/creator-packs.json',
 }
 const VALIDATORS: Record<ManifestType, any> = {
  zim_categories: zimCategoriesSpecSchema,
  maps: mapsSpecSchema,
  wikipedia: wikipediaSpecSchema,
  creator_packs: creatorPacksSpecSchema,
 }
 export class CollectionManifestService {
  private readonly mapStoragePath = '/storage/maps'
  // ---- Spec management ----
  async fetchAndCacheSpec(type: ManifestType): Promise<boolean> {
    try {
      const response = await axios.get(SPEC_URLS[type], { timeout: 15000 })
      const validated = await vine.validate({
        schema: VALIDATORS[type],
        data: response.data,
      })
      const existing = await CollectionManifest.find(type)
      const specVersion = validated.spec_version
      if (existing) {
        const changed = existing.spec_version !== specVersion
        existing.spec_version = specVersion
        existing.spec_data = validated
        existing.fetched_at = DateTime.now()
        await existing.save()
        return changed
      }
      await CollectionManifest.create({
        type,
        spec_version: specVersion,
        spec_data: validated,
        fetched_at: DateTime.now(),
      })
      return true
    } catch (error) {
      logger.error(`[CollectionManifestService] Failed to fetch spec for ${type}:`, error?.message || error)
      return false
    }
  }
  async getCachedSpec<T>(type: ManifestType): Promise<T | null> {
    const manifest = await CollectionManifest.find(type)
    if (!manifest) return null
    return manifest.spec_data as T
  }
  async getSpecWithFallback<T>(type: ManifestType): Promise<T | null> {
    try {
      await this.fetchAndCacheSpec(type)
    } catch {
      // Fetch failed, will fall back to cache
    }
    return this.getCachedSpec<T>(type)
  }
  // ---- Status computation ----
  async getCategoriesWithStatus(): Promise<CategoryWithStatus[]> {
    const spec = await this.getSpecWithFallback<ZimCategoriesSpec>('zim_categories')
    if (!spec) return []
    // Include 'dataset' rows alongside 'zim' so a curated tier carrying the FDA
    // drug dataset reads "installed" once the ingest writes its row (the tier-
    // status math below treats every resolved resource id uniformly — a dataset
    // resource is just another id to account for).
    const installedResources = await InstalledResource.query().whereIn('resource_type', [
      'zim',
      'dataset',
    ])
    const installedMap = new Map(installedResources.map((r) => [r.resource_id, r]))
    // In-flight ZIM + dataset download resource IDs from the BullMQ queues. Used
    // to surface the user's tier intent immediately on submit, before any single
    // file has finished downloading. Failed jobs are excluded so a stuck queue
    // entry doesn't keep claiming the user's pick forever.
    const inFlightIds = await this.getInFlightZimResourceIds()
    // Whether the in-flight drug dataset is in its INGEST (indexing) phase — the
    // download finished and the heavy ingest is running. Lets the wizard card
    // flip "(downloading)" → "(indexing)" at the handoff (Req 7) instead of
    // showing a stale "downloading" through the long index.
    const drugIndexing = await this.isDrugDatasetIndexing()
    return spec.categories.map((category) => {
      const installedTierSlug = this.getInstalledTierForCategory(category.tiers, installedMap)
      const downloadingTierSlug = this.getDownloadingTierForCategory(
        category.tiers,
        installedMap,
        inFlightIds,
        installedTierSlug
      )
      // Only mark "indexing" when this category's downloading tier actually
      // carries a dataset resource that is the one indexing.
      const downloadingTierIndexing =
        drugIndexing && downloadingTierSlug
          ? CollectionManifestService.resolveTierResources(
              category.tiers.find((t) => t.slug === downloadingTierSlug)!,
              category.tiers
            ).some((r) => r.type === 'dataset')
          : false
      return { ...category, installedTierSlug, downloadingTierSlug, downloadingTierIndexing }
    })
  }
  /**
   * True when the FDA drug dataset's INGEST is in flight while its DOWNLOAD is
   * not — i.e. the handoff into the indexing phase. Drives the wizard card's
   * "(indexing)" label. Defensive: any queue read failure returns false (the card
   * just keeps showing "(downloading)") rather than breaking the categories list.
   */
  private async isDrugDatasetIndexing(): Promise<boolean> {
    try {
      const { DownloadDrugDataJob } = await import('#jobs/download_drug_data_job')
      const { IngestDrugDataJob } = await import('#jobs/ingest_drug_data_job')
      const queueService = QueueService.getInstance()
      const ingestQueue = queueService.getQueue(IngestDrugDataJob.queue)
      const ingestJobs = await ingestQueue.getJobs(['active', 'waiting', 'delayed'])
      if (ingestJobs.length === 0) return false
      const downloadQueue = queueService.getQueue(DownloadDrugDataJob.queue)
      const downloadJobs = await downloadQueue.getJobs(['active', 'waiting', 'delayed'])
      return downloadJobs.length === 0
    } catch (error: any) {
      logger.warn(
        '[CollectionManifestService] Could not determine drug indexing state:',
        error?.message || error
      )
      return false
    }
  }
  private async getInFlightZimResourceIds(): Promise<Set<string>> {
    const ids = new Set<string>()
    try {
      const queue = QueueService.getInstance().getQueue(RunDownloadJob.queue)
      const jobs = await queue.getJobs(['waiting', 'active', 'delayed'])
      for (const job of jobs) {
        if (job.data?.filetype !== 'zim') continue
        const resourceId = job.data?.resourceMetadata?.resource_id
        if (typeof resourceId === 'string') ids.add(resourceId)
      }
    } catch (error) {
      // Don't fail the whole categories endpoint if the queue is briefly
      // unreachable — just report no in-flight downloads.
      logger.warn('[CollectionManifestService] Could not read download queue:', error?.message || error)
    }
    // Also surface an in-flight curated-tier drug-dataset install. The drug
    // download/ingest run on their own queues (not RunDownloadJob), carrying the
    // dataset's resource id in resourceMeta — read it so the wizard shows the
    // Medicine tier as "downloading" the moment the user opts in, mirroring the
    // ZIM behaviour. Scanned independently so a missing drug queue can't blank
    // the ZIM in-flight set above.
    await this.addInFlightDrugDatasetId(ids)
    return ids
  }
  /**
   * Add the FDA drug dataset's manifest resource id to `ids` when its download or
   * ingest is in flight. The dataset only counts as "downloading" while no
   * install-state row exists yet — once ingest writes the row it is "installed"
   * (handled by the installedMap), so a still-running ingest correctly reads as
   * the in-flight tier intent here. resourceMeta is only present on a curated-tier
   * install, so a manual (non-tier) drug download never claims a tier slug.
   */
  private async addInFlightDrugDatasetId(ids: Set<string>): Promise<void> {
    try {
      const { DownloadDrugDataJob } = await import('#jobs/download_drug_data_job')
      const { IngestDrugDataJob } = await import('#jobs/ingest_drug_data_job')
      const queueService = QueueService.getInstance()
      for (const queueName of [DownloadDrugDataJob.queue, IngestDrugDataJob.queue]) {
        const queue = queueService.getQueue(queueName)
        const jobs = await queue.getJobs(['waiting', 'active', 'delayed'])
        for (const job of jobs) {
          const resourceId = job.data?.resourceMeta?.resourceId
          if (typeof resourceId === 'string') ids.add(resourceId)
        }
      }
    } catch (error: any) {
      logger.warn(
        '[CollectionManifestService] Could not read drug dataset queue:',
        error?.message || error
      )
    }
  }
  /**
   * Highest tier whose every resource is installed OR has an in-flight
   * download. Returns undefined when there are no in-flight downloads for this
   * category, or when the result would just duplicate installedTierSlug (i.e.
   * everything that's downloading is already installed — nothing new to show).
   */
  getDownloadingTierForCategory(
    tiers: SpecTier[],
    installedMap: Map<string, InstalledResource>,
    inFlightIds: Set<string>,
    installedTierSlug: string | undefined
  ): string | undefined {
    if (inFlightIds.size === 0) return undefined
    // Cheap pre-check: any of this category's resources actually in flight?
    const anyInFlight = tiers.some((tier) =>
      CollectionManifestService.resolveTierResources(tier, tiers).some((r) => inFlightIds.has(r.id))
    )
    if (!anyInFlight) return undefined
    const reversedTiers = [...tiers].reverse()
    for (const tier of reversedTiers) {
      const resolved = CollectionManifestService.resolveTierResources(tier, tiers)
      if (resolved.length === 0) continue
      const allAccountedFor = resolved.every(
        (r) => installedMap.has(r.id) || inFlightIds.has(r.id)
      )
      if (allAccountedFor) {
        return tier.slug === installedTierSlug ? undefined : tier.slug
      }
    }
    return undefined
  }
  async getMapCollectionsWithStatus(): Promise<CollectionWithStatus[]> {
    const spec = await this.getSpecWithFallback<MapsSpec>('maps')
    if (!spec) return []
    const installedResources = await InstalledResource.query().where('resource_type', 'map')
    const installedIds = new Set(installedResources.map((r) => r.resource_id))
    return spec.collections.map((collection) => {
      const installedCount = collection.resources.filter((r) => installedIds.has(r.id)).length
      return {
        ...collection,
        all_installed: installedCount === collection.resources.length,
        installed_count: installedCount,
        total_count: collection.resources.length,
      }
    })
  }
  /**
   * Per-pack install status for Creator Packs. Packs are single ZIMs, so this is
   * a one-resource-per-pack join (simpler than the tiered category logic):
   * catalog ⋈ InstalledResource (resource_type 'zim', matched on resource_id) ⋈
   * the in-flight download queue. `available_update_version` is set when an
   * installed pack's version trails the catalog (a creator published a rebuild).
   */
  async getCreatorPacksWithStatus(): Promise<CreatorPackWithStatus[]> {
    const spec = await this.getSpecWithFallback<CreatorPacksSpec>('creator_packs')
    if (!spec) return []
    const installedResources = await InstalledResource.query().where('resource_type', 'zim')
    const installedMap = new Map(installedResources.map((r) => [r.resource_id, r]))
    const inFlightIds = await this.getInFlightZimResourceIds()
    return spec.packs.map((pack) => {
      const installed = installedMap.get(pack.resource_id)
      if (installed) {
        const hasUpdate = installed.version !== pack.version
        return {
          ...pack,
          status: 'installed' as const,
          installed_version: installed.version,
          ...(hasUpdate ? { available_update_version: pack.version } : {}),
        }
      }
      if (inFlightIds.has(pack.resource_id)) {
        return { ...pack, status: 'downloading' as const }
      }
      return { ...pack, status: 'available' as const }
    })
  }
  /**
   * Resource ids in the ZIM manifest that we host ourselves behind the
   * entitlement Worker (`auth: 'nomad_app_key'`).
   *
   * Used to keep gated content out of the Kiwix-catalog update path. Those
   * resources are not in the openzim catalog, so they can never legitimately
   * match there — but a resource-id collision would otherwise let a third-party
   * mirror present itself as a newer version and overwrite our content. Their
   * versions come from the manifest instead.
   *
   * Reads the CACHED spec rather than refetching: this sits on the scheduled
   * update-check path and does not need a network round-trip. A gated resource
   * cannot be installed without the manifest having been fetched first, so the
   * cache is always populated by the time it matters.
   *
   * Returns an empty set if the manifest has never been cached, which correctly
   * degrades to current behaviour rather than skipping every update.
   */
  async getGatedZimResourceIds(): Promise<Set<string>> {
    const ids = new Set<string>()
    const spec = await this.getCachedSpec<ZimCategoriesSpec>('zim_categories')
    if (!spec) return ids
    for (const category of spec.categories) {
      for (const tier of category.tiers) {
        for (const resource of tier.resources) {
          if (isGatedResource(resource)) ids.add(resource.id)
        }
      }
    }
    return ids
  }
  // ---- Tier resolution ----
  static resolveTierResources(tier: SpecTier, allTiers: SpecTier[]): SpecResource[] {
    const visited = new Set<string>()
    return CollectionManifestService._resolveTierResourcesInner(tier, allTiers, visited)
  }
  private static _resolveTierResourcesInner(
    tier: SpecTier,
    allTiers: SpecTier[],
    visited: Set<string>
  ): SpecResource[] {
    if (visited.has(tier.slug)) return [] // cycle detection
    visited.add(tier.slug)
    const resources: SpecResource[] = []
    if (tier.includesTier) {
      const included = allTiers.find((t) => t.slug === tier.includesTier)
      if (included) {
        resources.push(...CollectionManifestService._resolveTierResourcesInner(included, allTiers, visited))
      }
    }
    resources.push(...tier.resources)
    return resources
  }
  getInstalledTierForCategory(
    tiers: SpecTier[],
    installedMap: Map<string, InstalledResource>
  ): string | undefined {
    // Check from highest tier to lowest (tiers are ordered low to high in spec)
    const reversedTiers = [...tiers].reverse()
    for (const tier of reversedTiers) {
      const resolved = CollectionManifestService.resolveTierResources(tier, tiers)
      if (resolved.length === 0) continue
      const allInstalled = resolved.every((r) => installedMap.has(r.id))
      if (allInstalled) {
        return tier.slug
      }
    }
    return undefined
  }
  // ---- Filename parsing ----
  static parseZimFilename(filename: string): { resource_id: string; version: string } | null {
    const name = filename.replace(/\.zim$/, '')
    const match = name.match(/^(.+)_(\d{4}-\d{2})$/)
    if (!match) return null
    return { resource_id: match[1], version: match[2] }
  }
  static parseMapFilename(filename: string): { resource_id: string; version: string } | null {
    const name = filename.replace(/\.pmtiles$/, '')
    const match = name.match(/^(.+)_(\d{4}-\d{2})$/)
    if (!match) return null
    return { resource_id: match[1], version: match[2] }
  }
  // ---- Filesystem reconciliation ----
  async reconcileFromFilesystem(): Promise<{ zim: number; map: number }> {
    let zimCount = 0
    let mapCount = 0
    console.log("RECONCILING FILESYSTEM MANIFESTS...")
    // Reconcile ZIM files
    try {
      const zimDir = join(process.cwd(), ZIM_STORAGE_PATH)
      await ensureDirectoryExists(zimDir)
      const zimItems = await listDirectoryContents(zimDir)
      const zimFiles = zimItems.filter((f) => f.name.endsWith('.zim'))
      console.log(`Found ${zimFiles.length} ZIM files on disk. Reconciling with database...`)
      // Get spec for URL lookup
      const zimSpec = await this.getCachedSpec<ZimCategoriesSpec>('zim_categories')
      const specResourceMap = new Map<string, SpecResource>()
      if (zimSpec) {
        for (const cat of zimSpec.categories) {
          for (const tier of cat.tiers) {
            for (const res of tier.resources) {
              specResourceMap.set(res.id, res)
            }
          }
        }
      }
      const seenZimIds = new Set<string>()
      // Only skip the single Wikipedia file tracked by WikipediaSelection — not every file
      // starting with `wikipedia_en_`. Curated category tiers (e.g. Medicine → Comprehensive)
      // ship Wikipedia-themed ZIMs like `wikipedia_en_medicine_maxi` that must reconcile
      // normally; otherwise their InstalledResource row gets wiped on every restart and the
      // tier detection silently downgrades.
      const wikipediaSelection = await WikipediaSelection.query().first()
      const managedWikipediaFilename = wikipediaSelection?.filename ?? null
      for (const file of zimFiles) {
        console.log(`Processing ZIM file: ${file.name}`)
        if (managedWikipediaFilename && file.name === managedWikipediaFilename) continue
        const parsed = CollectionManifestService.parseZimFilename(file.name)
        console.log(`Parsed ZIM filename:`, parsed)
        if (!parsed) continue
        seenZimIds.add(parsed.resource_id)
        const specRes = specResourceMap.get(parsed.resource_id)
        const filePath = join(zimDir, file.name)
        const stats = await getFileStatsIfExists(filePath)
        await InstalledResource.updateOrCreate(
          { resource_id: parsed.resource_id, resource_type: 'zim' },
          {
            version: parsed.version,
            url: specRes?.url || '',
            file_path: filePath,
            file_size_bytes: stats ? Number(stats.size) : null,
            installed_at: DateTime.now(),
          }
        )
        zimCount++
      }
      // Remove entries for ZIM files no longer on disk
      const existingZim = await InstalledResource.query().where('resource_type', 'zim')
      for (const entry of existingZim) {
        if (!seenZimIds.has(entry.resource_id)) {
          await entry.delete()
        }
      }
    } catch (error) {
      logger.error('[CollectionManifestService] Error reconciling ZIM files:', error)
    }
    // Reconcile map files
    try {
      const mapDir = join(process.cwd(), this.mapStoragePath, 'pmtiles')
      await ensureDirectoryExists(mapDir)
      const mapItems = await listDirectoryContents(mapDir)
      const mapFiles = mapItems.filter((f) => f.name.endsWith('.pmtiles'))
      // Get spec for URL/version lookup
      const mapSpec = await this.getCachedSpec<MapsSpec>('maps')
      const mapResourceMap = new Map<string, SpecResource>()
      if (mapSpec) {
        for (const col of mapSpec.collections) {
          for (const res of col.resources) {
            mapResourceMap.set(res.id, res)
          }
        }
      }
      const seenMapIds = new Set<string>()
      for (const file of mapFiles) {
        const parsed = CollectionManifestService.parseMapFilename(file.name)
        if (!parsed) continue
        seenMapIds.add(parsed.resource_id)
        const specRes = mapResourceMap.get(parsed.resource_id)
        const filePath = join(mapDir, file.name)
        const stats = await getFileStatsIfExists(filePath)
        await InstalledResource.updateOrCreate(
          { resource_id: parsed.resource_id, resource_type: 'map' },
          {
            version: parsed.version,
            url: specRes?.url || '',
            file_path: filePath,
            file_size_bytes: stats ? Number(stats.size) : null,
            installed_at: DateTime.now(),
          }
        )
        mapCount++
      }
      // Remove entries for map files no longer on disk
      const existingMaps = await InstalledResource.query().where('resource_type', 'map')
      for (const entry of existingMaps) {
        if (!seenMapIds.has(entry.resource_id)) {
          await entry.delete()
        }
      }
    } catch (error) {
      logger.error('[CollectionManifestService] Error reconciling map files:', error)
    }
    logger.info(`[CollectionManifestService] Reconciled ${zimCount} ZIM files, ${mapCount} map files`)
    return { zim: zimCount, map: mapCount }
  }
 }
--- a/admin/app/services/collection_update_service.ts
+++ b/admin/app/services/collection_update_service.ts
@ -1,197 +0,0 @@
 import logger from '@adonisjs/core/services/logger'
 import axios from 'axios'
 import { DateTime } from 'luxon'
 import InstalledResource from '#models/installed_resource'
 import { RunDownloadJob } from '../jobs/run_download_job.js'
 import { ZIM_STORAGE_PATH } from '../utils/fs.js'
 import { join } from 'path'
 import type {
  ResourceUpdateInfo,
  ContentUpdateCheckResult,
 } from '../../types/collections.js'
 import { KiwixCatalogService, reconcileResourceUpdateState } from './kiwix_catalog_service.js'
 import { CollectionManifestService } from './collection_manifest_service.js'
 const MAP_STORAGE_PATH = '/storage/maps'
 const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
 const PMTILES_MIME_TYPES = ['application/vnd.pmtiles', 'application/octet-stream']
 export class CollectionUpdateService {
  /**
   * Check every installed resource against the upstream catalogs locally (Kiwix
   * OPDS for ZIMs, GitHub for maps) — no longer routed through the external
   * project-nomad-api. Side-effect: persists each resource's available-update
   * state (version + cool-off anchor) so the auto-updater can act on it later.
   */
  async checkForUpdates(): Promise<ContentUpdateCheckResult> {
    // ZIM/map catalog update path only — exclude `dataset` resources (e.g. the
    // FDA drug labels), which are not filename-versioned and get their own
    // freshness path. No-op today (no dataset rows are written in this slice).
    const allInstalled = await InstalledResource.query().whereNot('resource_type', 'dataset')
    // Content we host ourselves is versioned by the manifest, not by the Kiwix
    // catalog, so it has no business in this check. Excluding it also means a
    // resource-id collision can't let a third-party mirror present itself as a
    // newer version and overwrite our content. See resolveZimDownload, which
    // pins the same resources to their manifest URL on the install path.
    const gatedIds = await new CollectionManifestService().getGatedZimResourceIds()
    const installed = allInstalled.filter(
      (r) => !(r.resource_type === 'zim' && gatedIds.has(r.resource_id))
    )
    if (installed.length === 0) {
      return {
        updates: [],
        checked_at: new Date().toISOString(),
      }
    }
    try {
      const catalog = new KiwixCatalogService()
      const latestByKey = await catalog.getLatestForResources(
        // `dataset` rows are filtered out above, so the type narrows to ZIM/map.
        installed.map((r) => ({
          resource_id: r.resource_id,
          resource_type: r.resource_type as 'zim' | 'map',
        }))
      )
      const now = DateTime.now()
      const updates: ResourceUpdateInfo[] = []
      for (const resource of installed) {
        const latest = latestByKey.get(`${resource.resource_type}:${resource.resource_id}`) ?? null
        await reconcileResourceUpdateState(resource, latest, now)
        if (latest && latest.version > resource.version) {
          updates.push({
            resource_id: resource.resource_id,
            resource_type: resource.resource_type as 'zim' | 'map',
            installed_version: resource.version,
            latest_version: latest.version,
            download_url: latest.download_url,
            size_bytes: latest.size_bytes || undefined,
          })
        }
      }
      logger.info(
        `[CollectionUpdateService] Local update check complete: ${updates.length} update(s) available`
      )
      const enriched = await this.enrichWithSizes(updates)
      return {
        updates: enriched,
        checked_at: new Date().toISOString(),
      }
    } catch (error) {
      const message = error instanceof Error ? error.message : 'Unknown error during update check'
      logger.error(`[CollectionUpdateService] Failed to check for updates: ${message}`)
      return {
        updates: [],
        checked_at: new Date().toISOString(),
        error: 'Failed to check for content updates. Please try again later.',
      }
    }
  }
  async applyUpdate(
    update: ResourceUpdateInfo,
    options?: { auto?: boolean }
  ): Promise<{ success: boolean; jobId?: string; error?: string }> {
    // Check if a download is already in progress for this URL
    const existingJob = await RunDownloadJob.getByUrl(update.download_url)
    if (existingJob) {
      const state = await existingJob.getState()
      if (state === 'active' || state === 'waiting' || state === 'delayed') {
        return {
          success: false,
          error: `A download is already in progress for ${update.resource_id}`,
        }
      }
    }
    const filename = this.buildFilename(update)
    const filepath = this.buildFilepath(update, filename)
    const result = await RunDownloadJob.dispatch({
      url: update.download_url,
      filepath,
      timeout: 30000,
      allowedMimeTypes:
        update.resource_type === 'zim' ? ZIM_MIME_TYPES : PMTILES_MIME_TYPES,
      filetype: update.resource_type,
      title: update.resource_id,
      totalBytes: update.size_bytes,
      resourceMetadata: {
        resource_id: update.resource_id,
        version: update.latest_version,
        collection_ref: null,
        auto: options?.auto ?? false,
      },
    })
    if (!result || !result.job) {
      return { success: false, error: 'Failed to dispatch download job' }
    }
    logger.info(
      `[CollectionUpdateService] Dispatched update download for ${update.resource_id}: ${update.installed_version} → ${update.latest_version}`
    )
    return { success: true, jobId: result.job.id }
  }
  async applyAllUpdates(
    updates: ResourceUpdateInfo[]
  ): Promise<{ results: Array<{ resource_id: string; success: boolean; jobId?: string; error?: string }> }> {
    const results = await Promise.all(
      updates.map(async (update) => {
        const result = await this.applyUpdate(update)
        return { resource_id: update.resource_id, ...result }
      })
    )
    return { results }
  }
  /**
   * Fetch Content-Length for each update URL in parallel. HEAD failures are non-fatal —
   * the update row just renders without a size. Bounded to HEAD_TIMEOUT_MS so a slow
   * mirror doesn't block the whole check.
   */
  private async enrichWithSizes(updates: ResourceUpdateInfo[]): Promise<ResourceUpdateInfo[]> {
    const HEAD_TIMEOUT_MS = 5000
    return await Promise.all(
      updates.map(async (update) => {
        if (update.size_bytes) return update // Trust upstream if it already gave us one
        try {
          const head = await axios.head(update.download_url, {
            timeout: HEAD_TIMEOUT_MS,
            maxRedirects: 5,
            validateStatus: (s) => s >= 200 && s < 400,
          })
          const len = Number(head.headers['content-length'])
          return Number.isFinite(len) && len > 0 ? { ...update, size_bytes: len } : update
        } catch {
          return update
        }
      })
    )
  }
  private buildFilename(update: ResourceUpdateInfo): string {
    if (update.resource_type === 'zim') {
      return `${update.resource_id}_${update.latest_version}.zim`
    }
    return `${update.resource_id}_${update.latest_version}.pmtiles`
  }
  private buildFilepath(update: ResourceUpdateInfo, filename: string): string {
    if (update.resource_type === 'zim') {
      return join(process.cwd(), ZIM_STORAGE_PATH, filename)
    }
    return join(process.cwd(), MAP_STORAGE_PATH, 'pmtiles', filename)
  }
 }
--- a/admin/app/services/condition_service.ts
+++ b/admin/app/services/condition_service.ts
@ -1,329 +0,0 @@
 import db from '@adonisjs/lucid/services/db'
 import logger from '@adonisjs/core/services/logger'
 import { CONDITIONS_FILE } from '../data/conditions.js'
 import { NATURAL_REMEDIES_FILE } from '../data/natural_remedies.js'
 import { HOME_REMEDIES_FILE } from '../data/home_remedies.js'
 import {
  parseConditionsFile,
  parseNaturalRemediesFile,
  findConditionBySlug,
  toConditionSummary,
  buildIndicationQuery,
  orderOtcFirst,
  remediesForCondition,
  remediesForFreeText,
 } from '../../util/conditions.js'
 import { PRODUCT_TYPES } from '../../types/drug_reference.js'
 import type {
  Condition,
  ConditionSummary,
  ConditionDrugsResult,
  NaturalRemedy,
  NaturalRemediesFile,
 } from '../../types/conditions.js'
 import type { DrugSearchResult } from '../../types/drug_reference.js'
 /**
 * "When to use what" — condition-first service (Phase 1 + Phase 2).
 *
 * Resolves a curated condition (or a free-text situation) to the OTC drugs whose
 * FDA label indications match it, plus the natural remedies (NCCIH) whose curated
 * condition mapping includes the resolved slug.
 *
 * Reuses the Drug Reference indication-search machinery: the same combined-FULLTEXT
 * index (ft_drug_labels_name_indications), the same MAX(MATCH …) aggregate pattern
 * required under MySQL 8 ONLY_FULL_GROUP_BY, and the same brand+generic collapse —
 * then re-buckets results OTC-first.
 *
 * Natural remedies are in-memory (no DB table, no migration) — one module-level
 * parse of NATURAL_REMEDIES_FILE, reused across all requests.
 */
 /**
 * Module-level merged remedies corpus (fail-soft): the NCCIH herbs plus the
 * non-herbal home-care measures (CDC/NIH/FDA), each entry tagged with its kind
 * so the UI can badge them apart. Slugs are disjoint between the two files
 * (validated at curation time).
 */
 const HERB_FILE = parseNaturalRemediesFile(NATURAL_REMEDIES_FILE)
 const HOME_FILE = parseNaturalRemediesFile(HOME_REMEDIES_FILE)
 const REMEDIES_FILE: NaturalRemediesFile = {
  version: HERB_FILE.version,
  source: HERB_FILE.source,
  remedies: [
    ...HERB_FILE.remedies.map((r) => ({ ...r, kind: 'herb' as const })),
    ...HOME_FILE.remedies.map((r) => ({ ...r, kind: 'self-care' as const })),
  ],
 }
 export class ConditionService {
  /** Parsed (fail-soft) curated spine — bad entries are dropped, not fatal. */
  private get spine(): Condition[] {
    return parseConditionsFile(CONDITIONS_FILE).conditions
  }
  /** All curated conditions as client-facing summaries (no searchTerms). */
  listConditions(): ConditionSummary[] {
    return this.spine.map(toConditionSummary)
  }
  /**
   * The full curated natural-remedies list (NCCIH herbs). Small enough to ship
   * as page props, which lets the Drug Reference search match remedies by name
   * client-side and offer a "Natural" browse with no extra round trip.
   */
  listRemedies(): NaturalRemedy[] {
    return REMEDIES_FILE.remedies
  }
  /**
   * The full curated spine (WITH searchTerms) for server-side matching such as
   * the drug-detail reverse link (situationsForIndications). Stays server-only —
   * searchTerms are a search-implementation detail the client never receives.
   */
  allConditions(): Condition[] {
    return this.spine
  }
  /** Find a curated condition by slug. Returns null when absent. */
  findCondition(slug: string): Condition | null {
    return findConditionBySlug(this.spine, slug)
  }
  /**
   * Resolve a curated condition (by slug) to its matching OTC drugs and natural
   * remedies. Returns null when the slug is not in the curated spine so the
   * controller can 404. Drugs are OTC-only and OTC-first-ordered.
   */
  async drugsForSlug(
    slug: string,
    limit = 50,
    opts?: { route?: string; sort?: 'relevance' | 'name' }
  ): Promise<ConditionDrugsResult | null> {
    const condition = this.findCondition(slug)
    if (!condition) return null
    const drugs = await this.searchIndications(condition.searchTerms, limit, opts)
    const remedies: NaturalRemedy[] = remediesForCondition(REMEDIES_FILE, slug)
    return { condition: toConditionSummary(condition), drugs, remedies }
  }
  /**
   * Resolve a free-text situation (off-list condition) to matching OTC drugs and
   * natural remedies. Treats the raw query as a single search term. Returns a
   * synthetic condition summary echoing the query so the UI can render a consistent
   * header.
   *
   * Remedy resolution for free text:
   *   1. Try to resolve the query to a curated condition slug (exact or substring
   *      match on slug/label). If found, use remediesForCondition — this is the
   *      primary path (e.g. "burns" typed free-form still finds aloe/tea-tree).
   *   2. Secondary fallback: remediesForFreeText searches remedy name/uses by
   *      substring so an unmapped query ("athlete's foot") can still surface a
   *      relevant remedy. Results from both paths are unioned and de-duped by slug.
   */
  async drugsForFreeText(
    query: string,
    limit = 50,
    opts?: { route?: string; sort?: 'relevance' | 'name' }
  ): Promise<ConditionDrugsResult> {
    const trimmed = query.trim()
    const drugs = trimmed.length > 0 ? await this.searchIndications([trimmed], limit, opts) : []
    // Phase 2: union condition-mapped + free-text-matched remedies.
    const remedyMap = new Map<string, NaturalRemedy>()
    if (trimmed.length > 0) {
      // Primary: resolve to a curated condition slug the same way the client-side
      // matchSituation helper does (case-insensitive exact/substring on slug+label).
      const q = trimmed.toLowerCase()
      const matchedCondition =
        this.spine.find((c) => c.slug.toLowerCase() === q || c.label.toLowerCase() === q) ??
        this.spine.find(
          (c) =>
            c.label.toLowerCase().includes(q) || c.slug.replace(/-/g, ' ').toLowerCase().includes(q)
        ) ??
        null
      if (matchedCondition) {
        for (const r of remediesForCondition(REMEDIES_FILE, matchedCondition.slug)) {
          remedyMap.set(r.slug, r)
        }
      }
      // Secondary: name/uses substring fallback.
      for (const r of remediesForFreeText(REMEDIES_FILE, trimmed)) {
        if (!remedyMap.has(r.slug)) remedyMap.set(r.slug, r)
      }
    }
    return {
      condition: { slug: '', label: trimmed, category: 'Search' },
      drugs,
      remedies: Array.from(remedyMap.values()),
    }
  }
  /**
   * Core search: FULLTEXT over (searchable_name, indications) for the OR-expanded
   * searchTerms, OTC-filtered, collapsed by brand+generic, OTC-first-ordered.
   *
   * Strategy mirrors DrugReferenceService:
   *   1. FULLTEXT NATURAL LANGUAGE MODE on the built query (requires the query
   *      to have a >= 3-char token; innodb_ft_min_token_size = 3).
   *   2. LIKE fallback when the built query is empty/too short OR FULLTEXT throws
   *      (e.g. the index is absent) — runs each term as a LIKE clause.
   * Both paths force product_type = OTC and then orderOtcFirst (a no-op on the
   * already-OTC set, but kept so a future relaxation of the OTC filter stays
   * correctly ordered).
   */
  async searchIndications(
    searchTerms: string[],
    limit = 50,
    opts?: { route?: string; sort?: 'relevance' | 'name' }
  ): Promise<DrugSearchResult[]> {
    const ftQuery = buildIndicationQuery(searchTerms)
    // A FULLTEXT query needs a token >= innodb_ft_min_token_size (3). If the
    // longest bare token is shorter, NATURAL LANGUAGE MODE returns nothing, so
    // go straight to LIKE.
    const longestToken = ftQuery
      .replace(/"/g, ' ')
      .split(/\s+/)
      .reduce((max, t) => Math.max(max, t.length), 0)
    const useFulltext = ftQuery.length > 0 && longestToken >= 3
    if (useFulltext) {
      try {
        const rows = await this.searchIndicationFulltext(ftQuery, limit, opts)
        return orderOtcFirst(rows)
      } catch (err) {
        logger.warn(
          `[ConditionService] FULLTEXT indication search failed, falling back to LIKE: ${
            err instanceof Error ? err.message : String(err)
          }`
        )
      }
    }
    const rows = await this.searchIndicationLike(searchTerms, limit, opts)
    return orderOtcFirst(rows)
  }
  /**
   * FULLTEXT OTC indication search.
   *
   * MATCHes over (searchable_name, indications) — must exactly match the
   * ft_drug_labels_name_indications index column list. The MAX(MATCH …) wrapper
   * is LOAD-BEARING: MySQL 8 ONLY_FULL_GROUP_BY rejects a bare MATCH() in SELECT
   * under GROUP BY; wrapping it in MAX() makes it an aggregate. Copied verbatim
   * from DrugReferenceService.searchIndicationFulltext, with a fixed OTC filter.
   */
  private async searchIndicationFulltext(
    ftQuery: string,
    limit: number,
    opts?: { route?: string; sort?: 'relevance' | 'name' }
  ): Promise<DrugSearchResult[]> {
    let sql = `
      SELECT
        MIN(id) AS id,
        brand_name,
        generic_name,
        MIN(manufacturer) AS manufacturer,
        MIN(route) AS route,
        MIN(product_type) AS product_type,
        COUNT(*) AS labelCount,
        MAX(MATCH(searchable_name, indications) AGAINST(? IN NATURAL LANGUAGE MODE)) AS relevance
      FROM drug_labels
      WHERE MATCH(searchable_name, indications) AGAINST(? IN NATURAL LANGUAGE MODE)
        AND product_type = ?
    `
    const bindings: unknown[] = [ftQuery, ftQuery, PRODUCT_TYPES.OTC]
    if (opts?.route) {
      sql += ' AND route LIKE ?'
      bindings.push(`%${opts.route.toUpperCase()}%`)
    }
    sql += `
      GROUP BY brand_name, generic_name
      ORDER BY ${opts?.sort === 'name' ? 'COALESCE(brand_name, generic_name) ASC' : 'relevance DESC'}
      LIMIT ?
    `
    bindings.push(limit)
    const rows = await db.rawQuery(sql, bindings)
    return this.mapSearchRows(rows[0])
  }
  /**
   * LIKE OTC indication fallback (FULLTEXT unavailable or query too short).
   *
   * ORs a LIKE clause per search term over `indications` so any synonym can
   * surface a match. OTC-filtered and collapsed identically to the FULLTEXT path.
   */
  private async searchIndicationLike(
    searchTerms: string[],
    limit: number,
    opts?: { route?: string; sort?: 'relevance' | 'name' }
  ): Promise<DrugSearchResult[]> {
    const terms = searchTerms.map((t) => t.trim()).filter((t) => t.length > 0)
    if (terms.length === 0) return []
    const likeClauses = terms.map(() => 'indications LIKE ?').join(' OR ')
    let sql = `
      SELECT
        MIN(id) AS id,
        brand_name,
        generic_name,
        MIN(manufacturer) AS manufacturer,
        MIN(route) AS route,
        MIN(product_type) AS product_type,
        COUNT(*) AS labelCount
      FROM drug_labels
      WHERE (${likeClauses})
        AND product_type = ?
    `
    const bindings: unknown[] = [...terms.map((t) => `%${t}%`), PRODUCT_TYPES.OTC]
    if (opts?.route) {
      sql += ' AND route LIKE ?'
      bindings.push(`%${opts.route.toUpperCase()}%`)
    }
    sql += `
      GROUP BY brand_name, generic_name
      ORDER BY ${opts?.sort === 'name' ? 'COALESCE(brand_name, generic_name) ASC' : 'brand_name ASC'}
      LIMIT ?
    `
    bindings.push(limit)
    const rows = await db.rawQuery(sql, bindings)
    return this.mapSearchRows(rows[0])
  }
  /** Map raw collapsed rows to DrugSearchResult — mirrors DrugReferenceService. */
  private mapSearchRows(rows: any[]): DrugSearchResult[] {
    if (!Array.isArray(rows)) return []
    return rows.map((row) => ({
      id: Number(row.id),
      brand_name: row.brand_name ?? null,
      generic_name: row.generic_name ?? null,
      manufacturer: row.manufacturer ?? null,
      route: row.route ?? null,
      product_type: row.product_type ?? null,
      labelCount: Number(row.labelCount ?? row.labelcount ?? 1),
    }))
  }
  /**
   * Current drug_labels row count — drives the index page empty-state (no data →
   * point the user to Drug Reference to download first). Mirrors
   * DrugReferenceService.rowCount; returns 0 on any error.
   */
  async drugRowCount(): Promise<number> {
    try {
      const result = await db.rawQuery('SELECT COUNT(*) AS cnt FROM drug_labels')
      const rows = result[0] as Array<{ cnt: number | string }>
      return Number(rows[0]?.cnt ?? 0)
    } catch {
      return 0
    }
  }
 }
--- a/admin/app/services/container_registry_service.ts
+++ b/admin/app/services/container_registry_service.ts
@ -1,550 +0,0 @@
 import logger from '@adonisjs/core/services/logger'
 import { isNewerVersion, parseMajorVersion } from '../utils/version.js'
 export interface ParsedImageReference {
  registry: string
  namespace: string
  repo: string
  tag: string
  /** Full name for registry API calls: namespace/repo */
  fullName: string
 }
 export interface AvailableUpdate {
  tag: string
  isLatest: boolean
  releaseUrl?: string
 }
 interface TokenCacheEntry {
  token: string
  expiresAt: number
 }
 const SEMVER_TAG_PATTERN = /^v?(\d+\.\d+(?:\.\d+)?)$/
 const PLATFORM_SUFFIXES = ['-arm64', '-amd64', '-alpine', '-slim', '-cuda', '-rocm']
 const REJECTED_TAGS = new Set(['latest', 'nightly', 'edge', 'dev', 'beta', 'alpha', 'canary', 'rc', 'test', 'debug'])
 export class ContainerRegistryService {
  private tokenCache = new Map<string, TokenCacheEntry>()
  private sourceUrlCache = new Map<string, string | null>()
  private releaseTagPrefixCache = new Map<string, string>()
  /**
   * Parse a Docker image reference string into its components.
   */
  parseImageReference(image: string): ParsedImageReference {
    let registry: string
    let remainder: string
    let tag = 'latest'
    // Split off the tag
    const lastColon = image.lastIndexOf(':')
    if (lastColon > -1 && !image.substring(lastColon).includes('/')) {
      tag = image.substring(lastColon + 1)
      image = image.substring(0, lastColon)
    }
    // Determine registry vs image path
    const parts = image.split('/')
    if (parts.length === 1) {
      // e.g. "nginx" → Docker Hub library image
      registry = 'registry-1.docker.io'
      remainder = `library/${parts[0]}`
    } else if (parts.length === 2 && !parts[0].includes('.') && !parts[0].includes(':')) {
      // e.g. "ollama/ollama" → Docker Hub user image
      registry = 'registry-1.docker.io'
      remainder = image
    } else {
      // e.g. "ghcr.io/kiwix/kiwix-serve" → custom registry
      registry = parts[0]
      remainder = parts.slice(1).join('/')
    }
    const namespaceParts = remainder.split('/')
    const repo = namespaceParts.pop()!
    const namespace = namespaceParts.join('/')
    return {
      registry,
      namespace,
      repo,
      tag,
      fullName: remainder,
    }
  }
  /**
   * Get an anonymous auth token for the given registry and repository.
   * NOTE: This could be expanded in the future to support private repo authentication
   */
  private async getToken(registry: string, fullName: string): Promise<string> {
    const cacheKey = `${registry}/${fullName}`
    const cached = this.tokenCache.get(cacheKey)
    if (cached && cached.expiresAt > Date.now()) {
      return cached.token
    }
    let tokenUrl: string
    if (registry === 'registry-1.docker.io') {
      tokenUrl = `https://auth.docker.io/token?service=registry.docker.io&scope=repository:${fullName}:pull`
    } else if (registry === 'ghcr.io') {
      tokenUrl = `https://ghcr.io/token?service=ghcr.io&scope=repository:${fullName}:pull`
    } else {
      // For other registries, try the standard v2 token endpoint
      tokenUrl = `https://${registry}/token?service=${registry}&scope=repository:${fullName}:pull`
    }
    const response = await this.fetchWithRetry(tokenUrl)
    if (!response.ok) {
      throw new Error(`Failed to get auth token from ${registry}: ${response.status}`)
    }
    const data = (await response.json()) as { token?: string; access_token?: string }
    const token = data.token || data.access_token || ''
    if (!token) {
      throw new Error(`No token returned from ${registry}`)
    }
    // Cache for 5 minutes (tokens usually last longer, but be conservative)
    this.tokenCache.set(cacheKey, {
      token,
      expiresAt: Date.now() + 5 * 60 * 1000,
    })
    return token
  }
  /**
   * List all tags for a given image from the registry.
   */
  async listTags(parsed: ParsedImageReference): Promise<string[]> {
    const token = await this.getToken(parsed.registry, parsed.fullName)
    const allTags: string[] = []
    let url = `https://${parsed.registry}/v2/${parsed.fullName}/tags/list?n=1000`
    while (url) {
      const response = await this.fetchWithRetry(url, {
        headers: { Authorization: `Bearer ${token}` },
      })
      if (!response.ok) {
        throw new Error(`Failed to list tags for ${parsed.fullName}: ${response.status}`)
      }
      const data = (await response.json()) as { tags?: string[] }
      if (data.tags) {
        allTags.push(...data.tags)
      }
      // Handle pagination via Link header. Per the OCI/Docker registry spec the next-page
      // URL is relative (e.g. "/v2/<repo>/tags/list?last=<tag>&n=1000"), so it must be
      // resolved against the registry origin before re-fetching — assigning the raw relative
      // path to `url` makes fetch() throw "Failed to parse URL". This silently broke update
      // checks for any repo with >1000 tags (e.g. ollama/ollama, filebrowser/filebrowser),
      // which is the root cause of #945. new URL(relative, base) also passes absolute
      // next-URLs through unchanged, so it's safe for registries that return those.
      const linkHeader = response.headers.get('link')
      if (linkHeader) {
        const match = linkHeader.match(/<([^>]+)>;\s*rel="next"/)
        url = match ? new URL(match[1], `https://${parsed.registry}`).toString() : ''
      } else {
        url = ''
      }
    }
    return allTags
  }
  /**
   * Check if a specific tag supports the given architecture by fetching its manifest.
   */
  async checkArchSupport(parsed: ParsedImageReference, tag: string, hostArch: string): Promise<boolean> {
    try {
      const token = await this.getToken(parsed.registry, parsed.fullName)
      const url = `https://${parsed.registry}/v2/${parsed.fullName}/manifests/${tag}`
      const response = await this.fetchWithRetry(url, {
        headers: {
          Authorization: `Bearer ${token}`,
          Accept: [
            'application/vnd.oci.image.index.v1+json',
            'application/vnd.docker.distribution.manifest.list.v2+json',
            'application/vnd.oci.image.manifest.v1+json',
            'application/vnd.docker.distribution.manifest.v2+json',
          ].join(', '),
        },
      })
      if (!response.ok) return true // If we can't check, assume it's compatible
      const manifest = (await response.json()) as {
        mediaType?: string
        manifests?: Array<{ platform?: { architecture?: string } }>
      }
      const mediaType = manifest.mediaType || response.headers.get('content-type') || ''
      // Manifest list — check if any platform matches
      if (
        mediaType.includes('manifest.list') ||
        mediaType.includes('image.index') ||
        manifest.manifests
      ) {
        const manifests = manifest.manifests || []
        return manifests.some(
          (m: any) => m.platform && m.platform.architecture === hostArch
        )
      }
      // Single manifest — assume compatible (can't easily determine arch without fetching config blob)
      return true
    } catch (error) {
      logger.warn(`[ContainerRegistryService] Error checking arch for ${tag}: ${error.message}`)
      return true // Assume compatible on error
    }
  }
  /**
   * Estimate the compressed download size (in bytes) of an image tag for the
   * given host architecture by summing its layer sizes from the manifest.
   *
   * Resolves a multi-arch manifest list/index down to the platform-specific
   * manifest before summing `layers[].size`. Returns null on any failure so
   * callers can fall back to a conservative fixed threshold rather than
   * silently skipping a disk pre-flight check.
   */
  async getImageDownloadSize(
    parsed: ParsedImageReference,
    tag: string,
    hostArch: string
  ): Promise<number | null> {
    try {
      const token = await this.getToken(parsed.registry, parsed.fullName)
      const manifestAccept = [
        'application/vnd.oci.image.index.v1+json',
        'application/vnd.docker.distribution.manifest.list.v2+json',
        'application/vnd.oci.image.manifest.v1+json',
        'application/vnd.docker.distribution.manifest.v2+json',
      ].join(', ')
      const fetchManifest = async (ref: string) =>
        this.fetchWithRetry(`https://${parsed.registry}/v2/${parsed.fullName}/manifests/${ref}`, {
          headers: { Authorization: `Bearer ${token}`, Accept: manifestAccept },
        })
      const topRes = await fetchManifest(tag)
      if (!topRes.ok) return null
      let manifest = (await topRes.json()) as {
        mediaType?: string
        layers?: Array<{ size?: number }>
        manifests?: Array<{ digest?: string; platform?: { architecture?: string } }>
      }
      // Multi-arch manifest list/index — resolve to the host-arch child manifest.
      if (manifest.manifests?.length) {
        const match =
          manifest.manifests.find((m) => m.platform?.architecture === hostArch) ||
          manifest.manifests[0]
        if (!match?.digest) return null
        const childRes = await fetchManifest(match.digest)
        if (!childRes.ok) return null
        manifest = (await childRes.json()) as { layers?: Array<{ size?: number }> }
      }
      if (!manifest.layers?.length) return null
      return manifest.layers.reduce((total, layer) => total + (layer.size || 0), 0)
    } catch (error) {
      logger.warn(
        `[ContainerRegistryService] Failed to get image size for ${parsed.fullName}:${tag}: ${error.message}`
      )
      return null
    }
  }
  /**
   * Extract the source repository URL from an image's OCI labels.
   * Uses the standardized `org.opencontainers.image.source` label.
   * Result is cached per image (not per tag).
   */
  async getSourceUrl(parsed: ParsedImageReference): Promise<string | null> {
    const cacheKey = `${parsed.registry}/${parsed.fullName}`
    if (this.sourceUrlCache.has(cacheKey)) {
      return this.sourceUrlCache.get(cacheKey)!
    }
    try {
      const token = await this.getToken(parsed.registry, parsed.fullName)
      // First get the manifest to find the config blob digest
      const manifestUrl = `https://${parsed.registry}/v2/${parsed.fullName}/manifests/${parsed.tag}`
      const manifestRes = await this.fetchWithRetry(manifestUrl, {
        headers: {
          Authorization: `Bearer ${token}`,
          Accept: [
            'application/vnd.oci.image.manifest.v1+json',
            'application/vnd.docker.distribution.manifest.v2+json',
            'application/vnd.oci.image.index.v1+json',
            'application/vnd.docker.distribution.manifest.list.v2+json',
          ].join(', '),
        },
      })
      if (!manifestRes.ok) {
        this.sourceUrlCache.set(cacheKey, null)
        return null
      }
      const manifest = (await manifestRes.json()) as {
        config?: { digest?: string }
        manifests?: Array<{ digest?: string; mediaType?: string; platform?: { architecture?: string } }>
      }
      // If this is a manifest list, pick the first manifest to get the config
      let configDigest = manifest.config?.digest
      if (!configDigest && manifest.manifests?.length) {
        const firstManifest = manifest.manifests[0]
        if (firstManifest.digest) {
          const childRes = await this.fetchWithRetry(
            `https://${parsed.registry}/v2/${parsed.fullName}/manifests/${firstManifest.digest}`,
            {
              headers: {
                Authorization: `Bearer ${token}`,
                Accept: 'application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json',
              },
            }
          )
          if (childRes.ok) {
            const childManifest = (await childRes.json()) as { config?: { digest?: string } }
            configDigest = childManifest.config?.digest
          }
        }
      }
      if (!configDigest) {
        this.sourceUrlCache.set(cacheKey, null)
        return null
      }
      // Fetch the config blob to read labels
      const blobUrl = `https://${parsed.registry}/v2/${parsed.fullName}/blobs/${configDigest}`
      const blobRes = await this.fetchWithRetry(blobUrl, {
        headers: { Authorization: `Bearer ${token}` },
      })
      if (!blobRes.ok) {
        this.sourceUrlCache.set(cacheKey, null)
        return null
      }
      const config = (await blobRes.json()) as {
        config?: { Labels?: Record<string, string> }
      }
      const sourceUrl = config.config?.Labels?.['org.opencontainers.image.source'] || null
      this.sourceUrlCache.set(cacheKey, sourceUrl)
      return sourceUrl
    } catch (error) {
      logger.warn(`[ContainerRegistryService] Failed to get source URL for ${cacheKey}: ${error.message}`)
      this.sourceUrlCache.set(cacheKey, null)
      return null
    }
  }
  /**
   * Detect whether a GitHub/GitLab repo uses a 'v' prefix on release tags.
   * Probes the GitHub API with the current tag to determine the convention,
   * then caches the result per source URL.
   */
  async detectReleaseTagPrefix(sourceUrl: string, sampleTag: string): Promise<string> {
    if (this.releaseTagPrefixCache.has(sourceUrl)) {
      return this.releaseTagPrefixCache.get(sourceUrl)!
    }
    try {
      const url = new URL(sourceUrl)
      if (url.hostname !== 'github.com') {
        this.releaseTagPrefixCache.set(sourceUrl, '')
        return ''
      }
      const cleanPath = url.pathname.replace(/\.git$/, '').replace(/\/$/, '')
      const strippedTag = sampleTag.replace(/^v/, '')
      const vTag = `v${strippedTag}`
      // Try both variants against GitHub's API — the one that 200s tells us the convention
      // Try v-prefixed first since it's more common
      const vRes = await this.fetchWithRetry(
        `https://api.github.com/repos${cleanPath}/releases/tags/${vTag}`,
        { headers: { Accept: 'application/vnd.github.v3+json', 'User-Agent': 'ProjectNomad' } },
        1
      )
      if (vRes.ok) {
        this.releaseTagPrefixCache.set(sourceUrl, 'v')
        return 'v'
      }
      const plainRes = await this.fetchWithRetry(
        `https://api.github.com/repos${cleanPath}/releases/tags/${strippedTag}`,
        { headers: { Accept: 'application/vnd.github.v3+json', 'User-Agent': 'ProjectNomad' } },
        1
      )
      if (plainRes.ok) {
        this.releaseTagPrefixCache.set(sourceUrl, '')
        return ''
      }
    } catch {
      // On error, fall through to default
    }
    // Default: no prefix modification
    this.releaseTagPrefixCache.set(sourceUrl, '')
    return ''
  }
  /**
   * Build a release URL for a specific tag given a source repository URL and
   * the detected release tag prefix convention.
   * Supports GitHub and GitLab URL patterns.
   */
  buildReleaseUrl(sourceUrl: string, tag: string, releaseTagPrefix: string): string | undefined {
    try {
      const url = new URL(sourceUrl)
      if (url.hostname === 'github.com' || url.hostname.includes('gitlab')) {
        const cleanPath = url.pathname.replace(/\.git$/, '').replace(/\/$/, '')
        const strippedTag = tag.replace(/^v/, '')
        const releaseTag = releaseTagPrefix ? `${releaseTagPrefix}${strippedTag}` : strippedTag
        return `${url.origin}${cleanPath}/releases/tag/${releaseTag}`
      }
    } catch {
      // Invalid URL, skip
    }
    return undefined
  }
  /**
   * Filter and sort tags to find compatible updates for a service.
   */
  filterCompatibleUpdates(
    tags: string[],
    currentTag: string,
    majorVersion: number
  ): string[] {
    return tags
      .filter((tag) => {
        // Must match semver pattern
        if (!SEMVER_TAG_PATTERN.test(tag)) return false
        // Reject known non-version tags
        if (REJECTED_TAGS.has(tag.toLowerCase())) return false
        // Reject platform suffixes
        if (PLATFORM_SUFFIXES.some((suffix) => tag.toLowerCase().endsWith(suffix))) return false
        // Must be same major version
        if (parseMajorVersion(tag) !== majorVersion) return false
        // Must be newer than current
        return isNewerVersion(tag, currentTag)
      })
      .sort((a, b) => (isNewerVersion(a, b) ? -1 : 1)) // Newest first
  }
  /**
   * High-level method to get available updates for a service.
   * Returns a sorted list of compatible newer versions (newest first).
   */
  async getAvailableUpdates(
    containerImage: string,
    hostArch: string,
    fallbackSourceRepo?: string | null
  ): Promise<AvailableUpdate[]> {
    const parsed = this.parseImageReference(containerImage)
    const currentTag = parsed.tag
    if (currentTag === 'latest') {
      logger.warn(
        `[ContainerRegistryService] Cannot check updates for ${containerImage} — using :latest tag`
      )
      return []
    }
    const majorVersion = parseMajorVersion(currentTag)
    // Fetch tags and source URL in parallel
    const [tags, ociSourceUrl] = await Promise.all([
      this.listTags(parsed),
      this.getSourceUrl(parsed),
    ])
    // OCI label takes precedence, fall back to DB-stored source_repo
    const sourceUrl = ociSourceUrl || fallbackSourceRepo || null
    const compatible = this.filterCompatibleUpdates(tags, currentTag, majorVersion)
    // Detect release tag prefix convention (e.g. 'v' vs no prefix) if we have a source URL
    let releaseTagPrefix = ''
    if (sourceUrl) {
      releaseTagPrefix = await this.detectReleaseTagPrefix(sourceUrl, currentTag)
    }
    // Check architecture support for the top candidates (limit checks to save API calls)
    const maxArchChecks = 10
    const results: AvailableUpdate[] = []
    for (const tag of compatible.slice(0, maxArchChecks)) {
      const supported = await this.checkArchSupport(parsed, tag, hostArch)
      if (supported) {
        results.push({
          tag,
          isLatest: results.length === 0,
          releaseUrl: sourceUrl ? this.buildReleaseUrl(sourceUrl, tag, releaseTagPrefix) : undefined,
        })
      }
    }
    // For remaining tags (beyond arch check limit), include them but mark as not latest
    for (const tag of compatible.slice(maxArchChecks)) {
      results.push({
        tag,
        isLatest: false,
        releaseUrl: sourceUrl ? this.buildReleaseUrl(sourceUrl, tag, releaseTagPrefix) : undefined,
      })
    }
    return results
  }
  /**
   * Fetch with retry and exponential backoff for rate limiting.
   */
  private async fetchWithRetry(
    url: string,
    init?: RequestInit,
    maxRetries = 3
  ): Promise<Response> {
    for (let attempt = 0; attempt <= maxRetries; attempt++) {
      const response = await fetch(url, init)
      if (response.status === 429 && attempt < maxRetries) {
        const retryAfter = response.headers.get('retry-after')
        const delay = retryAfter
          ? parseInt(retryAfter, 10) * 1000
          : Math.pow(2, attempt) * 1000
        logger.warn(
          `[ContainerRegistryService] Rate limited on ${url}, retrying in ${delay}ms`
        )
        await new Promise((resolve) => setTimeout(resolve, delay))
        continue
      }
      return response
    }
    throw new Error(`Failed to fetch ${url} after ${maxRetries} retries`)
  }
 }
--- a/admin/app/services/content_auto_update_service.ts
+++ b/admin/app/services/content_auto_update_service.ts
@ -1,599 +0,0 @@
 import logger from '@adonisjs/core/services/logger'
 import { DateTime } from 'luxon'
 import KVStore from '#models/kv_store'
 import InstalledResource from '#models/installed_resource'
 import { DownloadService } from '#services/download_service'
 import { CollectionUpdateService } from '#services/collection_update_service'
 import { DrugReferenceService } from '#services/drug_reference_service'
 import {
  KiwixCatalogService,
  reconcileResourceUpdateState,
  type CatalogResult,
 } from '#services/kiwix_catalog_service'
 import { isWithinWindow, parseWindowMinutes } from '../utils/update_window.js'
 import { recordResourceUpdateFailure } from '../utils/content_auto_update_backoff.js'
 import type { Blocker, PreflightResult } from '../utils/image_disk_preflight.js'
 /**
 * Content auto-update is opt-in via a single global master switch and runs on
 * its OWN window + per-window data cap (deliberately separate from the core/app
 * `autoUpdate.*` window, since ZIM downloads are multi-GB and bandwidth
 * sensitive). Defaults err toward an overnight window with no cap; the UI
 * strongly recommends setting a cap.
 */
 const DEFAULT_WINDOW_START = '02:00'
 const DEFAULT_WINDOW_END = '05:00'
 const DEFAULT_COOLOFF_HOURS = 72
 /** Whole-feature failures (e.g. catalog unreachable) before it self-disables. */
 const MAX_FEATURE_FAILURES = 3
 export interface ContentAutoUpdateConfig {
  /** Global master switch (`contentAutoUpdate.enabled`). */
  enabled: boolean
  windowStart: string
  windowEnd: string
  cooloffHours: number
  /** Max NEW bytes initiated per window instance. 0 = unlimited. */
  maxBytesPerWindow: number
 }
 /** Per-resource eligibility verdict (drives both selection and the status UI). */
 export interface ContentEligibility {
  eligible: boolean
  reason: string
  cooloffRemainingHours: number | null
 }
 /** An eligible resource paired with the catalog facts needed to download it. */
 export interface ContentCandidate {
  resource: InstalledResource
  version: string
  download_url: string
  size_bytes: number
  installed_at: DateTime
 }
 /** Outcome of the cap-bounded greedy selection. */
 export interface ContentSelection {
  selected: ContentCandidate[]
  /** Single files larger than the whole cap — never auto-started (manual only). */
  skippedOversize: ContentCandidate[]
  /** Fit the cap but not this window's remaining budget — retried next window. */
  deferred: ContentCandidate[]
 }
 export interface ContentAutoUpdateResourceStatus {
  resource_id: string
  resource_type: 'zim' | 'map'
  current_version: string
  available_update_version: string | null
  size_bytes: number | null
  eligible: boolean
  reason: string
  cooloff_remaining_hours: number | null
  exceeds_cap: boolean
  consecutive_failures: number
  auto_disabled_reason: string | null
 }
 export interface ContentAutoUpdateStatus extends ContentAutoUpdateConfig {
  withinWindow: boolean
  windowBytesUsed: number
  lastAttemptAt: string | null
  lastResult: string | null
  lastError: string | null
  autoDisabledReason: string | null
  resources: ContentAutoUpdateResourceStatus[]
 }
 /**
 * Decision + safety layer for automatic content (ZIM/map) updates. This is the
 * content-side counterpart to {@link AppAutoUpdateService}: it decides *whether*
 * each installed resource with an available update should be downloaded now
 * (master switch on + in the content window + past cool-off + within the data
 * cap) and then drives the existing manual download path
 * ({@link CollectionUpdateService.applyUpdate} → {@link RunDownloadJob}).
 *
 * It never installs synchronously — it dispatches resumable download jobs and
 * lets the existing job-completion path advance the installed version and
 * rebuild the Kiwix library.
 */
 export class ContentAutoUpdateService {
  constructor(
    private downloadService: DownloadService,
    private catalog: KiwixCatalogService = new KiwixCatalogService(),
    private collectionUpdateService: CollectionUpdateService = new CollectionUpdateService()
  ) {}
  /** Read the master switch plus the content-specific window/cool-off/cap. */
  async getConfig(): Promise<ContentAutoUpdateConfig> {
    const [enabled, windowStart, windowEnd, cooloffHours, maxBytes] = await Promise.all([
      KVStore.getValue('contentAutoUpdate.enabled'),
      KVStore.getValue('contentAutoUpdate.windowStart'),
      KVStore.getValue('contentAutoUpdate.windowEnd'),
      KVStore.getValue('contentAutoUpdate.cooloffHours'),
      KVStore.getValue('contentAutoUpdate.maxBytesPerWindow'),
    ])
    const parsedCooloff = Number(cooloffHours)
    const parsedCap = Number(maxBytes)
    return {
      enabled: enabled ?? false,
      windowStart: windowStart || DEFAULT_WINDOW_START,
      windowEnd: windowEnd || DEFAULT_WINDOW_END,
      // `Number(null) === 0`, so an unset value must fall through to the default
      // rather than silently resolving to a zero cool-off. An explicit 0 is honored.
      cooloffHours:
        cooloffHours !== null && Number.isFinite(parsedCooloff) && parsedCooloff >= 0
          ? parsedCooloff
          : DEFAULT_COOLOFF_HOURS,
      maxBytesPerWindow:
        maxBytes !== null && Number.isFinite(parsedCap) && parsedCap >= 0 ? parsedCap : 0,
    }
  }
  /**
   * Pure per-resource eligibility verdict. A resource is eligible when it has a
   * detected newer version, is not self-disabled, and is past its cool-off
   * (measured from first-detected). Version comparison is a lexicographic
   * compare of the YYYY-MM stamps, which sorts chronologically.
   */
  resourceEligibility(
    resource: InstalledResource,
    cooloffHours: number,
    now: DateTime
  ): ContentEligibility {
    if (!resource.available_update_version) {
      return { eligible: false, reason: 'Up to date', cooloffRemainingHours: null }
    }
    if (resource.auto_update_disabled_reason) {
      return {
        eligible: false,
        reason: 'Auto-update disabled after repeated failures',
        cooloffRemainingHours: null,
      }
    }
    if (!(resource.available_update_version > resource.version)) {
      return { eligible: false, reason: 'Up to date', cooloffRemainingHours: null }
    }
    if (!resource.available_update_first_seen_at) {
      return { eligible: false, reason: 'Cool-off pending', cooloffRemainingHours: cooloffHours }
    }
    const ageHours = now.diff(resource.available_update_first_seen_at, 'hours').hours
    const remaining = cooloffHours - ageHours
    if (remaining > 0) {
      const rounded = Math.ceil(remaining)
      return {
        eligible: false,
        reason: `In cool-off (${rounded}h remaining)`,
        cooloffRemainingHours: rounded,
      }
    }
    return {
      eligible: true,
      reason: `Eligible → ${resource.available_update_version}`,
      cooloffRemainingHours: 0,
    }
  }
  /**
   * Pure cap-bounded greedy selection. Oldest-installed first (stale content is
   * prioritized), tie-broken smallest-first for predictability.
   *
   * - size unknown (0) → deferred (can't budget safely)
   * - size > the WHOLE cap → skippedOversize (never auto-started; manual only)
   * - size ≤ remaining budget → selected
   * - otherwise → deferred (fits the cap, not this window)
   */
  selectUnderCap(
    candidates: ContentCandidate[],
    capBytes: number,
    usedBytes: number
  ): ContentSelection {
    const cap = capBytes > 0 ? capBytes : Number.POSITIVE_INFINITY
    let remaining = Math.max(0, cap - usedBytes)
    const selected: ContentCandidate[] = []
    const skippedOversize: ContentCandidate[] = []
    const deferred: ContentCandidate[] = []
    const ordered = [...candidates].sort((a, b) => {
      const at = a.installed_at?.toMillis?.() ?? 0
      const bt = b.installed_at?.toMillis?.() ?? 0
      if (at !== bt) return at - bt
      return a.size_bytes - b.size_bytes
    })
    for (const candidate of ordered) {
      if (candidate.size_bytes <= 0) {
        deferred.push(candidate)
      } else if (candidate.size_bytes > cap) {
        skippedOversize.push(candidate)
      } else if (candidate.size_bytes <= remaining) {
        selected.push(candidate)
        remaining -= candidate.size_bytes
      } else {
        deferred.push(candidate)
      }
    }
    return { selected, skippedOversize, deferred }
  }
  /**
   * Run-wide pre-flight: never auto-update content while ANY download is already
   * running. Because content downloads are multi-GB and resumable, an in-flight
   * download from a prior window naturally blocks new starts here — exactly the
   * "let in-flight finish, don't start new" behavior we want. Transient → `skip`.
   */
  async runGlobalPreflight(): Promise<PreflightResult> {
    const blockers: Blocker[] = []
    try {
      const downloads = await this.downloadService.listDownloadJobs()
      const active = downloads.filter(
        (d) => !!d.status && ['waiting', 'active', 'delayed'].includes(d.status)
      )
      if (active.length > 0) {
        blockers.push({ reason: `${active.length} download(s) in progress`, severity: 'skip' })
      }
    } catch (error) {
      const message = error instanceof Error ? error.message : String(error)
      logger.warn(`[ContentAutoUpdateService] Could not check active downloads: ${message}`)
    }
    return { ok: blockers.length === 0, blockers }
  }
  /**
   * Entry point invoked by ContentAutoUpdateJob. Gates on the master switch +
   * window, runs the local catalog check, then downloads as many eligible
   * resources as fit under the per-window data cap.
   */
  async attempt(): Promise<{ started: number; reason: string }> {
    const config = await this.getConfig()
    const now = DateTime.now()
    if (!config.enabled) {
      return { started: 0, reason: 'Content auto-update is disabled' }
    }
    if (!isWithinWindow(config.windowStart, config.windowEnd, now)) {
      const reason = `Outside update window (${config.windowStart}-${config.windowEnd})`
      await this.recordRun(reason)
      return { started: 0, reason }
    }
    try {
      // Reset the per-window budget once per window instance (the cron fires
      // hourly but a window can span several hours).
      await this.maybeResetWindowBudget(config, now)
      // Local catalog check + persist available-update state for every resource.
      // ZIM/map catalog path only — `dataset` resources are excluded here because
      // their freshness key (openFDA `export_date`) and apply path (the drug
      // download/ingest chain) don't fit the catalog-version model. They refresh
      // via `attemptDrugDataset()`, which the same content-update job runs under
      // the same master switch + window.
      const installed = await InstalledResource.query().whereNot('resource_type', 'dataset')
      const latestByKey = await this.catalog.getLatestForResources(
        // `dataset` rows are filtered out above, so the type narrows to ZIM/map.
        installed.map((r) => ({
          resource_id: r.resource_id,
          resource_type: r.resource_type as 'zim' | 'map',
        }))
      )
      for (const resource of installed) {
        const latest = latestByKey.get(`${resource.resource_type}:${resource.resource_id}`) ?? null
        await reconcileResourceUpdateState(resource, latest, now)
      }
      const eligible = installed.filter(
        (r) => this.resourceEligibility(r, config.cooloffHours, now).eligible
      )
      if (eligible.length === 0) {
        await this.recordFeatureSuccess()
        const reason = 'No eligible content updates'
        await this.recordRun(reason)
        return { started: 0, reason }
      }
      const global = await this.runGlobalPreflight()
      if (!global.ok) {
        await this.recordFeatureSuccess()
        const reason = `Pre-flight blocked: ${global.blockers.map((b) => b.reason).join('; ')}`
        await this.recordRun(reason)
        return { started: 0, reason }
      }
      const candidates: ContentCandidate[] = eligible.map((r) => {
        const latest = latestByKey.get(`${r.resource_type}:${r.resource_id}`) as CatalogResult
        return {
          resource: r,
          version: latest.version,
          download_url: latest.download_url,
          size_bytes: r.available_update_size_bytes ?? latest.size_bytes ?? 0,
          installed_at: r.installed_at,
        }
      })
      const usedBytes = await this.getWindowBytesUsed()
      const { selected, skippedOversize, deferred } = this.selectUnderCap(
        candidates,
        config.maxBytesPerWindow,
        usedBytes
      )
      let started = 0
      let failed = 0
      let initiatedBytes = 0
      for (const candidate of selected) {
        const result = await this.collectionUpdateService.applyUpdate(
          {
            resource_id: candidate.resource.resource_id,
            // `dataset` rows are filtered out of `installed` above, so ZIM/map.
            resource_type: candidate.resource.resource_type as 'zim' | 'map',
            installed_version: candidate.resource.version,
            latest_version: candidate.version,
            download_url: candidate.download_url,
            size_bytes: candidate.size_bytes || undefined,
          },
          { auto: true }
        )
        if (result.success) {
          // Success is NOT recorded here: applyUpdate only enqueues a resumable
          // download. The per-resource backoff is cleared once the download
          // actually completes (RunDownloadJob.onComplete) and incremented when it
          // fails terminally (the worker `failed` handler). Recording success on
          // dispatch would reset the counter every window and defeat self-disable.
          initiatedBytes += candidate.size_bytes
          started++
          logger.info(
            `[ContentAutoUpdateService] Started ${candidate.resource.resource_id} → ${candidate.version}`
          )
        } else {
          // A failure to even enqueue is a genuine auto-update failure; no job runs,
          // so no terminal `failed` event will follow — count it here.
          await recordResourceUpdateFailure(candidate.resource, result.error ?? 'dispatch failed')
          failed++
        }
      }
      if (initiatedBytes > 0) {
        await this.addWindowBytesUsed(initiatedBytes)
      }
      const parts = [`${started} started`]
      if (failed) parts.push(`${failed} failed`)
      if (skippedOversize.length) parts.push(`${skippedOversize.length} skipped (exceeds cap)`)
      if (deferred.length) parts.push(`${deferred.length} deferred (over budget)`)
      const reason = parts.join(', ')
      await this.recordFeatureSuccess()
      await this.recordRun(reason)
      logger.info(`[ContentAutoUpdateService] Run complete: ${reason}`)
      return { started, reason }
    } catch (error) {
      const message = error instanceof Error ? error.message : String(error)
      await this.recordFeatureFailure(message)
      await this.recordRun(`Failed: ${message}`)
      logger.error(`[ContentAutoUpdateService] Run failed: ${message}`)
      return { started: 0, reason: `Failed: ${message}` }
    }
  }
  /**
   * Freshness pass for the FDA drug dataset (`resource_type` 'dataset'), run by
   * the same hourly ContentAutoUpdateJob as the ZIM/map `attempt()` and gated on
   * the SAME master switch + window. The dataset's apply path differs from the
   * catalog loop (openFDA `export_date` + the drug download/ingest chain rather
   * than an InstalledResource catalog-version row), so it runs as its own step
   * instead of through `selectUnderCap` — but sharing `contentAutoUpdate.*` means
   * it never updates while content auto-update is off, and refreshes alongside
   * ZIMs and maps when it is on. Isolated so a drug-side failure can't affect the
   * catalog run.
   */
  async attemptDrugDataset(): Promise<{ started: number; reason: string }> {
    const config = await this.getConfig()
    if (!config.enabled) {
      return { started: 0, reason: 'Content auto-update is disabled' }
    }
    if (!isWithinWindow(config.windowStart, config.windowEnd, DateTime.now())) {
      return {
        started: 0,
        reason: `Outside update window (${config.windowStart}-${config.windowEnd})`,
      }
    }
    try {
      const result = await new DrugReferenceService().attemptAutoUpdate()
      return { started: result.started ? 1 : 0, reason: `drug dataset: ${result.reason}` }
    } catch (error) {
      const message = error instanceof Error ? error.message : String(error)
      logger.warn(`[ContentAutoUpdateService] Drug dataset freshness check failed: ${message}`)
      return { started: 0, reason: `drug dataset check failed: ${message}` }
    }
  }
  /**
   * Evaluate what the next run *would* do, without hitting the network,
   * persisting state, or dispatching anything. Operates on the available-update
   * state already persisted by the last check (manual or auto), so run a "Check
   * for Content Updates" first if you want fresh catalog data. Used by the
   * `content-auto-update:dry-run` command.
   */
  async dryRun(
    overrides: {
      now?: DateTime
      forceEnabled?: boolean
      cooloffHours?: number
      windowStart?: string
      windowEnd?: string
      maxBytesPerWindow?: number
      windowBytesUsed?: number
    } = {}
  ): Promise<{
    enabled: boolean
    withinWindow: boolean
    config: ContentAutoUpdateConfig
    eligibleCount: number
    selection: ContentSelection
  }> {
    const base = await this.getConfig()
    const config: ContentAutoUpdateConfig = {
      enabled: overrides.forceEnabled ? true : base.enabled,
      windowStart: overrides.windowStart ?? base.windowStart,
      windowEnd: overrides.windowEnd ?? base.windowEnd,
      cooloffHours: overrides.cooloffHours ?? base.cooloffHours,
      maxBytesPerWindow: overrides.maxBytesPerWindow ?? base.maxBytesPerWindow,
    }
    const now = overrides.now ?? DateTime.now()
    const withinWindow = isWithinWindow(config.windowStart, config.windowEnd, now)
    const pending = await InstalledResource.query()
      .whereNotNull('available_update_version')
      .whereNot('resource_type', 'dataset')
    const eligible = pending.filter(
      (r) => this.resourceEligibility(r, config.cooloffHours, now).eligible
    )
    const candidates: ContentCandidate[] = eligible.map((r) => ({
      resource: r,
      version: r.available_update_version!,
      download_url: '(dry-run)',
      size_bytes: r.available_update_size_bytes ?? 0,
      installed_at: r.installed_at,
    }))
    const usedBytes = overrides.windowBytesUsed ?? (await this.getWindowBytesUsed())
    const selection = this.selectUnderCap(candidates, config.maxBytesPerWindow, usedBytes)
    return {
      enabled: config.enabled,
      withinWindow,
      config,
      eligibleCount: eligible.length,
      selection,
    }
  }
  // ── Per-window budget ─────────────────────────────────────────────────────────
  /** Most-recent window-open boundary as an absolute timestamp (handles wrap). */
  windowStartBoundary(windowStart: string, now: DateTime): DateTime {
    const minutes = parseWindowMinutes(windowStart) ?? 0
    const todayStart = now.startOf('day').plus({ minutes })
    return now >= todayStart ? todayStart : todayStart.minus({ days: 1 })
  }
  /** Reset the window budget exactly once per entry into the window. */
  private async maybeResetWindowBudget(
    config: ContentAutoUpdateConfig,
    now: DateTime
  ): Promise<void> {
    const boundary = this.windowStartBoundary(config.windowStart, now)
    const resetAtRaw = await KVStore.getValue('contentAutoUpdate.windowResetAt')
    const resetAt = resetAtRaw ? DateTime.fromISO(resetAtRaw) : null
    if (!resetAt || !resetAt.isValid || resetAt < boundary) {
      await KVStore.setValue('contentAutoUpdate.windowBytesUsed', '0')
      await KVStore.setValue('contentAutoUpdate.windowResetAt', now.toISO()!)
    }
  }
  private async getWindowBytesUsed(): Promise<number> {
    const raw = await KVStore.getValue('contentAutoUpdate.windowBytesUsed')
    const num = Number(raw)
    return Number.isFinite(num) && num > 0 ? num : 0
  }
  private async addWindowBytesUsed(bytes: number): Promise<void> {
    const used = await this.getWindowBytesUsed()
    await KVStore.setValue('contentAutoUpdate.windowBytesUsed', String(used + bytes))
  }
  // ── Backoff + run recording ───────────────────────────────────────────────────
  // Per-resource backoff lives in ../utils/content_auto_update_backoff.ts so the
  // job-completion path and the worker `failed` handler can share it without an
  // import cycle. The feature-level backoff below stays here.
  /** Clear the feature-level backoff after a clean run. */
  private async recordFeatureSuccess(): Promise<void> {
    await KVStore.setValue('contentAutoUpdate.consecutiveFailures', '0')
    await KVStore.clearValue('contentAutoUpdate.lastError')
  }
  /** Record a whole-feature failure and self-disable the feature at the threshold. */
  private async recordFeatureFailure(reason: string): Promise<void> {
    const raw = await KVStore.getValue('contentAutoUpdate.consecutiveFailures')
    const failures = (Number(raw) || 0) + 1
    await KVStore.setValue('contentAutoUpdate.consecutiveFailures', String(failures))
    await KVStore.setValue('contentAutoUpdate.lastError', reason)
    if (failures >= MAX_FEATURE_FAILURES) {
      await KVStore.setValue('contentAutoUpdate.enabled', false)
      await KVStore.setValue(
        'contentAutoUpdate.autoDisabledReason',
        `Content auto-update disabled after ${failures} consecutive failures. Last error: ${reason}`
      )
      logger.error(
        `[ContentAutoUpdateService] Feature auto-disabled after ${failures} consecutive failures`
      )
    }
  }
  private async recordRun(reason: string): Promise<void> {
    await KVStore.setValue('contentAutoUpdate.lastAttemptAt', DateTime.now().toISO()!)
    await KVStore.setValue('contentAutoUpdate.lastResult', reason)
  }
  // ── Status snapshot ───────────────────────────────────────────────────────────
  /** Full state snapshot for the settings UI (resources with pending updates). */
  async getStatus(): Promise<ContentAutoUpdateStatus> {
    const config = await this.getConfig()
    const now = DateTime.now()
    const pending = await InstalledResource.query()
      .whereNotNull('available_update_version')
      .whereNot('resource_type', 'dataset')
    const resources: ContentAutoUpdateResourceStatus[] = pending.map((resource) => {
      const verdict = this.resourceEligibility(resource, config.cooloffHours, now)
      const size = resource.available_update_size_bytes ?? null
      const exceedsCap =
        config.maxBytesPerWindow > 0 && size !== null && size > config.maxBytesPerWindow
      return {
        resource_id: resource.resource_id,
        // `dataset` rows are filtered out of `pending` above, so ZIM/map.
        resource_type: resource.resource_type as 'zim' | 'map',
        current_version: resource.version,
        available_update_version: resource.available_update_version,
        size_bytes: size,
        eligible: verdict.eligible && !exceedsCap,
        reason: exceedsCap ? 'Exceeds data cap — update manually' : verdict.reason,
        cooloff_remaining_hours: verdict.cooloffRemainingHours,
        exceeds_cap: exceedsCap,
        consecutive_failures: resource.auto_update_consecutive_failures || 0,
        auto_disabled_reason: resource.auto_update_disabled_reason,
      }
    })
    const [lastAttemptAt, lastResult, lastError, autoDisabledReason, windowBytesUsed] =
      await Promise.all([
        KVStore.getValue('contentAutoUpdate.lastAttemptAt'),
        KVStore.getValue('contentAutoUpdate.lastResult'),
        KVStore.getValue('contentAutoUpdate.lastError'),
        KVStore.getValue('contentAutoUpdate.autoDisabledReason'),
        this.getWindowBytesUsed(),
      ])
    return {
      ...config,
      withinWindow: isWithinWindow(config.windowStart, config.windowEnd, now),
      windowBytesUsed,
      lastAttemptAt: lastAttemptAt || null,
      lastResult: lastResult || null,
      lastError: lastError || null,
      autoDisabledReason: autoDisabledReason || null,
      resources,
    }
  }
 }
--- a/admin/app/services/countries_service.ts
+++ b/admin/app/services/countries_service.ts
@ -1,308 +0,0 @@
 import { access, readFile, writeFile, mkdir } from 'fs/promises'
 import { join, resolve } from 'path'
 import { createHash } from 'crypto'
 import { tmpdir } from 'os'
 import logger from '@adonisjs/core/services/logger'
 import type { Country, CountryCode, CountryGroup } from '../../types/maps.js'
 interface NEFeature {
  type: 'Feature'
  properties: Record<string, any>
  geometry: unknown
 }
 interface NEFeatureCollection {
  type: 'FeatureCollection'
  features: NEFeature[]
 }
 const COUNTRY_GEOJSON_PATH = join(
  process.cwd(),
  'resources',
  'geodata',
  'ne_50m_admin_0_countries.geojson'
 )
 // Natural Earth country polygons are land-only (no territorial waters), so a
 // strict intersect leaves tiles fully over the ocean out of the extract —
 // coastal cities render as grey off their coast. Inflate each polygon outward
 // by ~11 km to pull in adjacent tiles without ballooning the extract size.
 const REGION_BUFFER_DEGREES = 0.1
 const GROUP_ORDER = [
  'north-america',
  'south-america',
  'europe',
  'africa',
  'asia',
  'oceania',
 ]
 const GROUP_META: Record<string, { id: string; name: string; description: string }> = {
  'North America': {
    id: 'north-america',
    name: 'North America',
    description: 'All countries in North America and the Caribbean.',
  },
  'South America': {
    id: 'south-america',
    name: 'South America',
    description: 'All countries in South America.',
  },
  Europe: {
    id: 'europe',
    name: 'Europe',
    description: 'All countries in Europe.',
  },
  Africa: {
    id: 'africa',
    name: 'Africa',
    description: 'All countries in Africa.',
  },
  Asia: {
    id: 'asia',
    name: 'Asia',
    description: 'All countries in Asia.',
  },
  Oceania: {
    id: 'oceania',
    name: 'Oceania',
    description: 'Australia, New Zealand, and Pacific island nations.',
  },
 }
 export class CountriesService {
  private static instance: CountriesService | null = null
  private loadPromise: Promise<void> | null = null
  private countries: Country[] = []
  private byCode: Map<CountryCode, { country: Country; feature: NEFeature }> = new Map()
  private groups: CountryGroup[] = []
  static getInstance(): CountriesService {
    if (!this.instance) {
      this.instance = new CountriesService()
    }
    return this.instance
  }
  private async ensureLoaded(): Promise<void> {
    if (this.byCode.size > 0) return
    if (!this.loadPromise) {
      this.loadPromise = this.load()
    }
    await this.loadPromise
  }
  private async load(): Promise<void> {
    const raw = await readFile(COUNTRY_GEOJSON_PATH, 'utf8')
    const fc = JSON.parse(raw) as NEFeatureCollection
    // Natural Earth reuses a sovereign state's ISO_A2 for its dependencies
    // (e.g. AU covers both Australia and Australian territories). Sort so the
    // sovereign mainland wins the ISO-code slot, and skip any subsequent
    // same-code dependency — otherwise the "AU" entry ends up being some tiny
    // island territory.
    const sortedFeatures = [...fc.features].sort((a, b) => typeRank(a) - typeRank(b))
    const countries: Country[] = []
    const byCode = new Map<CountryCode, { country: Country; feature: NEFeature }>()
    const groupCodes: Record<string, CountryCode[]> = {}
    for (const feature of sortedFeatures) {
      const p = feature.properties
      const code = resolveIso2(p)
      if (!code) continue
      if (byCode.has(code)) continue
      const continent = typeof p.CONTINENT === 'string' ? p.CONTINENT : 'Other'
      if (continent === 'Antarctica' || continent === 'Seven seas (open ocean)') continue
      const country: Country = {
        code,
        code3: resolveIso3(p) ?? code,
        name: typeof p.NAME === 'string' ? p.NAME : code,
        continent,
        subregion: typeof p.SUBREGION === 'string' ? p.SUBREGION : continent,
        population: typeof p.POP_EST === 'number' ? p.POP_EST : 0,
      }
      countries.push(country)
      byCode.set(code, { country, feature })
      if (GROUP_META[continent]) {
        const groupId = GROUP_META[continent].id
        if (!groupCodes[groupId]) groupCodes[groupId] = []
        groupCodes[groupId].push(code)
      }
    }
    countries.sort((a, b) => a.name.localeCompare(b.name))
    const groups: CountryGroup[] = GROUP_ORDER.flatMap((groupId) => {
      const meta = Object.values(GROUP_META).find((m) => m.id === groupId)
      if (!meta) return []
      const codes = (groupCodes[groupId] ?? []).slice().sort()
      if (codes.length === 0) return []
      return [{ id: meta.id, name: meta.name, description: meta.description, countries: codes }]
    })
    this.countries = countries
    this.byCode = byCode
    this.groups = groups
    logger.info(
      `[CountriesService] Loaded ${countries.length} countries across ${groups.length} groups`
    )
  }
  async list(): Promise<Country[]> {
    await this.ensureLoaded()
    return this.countries
  }
  async listGroups(): Promise<CountryGroup[]> {
    await this.ensureLoaded()
    return this.groups
  }
  /** Throws when a supplied code does not map to a known country. */
  async resolveCodes(codes: CountryCode[]): Promise<CountryCode[]> {
    await this.ensureLoaded()
    const normalized = [...new Set(codes.map((c) => c.toUpperCase()))].sort()
    const unknown = normalized.filter((c) => !this.byCode.has(c))
    if (unknown.length > 0) {
      throw new Error(`Unknown country code(s): ${unknown.join(', ')}`)
    }
    return normalized
  }
  /**
   * Filename is keyed on a hash of the sorted ISO codes + buffer size so
   * repeated calls with the same selection reuse the same path, and bumping
   * the buffer auto-invalidates stale files.
   */
  async writeRegionFile(codes: CountryCode[]): Promise<string> {
    await this.ensureLoaded()
    const resolved = await this.resolveCodes(codes)
    const key = `b${REGION_BUFFER_DEGREES}:${resolved.join(',')}`
    const hash = createHash('sha1').update(key).digest('hex').slice(0, 12)
    const dir = resolve(tmpdir(), 'nomad-pmtiles-regions')
    await mkdir(dir, { recursive: true })
    const filepath = join(dir, `region-${hash}.geojson`)
    try {
      await access(filepath)
      return filepath
    } catch {}
    const fc = {
      type: 'FeatureCollection',
      features: resolved.map((code) => {
        const entry = this.byCode.get(code)!
        return {
          type: 'Feature',
          properties: { iso: code, name: entry.country.name },
          geometry: bufferGeometry(entry.feature.geometry, REGION_BUFFER_DEGREES),
        }
      }),
    }
    await writeFile(filepath, JSON.stringify(fc))
    return filepath
  }
 }
 function typeRank(f: NEFeature): number {
  const t = typeof f.properties.TYPE === 'string' ? f.properties.TYPE : ''
  if (t === 'Sovereign country') return 0
  if (t === 'Country') return 1
  if (t === 'Sovereignty') return 2
  if (t === 'Disputed') return 3
  if (t === 'Dependency') return 4
  return 5
 }
 function resolveIso2(p: Record<string, any>): CountryCode | null {
  // Natural Earth's ISO_A2 sometimes holds political escapes like "CN-TW" for
  // Taiwan or "-99" for countries involved in disputes. Only accept clean
  // 2-letter codes; fall back to ISO_A2_EH (which reliably has the real code).
  const primary = typeof p.ISO_A2 === 'string' ? p.ISO_A2 : null
  if (primary && /^[A-Z]{2}$/i.test(primary)) return primary.toUpperCase()
  const fallback = typeof p.ISO_A2_EH === 'string' ? p.ISO_A2_EH : null
  if (fallback && /^[A-Z]{2}$/i.test(fallback)) return fallback.toUpperCase()
  return null
 }
 /**
 * Inflate each polygon ring outward by `buffer` degrees via per-vertex
 * averaged-normal offset. Not geodesically accurate — but at small buffers
 * (<= 0.2°) it's within a few percent of a proper geodesic buffer at
 * country scale, which is plenty for tile-inclusion purposes.
 */
 function bufferGeometry(geometry: unknown, buffer: number): unknown {
  const geom = geometry as { type: string; coordinates: any }
  if (geom?.type === 'Polygon') {
    return { type: 'Polygon', coordinates: bufferPolygonRings(geom.coordinates, buffer) }
  }
  if (geom?.type === 'MultiPolygon') {
    return {
      type: 'MultiPolygon',
      coordinates: geom.coordinates.map((poly: number[][][]) =>
        bufferPolygonRings(poly, buffer)
      ),
    }
  }
  return geometry
 }
 function bufferPolygonRings(rings: number[][][], buffer: number): number[][][] {
  return rings.map((ring) => bufferRing(ring, buffer))
 }
 function bufferRing(ring: number[][], buffer: number): number[][] {
  if (ring.length < 4) return ring
  const sign = signedArea(ring) > 0 ? 1 : -1
  const n = ring.length - 1
  const out: number[][] = []
  for (let i = 0; i < n; i++) {
    const prev = ring[(i - 1 + n) % n]
    const curr = ring[i]
    const next = ring[(i + 1) % n]
    const e1x = curr[0] - prev[0]
    const e1y = curr[1] - prev[1]
    const e2x = next[0] - curr[0]
    const e2y = next[1] - curr[1]
    const l1 = Math.hypot(e1x, e1y) || 1
    const l2 = Math.hypot(e2x, e2y) || 1
    const n1x = (e1y / l1) * sign
    const n1y = (-e1x / l1) * sign
    const n2x = (e2y / l2) * sign
    const n2y = (-e2x / l2) * sign
    const sumX = n1x + n2x
    const sumY = n1y + n2y
    const sl = Math.hypot(sumX, sumY) || 1
    out.push([curr[0] + (sumX / sl) * buffer, curr[1] + (sumY / sl) * buffer])
  }
  out.push(out[0])
  return out
 }
 function signedArea(ring: number[][]): number {
  let a = 0
  for (let i = 0; i < ring.length - 1; i++) {
    a += ring[i][0] * ring[i + 1][1] - ring[i + 1][0] * ring[i][1]
  }
  return a / 2
 }
 function resolveIso3(p: Record<string, any>): string | null {
  const primary = typeof p.ISO_A3 === 'string' ? p.ISO_A3 : null
  if (primary && primary !== '-99') return primary.toUpperCase()
  const fallback = typeof p.ISO_A3_EH === 'string' ? p.ISO_A3_EH : null
  if (fallback && fallback !== '-99') return fallback.toUpperCase()
  const adm = typeof p.ADM0_A3 === 'string' ? p.ADM0_A3 : null
  if (adm && adm !== '-99') return adm.toUpperCase()
  return null
 }
--- a/admin/app/services/creator_pack_service.ts
+++ b/admin/app/services/creator_pack_service.ts
@ -1,177 +0,0 @@
 import env from '#start/env'
 import logger from '@adonisjs/core/services/logger'
 import { join } from 'node:path'
 import { DockerService } from '#services/docker_service'
 import { ZimService } from '#services/zim_service'
 import { CollectionManifestService } from '#services/collection_manifest_service'
 import { RunDownloadJob } from '#jobs/run_download_job'
 import { ZIM_STORAGE_PATH } from '../utils/fs.js'
 import { SERVICE_NAMES } from '../../constants/service_names.js'
 import type { CreatorPacksSpec, CreatorPackWithStatus } from '../../types/collections.js'
 // ZIMs report as octet-stream from the Worker (Content-Type is set explicitly to
 // application/octet-stream in worker/src/index.js). Mirror the ZimService allowlist.
 const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
 // Default entitlement Worker origin. Overridable via CREATOR_PACKS_WORKER_BASE so
 // we can move to a branded packs.projectnomad.us domain without a code change.
 const DEFAULT_WORKER_BASE = 'https://nomad-packs-worker.chris-556.workers.dev'
 /**
 * Outcome of an install request. Discriminated on `code` so the controller can
 * map each case to an HTTP status without string-matching thrown messages.
 * (See feedback: typed failure codes over ad-hoc Error parsing.)
 */
 export type InstallPackResult =
  | { code: 'dispatched'; filename: string }
  | { code: 'already_installed' }
  | { code: 'already_downloading' }
  | { code: 'not_found' }
  | { code: 'not_configured' }
 export type UninstallPackResult =
  | { code: 'uninstalled'; filename: string }
  | { code: 'not_installed' }
 /**
 * Creator Packs install rail. Diverges from the curated-collections rail in
 * exactly one way: instead of a static manifest URL, the ZIM is fetched from the
 * entitlement Worker with a bearer `Authorization` header. From RunDownloadJob
 * onward everything is the shared rail (resumable download → InstalledResource →
 * Kiwix rebuildFromDisk). Video ZIMs opt out of KB embedding via skip_embedding.
 */
 export class CreatorPackService {
  private dockerService: DockerService
  private manifestService: CollectionManifestService
  constructor(dockerService?: DockerService, manifestService?: CollectionManifestService) {
    this.dockerService = dockerService ?? new DockerService()
    this.manifestService = manifestService ?? new CollectionManifestService()
  }
  private get workerBase(): string {
    return (env.get('CREATOR_PACKS_WORKER_BASE') || DEFAULT_WORKER_BASE).replace(/\/+$/, '')
  }
  /**
   * Whether this build can actually install packs — i.e. the release-injected
   * app key is present. Forks built from source have no key; the UI uses this to
   * HIDE the Creator Packs surfaces entirely rather than show install buttons
   * that would 503. The public catalog still lists, but nothing is installable.
   */
  isConfigured(): boolean {
    return !!env.get('CREATOR_PACKS_APP_KEY')
  }
  /** Stable, per-version download URL. Not signed — the gate is the auth header. */
  packUrl(id: string, version: string): string {
    return `${this.workerBase}/packs/${id}_${version}.zim`
  }
  async listPacksWithStatus(): Promise<CreatorPackWithStatus[]> {
    return this.manifestService.getCreatorPacksWithStatus()
  }
  /**
   * Install a pack by catalog id: ensure Kiwix is present (auto-install if not),
   * then dispatch the gated ZIM download onto the shared rail.
   */
  async installPack(packId: string): Promise<InstallPackResult> {
    const appKey = env.get('CREATOR_PACKS_APP_KEY')
    if (!appKey) {
      logger.error('[CreatorPackService] CREATOR_PACKS_APP_KEY is not set; cannot install packs')
      return { code: 'not_configured' }
    }
    const spec = await this.manifestService.getSpecWithFallback<CreatorPacksSpec>('creator_packs')
    const pack = spec?.packs.find((p) => p.id === packId)
    if (!pack) {
      return { code: 'not_found' }
    }
    const { default: InstalledResource } = await import('#models/installed_resource')
    const existing = await InstalledResource.query()
      .where('resource_type', 'zim')
      .where('resource_id', pack.resource_id)
      .first()
    if (existing && existing.version === pack.version) {
      return { code: 'already_installed' }
    }
    const url = this.packUrl(pack.id, pack.version)
    if (await RunDownloadJob.getActiveByUrl(url)) {
      return { code: 'already_downloading' }
    }
    // Kiwix is a hard prerequisite — it's what serves the pack. Auto-install it if
    // absent (its own preinstall bootstraps a mini-wiki ZIM so the container can
    // start). Best-effort: if the install can't be kicked off we still download the
    // pack so the bytes are on disk; Kiwix hot-reloads it once installed.
    await this.ensureKiwixInstalled()
    const filename = `${pack.id}_${pack.version}.zim`
    const filepath = join(process.cwd(), ZIM_STORAGE_PATH, filename)
    await RunDownloadJob.dispatch({
      url,
      filepath,
      timeout: 30000,
      allowedMimeTypes: ZIM_MIME_TYPES,
      filetype: 'zim',
      title: pack.name,
      totalBytes: pack.size_mb ? pack.size_mb * 1024 * 1024 : undefined,
      requestHeaders: { Authorization: `Bearer ${appKey}` },
      resourceMetadata: {
        resource_id: pack.resource_id,
        version: pack.version,
        collection_ref: 'creator-packs',
        skip_embedding: true,
      },
    })
    logger.info(`[CreatorPackService] Dispatched download for pack ${pack.id} (${filename})`)
    return { code: 'dispatched', filename }
  }
  /**
   * Uninstall an installed pack: delete the ZIM (which via ZimService.delete also
   * removes it from the Kiwix library and clears its InstalledResource row).
   * Uses the INSTALLED version (not the catalog's) so we remove the file that's
   * actually on disk even if a newer version has since been published.
   */
  async uninstallPack(packId: string): Promise<UninstallPackResult> {
    const { default: InstalledResource } = await import('#models/installed_resource')
    const installed = await InstalledResource.query()
      .where('resource_type', 'zim')
      .where('resource_id', packId)
      .first()
    if (!installed) {
      return { code: 'not_installed' }
    }
    const filename = `${packId}_${installed.version}.zim`
    const zimService = new ZimService(this.dockerService)
    await zimService.delete(filename)
    logger.info(`[CreatorPackService] Uninstalled pack ${packId} (${filename})`)
    return { code: 'uninstalled', filename }
  }
  private async ensureKiwixInstalled(): Promise<void> {
    try {
      const kiwixUrl = await this.dockerService.getServiceURL(SERVICE_NAMES.KIWIX)
      if (kiwixUrl) return // already installed and resolvable
      logger.info('[CreatorPackService] Kiwix not installed; auto-installing before pack download')
      const result = await this.dockerService.createContainerPreflight(SERVICE_NAMES.KIWIX)
      if (!result.success) {
        // e.g. "already installing" — not fatal, the pack download proceeds regardless.
        logger.warn(`[CreatorPackService] Kiwix preflight did not start: ${result.message}`)
      }
    } catch (error: any) {
      logger.warn(
        `[CreatorPackService] Kiwix auto-install failed (continuing with pack download): ${error?.message || error}`
      )
    }
  }
 }
--- a/admin/app/services/custom_app_guard.ts
+++ b/admin/app/services/custom_app_guard.ts
@ -1,167 +0,0 @@
 import { dirname, normalize } from 'node:path'
 import env from '#start/env'
 /**
 * Security guardrails for user-defined ("custom app") containers.
 *
 * project-nomad runs containers as host siblings via the mounted Docker socket (DooD), so a
 * misconfigured bind mount or image is a real host-takeover vector. The posture here is
 * "guardrails with warnings": hard-block the genuinely catastrophic, warn-but-allow the merely
 * risky so a trusted admin keeps their power without an easy foot-gun.
 */
 export interface GuardEvaluation {
  /** Hard rejections — the install cannot proceed until these are fixed. */
  blocked: string[]
  /** Advisory warnings — overridable via the "install anyway" force flag. */
  warnings: string[]
 }
 /** Absolute host directories that must never be bind-mounted into a custom container. */
 const SYSTEM_BLOCK_PREFIXES = ['/etc', '/proc', '/sys', '/boot', '/dev', '/run', '/var/run']
 /** Registries we ship curated apps from; anything else is allowed but warned on. */
 const TRUSTED_REGISTRIES = ['docker.io', 'registry-1.docker.io', 'ghcr.io', 'lscr.io', 'quay.io']
 /** Resolve the managed storage root (where bind mounts are expected to live). */
 export function getStorageRoot(): string {
  return normalize(env.get('NOMAD_STORAGE_PATH', '/opt/project-nomad/storage')).replace(/\/+$/, '')
 }
 /** Normalize an absolute path: collapse `..`/`.` segments and strip any trailing slash. */
 function normalizeHostPath(p: string): string {
  return normalize(p).replace(/\/+$/, '') || '/'
 }
 /** True when `child` equals `ancestor` or sits beneath it. */
 function isWithin(child: string, ancestor: string): boolean {
  return child === ancestor || child.startsWith(ancestor + '/')
 }
 /**
 * Evaluate user-supplied bind mounts. Hard-blocks the Docker socket, core system directories,
 * and any mount at or above project-nomad's own install tree (which would expose its code/data).
 * Warns on any host path outside the managed storage root.
 */
 export function evaluateBindMounts(
  volumes: { host_path: string; container_path: string }[]
 ): GuardEvaluation {
  const blocked: string[] = []
  const warnings: string[] = []
  const storageRoot = getStorageRoot()
  // The install tree is the parent of the storage root (e.g. /opt/project-nomad). Mounting it —
  // or any ancestor, up to and including `/` — would hand a container project-nomad's own files.
  const installRoot = dirname(storageRoot)
  for (const { host_path: hostPath, container_path: containerPath } of volumes) {
    const host = normalizeHostPath(hostPath)
    if (!hostPath.startsWith('/')) {
      blocked.push(`Volume host path "${hostPath}" must be an absolute path.`)
      continue
    }
    if (!containerPath.startsWith('/')) {
      blocked.push(`Volume container path "${containerPath}" must be an absolute path.`)
      continue
    }
    // A colon is Docker's bind delimiter (host:container:options). A path containing one would be
    // re-split by Docker into a different mount than the one validated here — reject it outright so
    // the checks below can't be bypassed by a parse-differential. (The validator blocks this too;
    // this keeps the guard self-defending for any caller that skips validation.)
    if (hostPath.includes(':') || containerPath.includes(':')) {
      blocked.push(`Volume paths must not contain a colon (":"): "${hostPath}" → "${containerPath}".`)
      continue
    }
    // The Docker socket is the most dangerous mount of all — full control of the host daemon.
    if (host.endsWith('docker.sock') || /\/docker\.sock$/.test(host)) {
      blocked.push(
        `Mounting the Docker socket ("${hostPath}") is not allowed — it grants full host control.`
      )
      continue
    }
    // Core system directories.
    if (host === '/' || SYSTEM_BLOCK_PREFIXES.some((p) => isWithin(host, p))) {
      blocked.push(`Mounting system directory "${hostPath}" is not allowed.`)
      continue
    }
    // At or above project-nomad's own install tree (covers `/`, `/opt`, `/opt/project-nomad`).
    if (host === installRoot || isWithin(installRoot, host)) {
      blocked.push(
        `Mounting "${hostPath}" would expose project-nomad's own files and is not allowed.`
      )
      continue
    }
    // Anything outside the managed storage root is allowed but flagged.
    if (!isWithin(host, storageRoot)) {
      warnings.push(
        `Volume "${hostPath}" is outside the managed storage root (${storageRoot}). Make sure you trust this image with access to that path.`
      )
    }
  }
  return { blocked, warnings }
 }
 /**
 * Evaluate a Docker image reference. Hard-blocks malformed references; warns on moving tags
 * (`:latest`/untagged) and images from registries outside the trusted set.
 */
 export function evaluateImageReference(image: string): GuardEvaluation {
  const blocked: string[] = []
  const warnings: string[] = []
  const ref = image.trim()
  // Loose validity check: no whitespace/control chars, and a sane character set for an image ref.
  if (!ref || /\s/.test(ref) || !/^[\w./:@-]+$/.test(ref)) {
    blocked.push(`"${image}" is not a valid image reference.`)
    return { blocked, warnings }
  }
  // Split off any digest, then any tag, to inspect the registry and tag.
  const [nameAndTag] = ref.split('@')
  const firstSegment = nameAndTag.split('/')[0]
  const hasRegistryHost =
    nameAndTag.includes('/') && (firstSegment.includes('.') || firstSegment.includes(':'))
  const registry = hasRegistryHost ? firstSegment.split(':')[0] : 'docker.io'
  if (!TRUSTED_REGISTRIES.includes(registry)) {
    warnings.push(
      `Image is from "${registry}", which is outside project-nomad's trusted registries. Only install images you trust.`
    )
  }
  // Determine the tag (ignore a colon that's part of a registry host:port in the first segment).
  const remainder = hasRegistryHost ? nameAndTag.slice(firstSegment.length + 1) : nameAndTag
  const tag = remainder.includes(':') ? remainder.split(':').pop() : undefined
  const hasDigest = ref.includes('@sha256:')
  if (!hasDigest && (!tag || tag === 'latest')) {
    warnings.push(
      `Image "${image}" uses a moving tag (${tag ? ':latest' : 'no tag'}). Pin a specific version for reproducible installs.`
    )
  }
  return { blocked, warnings }
 }
 /** Combine bind-mount and image evaluations into a single result. */
 export function evaluateCustomApp(input: {
  image?: string
  volumes?: { host_path: string; container_path: string }[]
 }): GuardEvaluation {
  const bind = evaluateBindMounts(input.volumes ?? [])
  const img = input.image ? evaluateImageReference(input.image) : { blocked: [], warnings: [] }
  return {
    blocked: [...bind.blocked, ...img.blocked],
    warnings: [...bind.warnings, ...img.warnings],
  }
 }
 /** Default resource caps applied to custom containers unless the user overrides them. */
 export const DEFAULT_MEMORY_MB = 1024
 export const DEFAULT_CPUS = 1
--- a/admin/app/services/docker_service.ts
+++ b/admin/app/services/docker_service.ts
--- a/admin/app/services/docs_service.ts
+++ b/admin/app/services/docs_service.ts
@ -3,24 +3,9 @@ import { streamToString } from '../../util/docs.js'
 import { getFile, getFileStatsIfExists, listDirectoryContentsRecursive } from '../utils/fs.js'
 import path from 'path'
 import InternalServerErrorException from '#exceptions/internal_server_error_exception'
 import logger from '@adonisjs/core/services/logger'
 export class DocsService {
  private docsPath = path.join(process.cwd(), 'docs')
  private static readonly DOC_ORDER: Record<string, number> = {
    'home': 1,
    'getting-started': 2,
    'use-cases': 3,
    'supply-depot-apps': 4,
    'drug-reference': 5,
    'community-add-ons': 6,
    'updates': 7,
    'faq': 8,
    'about': 9,
    'release-notes': 10,
  }
  async getDocs() {
    const contents = await listDirectoryContentsRecursive(this.docsPath)
    const files: Array<{ title: string; slug: string }> = []
@ -35,11 +20,7 @@ export class DocsService {
      }
    }
-    return files.sort((a, b) => {
+    return files.sort((a, b) => a.title.localeCompare(b.title))
      const orderA = DocsService.DOC_ORDER[a.slug] ?? 999
      const orderB = DocsService.DOC_ORDER[b.slug] ?? 999
      return orderA - orderB
    })
  }
  parse(content: string) {
@ -51,13 +32,13 @@ export class DocsService {
      // Filter out attribute-undefined errors which may be caused by emojis and special characters
      const criticalErrors = errors.filter((e) => e.error.id !== 'attribute-undefined')
      if (criticalErrors.length > 0) {
-        logger.error('Markdoc validation errors:', errors.map((e) => JSON.stringify(e.error)).join(', '))
+        console.error('Markdoc validation errors:', errors.map((e) => JSON.stringify(e.error)).join(', '))
        throw new Error('Markdoc validation failed')
      }
      return Markdoc.transform(ast, config)
    } catch (error) {
-      logger.error('Error parsing Markdoc content:', error)
+      console.log('Error parsing Markdoc content:', error)
      throw new InternalServerErrorException(`Error parsing content: ${(error as Error).message}`)
    }
  }
@ -70,19 +51,12 @@ export class DocsService {
      const filename = _filename.endsWith('.md') ? _filename : `${_filename}.md`
-      // Prevent path traversal — resolved path must stay within the docs directory
+      const fileExists = await getFileStatsIfExists(path.join(this.docsPath, filename))
      const basePath = path.resolve(this.docsPath)
      const fullPath = path.resolve(path.join(this.docsPath, filename))
      if (!fullPath.startsWith(basePath + path.sep)) {
        throw new Error('Invalid document slug')
      }
      const fileExists = await getFileStatsIfExists(fullPath)
      if (!fileExists) {
        throw new Error(`File not found: ${filename}`)
      }
-      const fileStream = await getFile(fullPath, 'stream')
+      const fileStream = await getFile(path.join(this.docsPath, filename), 'stream')
      if (!fileStream) {
        throw new Error(`Failed to read file stream: ${filename}`)
      }
@ -93,18 +67,9 @@ export class DocsService {
    }
  }
  private static readonly TITLE_OVERRIDES: Record<string, string> = {
    'faq': 'FAQ',
    'community-add-ons': 'Community Add-Ons',
  }
  private prettify(filename: string) {
    const slug = filename.replace(/\.md$/, '')
    if (DocsService.TITLE_OVERRIDES[slug]) {
      return DocsService.TITLE_OVERRIDES[slug]
    }
    // Remove hyphens, underscores, and file extension
-    const cleaned = slug.replace(/_/g, ' ').replace(/-/g, ' ')
+    const cleaned = filename.replace(/_/g, ' ').replace(/\.md$/, '').replace(/-/g, ' ')
    // Convert to Title Case
    const titleCased = cleaned.replace(/\b\w/g, (char) => char.toUpperCase())
    return titleCased.charAt(0).toUpperCase() + titleCased.slice(1)
@ -150,58 +115,6 @@ export class DocsService {
            class: { type: String }
          }
        },
        table: {
          render: 'Table',
        },
        thead: {
          render: 'TableHead',
        },
        tbody: {
          render: 'TableBody',
        },
        tr: {
          render: 'TableRow',
        },
        th: {
          render: 'TableHeader',
        },
        td: {
          render: 'TableCell',
        },
        paragraph: {
          render: 'Paragraph',
        },
        image: {
          render: 'Image',
          attributes: {
            src: { type: String, required: true },
            alt: { type: String },
            title: { type: String },
          },
        },
        link: {
          render: 'Link',
          attributes: {
            href: { type: String, required: true },
            title: { type: String },
          },
        },
        fence: {
          render: 'CodeBlock',
          attributes: {
            content: { type: String },
            language: { type: String },
          },
        },
        code: {
          render: 'InlineCode',
          attributes: {
            content: { type: String },
          },
        },
        hr: {
          render: 'HorizontalRule',
        },
      },
    }
  }
--- a/admin/app/services/download_service.ts
+++ b/admin/app/services/download_service.ts
@ -1,428 +1,25 @@
 import { inject } from '@adonisjs/core'
 import { QueueService } from './queue_service.js'
 import { RunDownloadJob } from '#jobs/run_download_job'
-import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job'
+import { DownloadJobWithProgress } from '../../types/downloads.js'
 import type { RunExtractPmtilesJobParams } from '#jobs/run_extract_pmtiles_job'
 import { DownloadModelJob } from '#jobs/download_model_job'
 import { DownloadDrugDataJob } from '#jobs/download_drug_data_job'
 import { DownloadJobWithProgress, DownloadProgressData, RunDownloadJobParams } from '../../types/downloads.js'
 import type { Job, Queue } from 'bullmq'
 import { normalize } from 'path'
 import { deleteFileIfExists } from '../utils/fs.js'
 import transmit from '@adonisjs/transmit/services/main'
 import { BROADCAST_CHANNELS } from '../../constants/broadcast.js'
 type FileJobState = 'waiting' | 'active' | 'delayed' | 'failed'
 type TaggedJob = { job: Job; state: FileJobState }
@inject()
 export class DownloadService {
  constructor(private queueService: QueueService) {}
  private parseProgress(progress: any): { percent: number; downloadedBytes?: number; totalBytes?: number; lastProgressTime?: number } {
    if (typeof progress === 'object' && progress !== null && 'percent' in progress) {
      const p = progress as DownloadProgressData
      return {
        percent: p.percent,
        downloadedBytes: p.downloadedBytes,
        totalBytes: p.totalBytes,
        lastProgressTime: p.lastProgressTime,
      }
    }
    // Backward compat: plain integer from in-flight jobs during upgrade
    return { percent: parseInt(String(progress), 10) || 0 }
  }
  /** Fetch all non-completed jobs from a queue, tagged with their current BullMQ state */
  private async fetchJobsWithStates(queueName: string): Promise<TaggedJob[]> {
    const queue = this.queueService.getQueue(queueName)
    const [waiting, active, delayed, failed] = await Promise.all([
      queue.getJobs(['waiting']),
      queue.getJobs(['active']),
      queue.getJobs(['delayed']),
      queue.getJobs(['failed']),
    ])
    return [
      ...waiting.map((j) => ({ job: j, state: 'waiting' as const })),
      ...active.map((j) => ({ job: j, state: 'active' as const })),
      ...delayed.map((j) => ({ job: j, state: 'delayed' as const })),
      ...failed.map((j) => ({ job: j, state: 'failed' as const })),
      // A job id can outlive its payload hash — BullMQ still returns an entry for
      // it, with `data` empty. One of those in the failed set used to throw on
      // every poll of this endpoint (normalize(undefined)), and because failed
      // jobs are never evicted the endpoint stayed broken until Redis was cleared
      // by hand. Drop them: with no payload there is nothing to show anyway.
    ].filter(({ job }) => job?.id != null && job.data != null)
  }
  async listDownloadJobs(filetype?: string): Promise<DownloadJobWithProgress[]> {
-    const modelQueue = this.queueService.getQueue(DownloadModelJob.queue)
+    const queue = this.queueService.getQueue(RunDownloadJob.queue)
-    const [fileTagged, extractTagged, modelJobs, drugTagged] = await Promise.all([
+    const jobs = await queue.getJobs(['waiting', 'active', 'delayed'])
      this.fetchJobsWithStates(RunDownloadJob.queue),
      this.fetchJobsWithStates(RunExtractPmtilesJob.queue),
      modelQueue.getJobs(['waiting', 'active', 'delayed', 'failed']),
      this.fetchJobsWithStates(DownloadDrugDataJob.queue),
    ])
-    const fileDownloads = fileTagged.map(({ job, state }) => {
+    return jobs
-      const parsed = this.parseProgress(job.progress)
+      .map((job) => ({
      return {
        jobId: job.id!.toString(),
        url: job.data.url,
-        progress: parsed.percent,
+        progress: parseInt(job.progress.toString(), 10),
-        filepath: job.data.filepath ? normalize(job.data.filepath) : '',
+        filepath: normalize(job.data.filepath),
        filetype: job.data.filetype,
-        title: job.data.title || undefined,
+      }))
-        downloadedBytes: parsed.downloadedBytes,
+      .filter((job) => !filetype || job.filetype === filetype)
        totalBytes: parsed.totalBytes || job.data.totalBytes || undefined,
        lastProgressTime: parsed.lastProgressTime,
        status: state,
        failedReason: job.failedReason || undefined,
      }
    })
    const extractDownloads = extractTagged.map(({ job, state }) => {
      const parsed = this.parseProgress(job.progress)
      return {
        jobId: job.id!.toString(),
        url: job.data.sourceUrl,
        progress: parsed.percent,
        filepath: job.data.outputFilepath ? normalize(job.data.outputFilepath) : '',
        filetype: job.data.filetype || 'map',
        title: job.data.title || undefined,
        downloadedBytes: parsed.downloadedBytes,
        totalBytes: parsed.totalBytes || job.data.estimatedBytes || undefined,
        lastProgressTime: parsed.lastProgressTime,
        status: state,
        failedReason: job.failedReason || undefined,
      }
    })
    const modelDownloads = modelJobs.map((job) => ({
      jobId: job.id!.toString(),
      url: job.data.modelName || 'Unknown Model',
      progress: parseInt(job.progress.toString(), 10),
      filepath: job.data.modelName || 'Unknown Model',
      filetype: 'model',
      status: (job.failedReason ? 'failed' : 'active') as 'active' | 'failed',
      failedReason: job.failedReason || undefined,
    }))
    // FDA drug dataset — DOWNLOAD phase only, collapsed to ONE card. The job fans
    // the manifest's N partitions into continuations under AUTO-GENERATED jobIds
    // (only part 0 runs under the deterministic jobId), and the queue is
    // concurrency 1, so at most one part is ever in flight. Filtering to the
    // deterministic jobId would track only part 0 and then drop the card while
    // parts 2..N keep downloading. Instead represent the whole download with the
    // single in-flight job's aggregate progress (the progress emit already spans
    // all parts), and always report the deterministic jobId so the cancel/remove
    // button routes to _cancelDrugDownloadJob whichever part is active. The heavy
    // ingest is EXCLUDED here — it stays on the IngestStatus surface.
    const drugInFlight =
      drugTagged.find(({ state }) => state === 'active') ??
      drugTagged.find(({ state }) => state === 'waiting' || state === 'delayed') ??
      drugTagged.find(({ state }) => state === 'failed')
    const drugDownloads = drugInFlight
      ? [drugInFlight].map(({ job, state }) => {
          const parsed = this.parseProgress(job.progress)
          return {
            jobId: DownloadDrugDataJob.jobId,
            url: 'https://api.fda.gov/download.json',
            progress: parsed.percent,
            filepath: job.data.currentPartName || 'FDA Drug Reference',
            filetype: 'drug-data',
            title: 'FDA Drug Reference',
            downloadedBytes: parsed.downloadedBytes,
            totalBytes: parsed.totalBytes,
            lastProgressTime: parsed.lastProgressTime,
            status: state,
            failedReason: job.failedReason || undefined,
          }
        })
      : []
    const allDownloads = [...fileDownloads, ...extractDownloads, ...modelDownloads, ...drugDownloads]
    const filtered = allDownloads.filter((job) => !filetype || job.filetype === filetype)
    return filtered.sort((a, b) => {
      if (a.status === 'failed' && b.status !== 'failed') return 1
      if (a.status !== 'failed' && b.status === 'failed') return -1
      return b.progress - a.progress
    })
  }
  async removeFailedJob(jobId: string): Promise<void> {
    for (const queueName of [
      RunDownloadJob.queue,
      RunExtractPmtilesJob.queue,
      DownloadModelJob.queue,
      DownloadDrugDataJob.queue,
    ]) {
      const queue = this.queueService.getQueue(queueName)
      const job = await queue.getJob(jobId)
      if (job) {
        try {
          await job.remove()
        } catch {
          // Job may be locked by the worker after cancel. Remove the stale lock and retry.
          try {
            const client = await queue.client
            await client.del(`bull:${queueName}:${jobId}:lock`)
            await job.remove()
          } catch {
            // Last resort: already removed or truly stuck
          }
        }
        return
      }
    }
  }
  async retryFailedJob(jobId: string): Promise<{ success: boolean; message: string }> {
    // Search both the file download queue and the model download queue
    for (const queueName of [RunDownloadJob.queue, DownloadModelJob.queue]) {
      const queue = this.queueService.getQueue(queueName)
      const job = await queue.getJob(jobId)
      if (job) {
        // For Ollama model downloads, re-dispatch with the model name
        if (queueName === DownloadModelJob.queue) {
          const modelName = job.data.modelName
          if (!modelName) {
            return { success: false, message: 'Cannot retry: model name not found in job data' }
          }
          await DownloadModelJob.dispatch({ modelName })
          await job.remove().catch(() => {})
          return { success: true, message: `Retrying download for model ${modelName}` }
        }
        // For file downloads (zim, map, etc.), re-dispatch with original params
        const params = job.data as RunDownloadJobParams
        if (!params.url || !params.filepath) {
          return { success: false, message: 'Cannot retry: missing URL or filepath in job data' }
        }
        // Remove the old failed job, then dispatch a fresh one
        await job.remove().catch(() => {})
        await RunDownloadJob.dispatch(params)
        return { success: true, message: `Retrying download for ${params.url}` }
      }
    }
    return { success: false, message: 'Failed job not found. It may have already been dismissed.' }
  }
  async cancelJob(jobId: string): Promise<{ success: boolean; message: string }> {
    const queue = this.queueService.getQueue(RunDownloadJob.queue)
    const job = await queue.getJob(jobId)
    if (job) {
      return await this._cancelFileDownloadJob(jobId, job, queue)
    }
    const extractQueue = this.queueService.getQueue(RunExtractPmtilesJob.queue)
    const extractJob = await extractQueue.getJob(jobId)
    if (extractJob) {
      return await this._cancelExtractJob(jobId, extractJob, extractQueue)
    }
    const modelQueue = this.queueService.getQueue(DownloadModelJob.queue)
    const modelJob = await modelQueue.getJob(jobId)
    if (modelJob) {
      return await this._cancelModelDownloadJob(jobId, modelJob, modelQueue)
    }
    // FDA drug dataset: cancel is matched on the deterministic jobId only (the
    // single card the aggregator shows). The continuation parts run under
    // auto-generated ids, so cancelling must stop the whole CHAIN, not just the
    // current part — otherwise the next continuation fires after we remove one.
    if (jobId === DownloadDrugDataJob.jobId) {
      return await this._cancelDrugDownloadJob()
    }
    return { success: true, message: 'Job not found (may have already completed)' }
  }
  /**
   * Cancel the FDA drug-data download.
   *
   * The drug job has no Redis cancel-signal / AbortController (unlike
   * RunDownloadJob), and it self-continues into the next part under a fresh jobId.
   * So a v1 cancel that only removed the current job would leave the next
   * continuation to fire. Instead we obliterate the single-purpose drug-download
   * queue (force = removes the active/locked job too), which drops the current
   * part AND every queued continuation in one shot. Scoped to the drug-download
   * queue only — the ingest queue and everything else are untouched. The on-disk
   * parts are intentionally LEFT in place: they're resumable, and a re-trigger
   * picks up from the .tmp rather than re-downloading. (Tracked as the v1 choice
   * for cancel depth — full cross-process signal cancel is a follow-up.)
   */
  private async _cancelDrugDownloadJob(): Promise<{ success: boolean; message: string }> {
    const queue = this.queueService.getQueue(DownloadDrugDataJob.queue)
    try {
      await queue.obliterate({ force: true })
      return { success: true, message: 'Drug data download cancelled' }
    } catch (err) {
      const msg = err instanceof Error ? err.message : String(err)
      return { success: false, message: `Could not cancel drug data download: ${msg}` }
    }
  }
  private async _cancelExtractJob(
    jobId: string,
    job: Job<RunExtractPmtilesJobParams>,
    queue: Queue<RunExtractPmtilesJobParams>
  ): Promise<{ success: boolean; message: string }> {
    const outputFilepath = job.data.outputFilepath
    await RunExtractPmtilesJob.signalCancel(jobId)
    // Same-process fallback when worker and API share a process
    RunExtractPmtilesJob.childProcesses.get(jobId)?.kill('SIGTERM')
    RunExtractPmtilesJob.childProcesses.delete(jobId)
    await this._pollForTerminalState(job, jobId)
    await this._removeJobWithLockFallback(job, queue, RunExtractPmtilesJob.queue, jobId)
    if (outputFilepath) {
      try {
        await deleteFileIfExists(outputFilepath)
      } catch {
        // File may not exist yet (subprocess may not have opened it)
      }
    }
    return { success: true, message: 'Extract cancelled and partial file deleted' }
  }
  /** Cancel a content download (zim, map, pmtiles, etc.) */
  private async _cancelFileDownloadJob(
    jobId: string,
    job: any,
    queue: any
  ): Promise<{ success: boolean; message: string }> {
    const filepath = job.data.filepath
    // Signal the worker process to abort the download via Redis
    await RunDownloadJob.signalCancel(jobId)
    // Also try in-memory abort (works if worker is in same process)
    RunDownloadJob.abortControllers.get(jobId)?.abort('user-cancel')
    RunDownloadJob.abortControllers.delete(jobId)
    await this._pollForTerminalState(job, jobId)
    await this._removeJobWithLockFallback(job, queue, RunDownloadJob.queue, jobId)
    // Delete the partial file from disk
    if (filepath) {
      try {
        await deleteFileIfExists(filepath)
        // Also try .tmp in case PR #448 staging is merged
        await deleteFileIfExists(filepath + '.tmp')
      } catch {
        // File may not exist yet (waiting job)
      }
    }
    // If this was a Wikipedia download, update selection status to failed
    // (the worker's failed event may not fire if we removed the job first)
    if (job.data.filetype === 'zim' && job.data.url?.includes('wikipedia_en_')) {
      try {
        const { DockerService } = await import('#services/docker_service')
        const { ZimService } = await import('#services/zim_service')
        const dockerService = new DockerService()
        const zimService = new ZimService(dockerService)
        await zimService.onWikipediaDownloadComplete(job.data.url, false)
      } catch {
        // Best effort
      }
    }
    return { success: true, message: 'Download cancelled and partial file deleted' }
  }
  /** Cancel an Ollama model download — mirrors the file cancel pattern but skips file cleanup */
  private async _cancelModelDownloadJob(
    jobId: string,
    job: any,
    queue: any
  ): Promise<{ success: boolean; message: string }> {
    const modelName: string = job.data?.modelName ?? 'unknown'
    // Signal the worker process to abort the pull via Redis
    await DownloadModelJob.signalCancel(jobId)
    // Also try in-memory abort (works if worker is in same process)
    DownloadModelJob.abortControllers.get(jobId)?.abort('user-cancel')
    DownloadModelJob.abortControllers.delete(jobId)
    await this._pollForTerminalState(job, jobId)
    await this._removeJobWithLockFallback(job, queue, DownloadModelJob.queue, jobId)
    // Broadcast a cancelled event so the frontend hook clears the entry. We use percent: -2
    // (distinct from -1 = error) so the hook can route it to a 2s auto-clear instead of the
    // 15s error display. The frontend ALSO removes the entry optimistically from the API
    // response, so this is belt-and-suspenders for cases where the SSE arrives first.
    transmit.broadcast(BROADCAST_CHANNELS.OLLAMA_MODEL_DOWNLOAD, {
      model: modelName,
      jobId,
      percent: -2,
      status: 'cancelled',
      timestamp: new Date().toISOString(),
    })
    // Note on partial blob cleanup: Ollama manages model blobs internally at
    // /root/.ollama/models/blobs/. We deliberately do NOT call /api/delete here — Ollama's
    // expected behavior is to retain partial blobs so a re-pull resumes from where it left
    // off. If the user wants to reclaim that space, they can re-pull and let it complete,
    // or delete the partially-downloaded model from the AI Settings page.
    return { success: true, message: 'Model download cancelled' }
  }
  /** Wait up to 4s (250ms intervals) for the job to reach a terminal state */
  private async _pollForTerminalState(job: any, jobId: string): Promise<void> {
    const POLL_INTERVAL_MS = 250
    const POLL_TIMEOUT_MS = 4000
    const deadline = Date.now() + POLL_TIMEOUT_MS
    while (Date.now() < deadline) {
      await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS))
      try {
        const state = await job.getState()
        if (state === 'failed' || state === 'completed' || state === 'unknown') {
          return
        }
      } catch {
        return // getState() throws if job is already gone
      }
    }
    console.warn(
      `[DownloadService] cancelJob: job ${jobId} did not reach terminal state within timeout, removing anyway`
    )
  }
  /** Remove a BullMQ job, clearing a stale worker lock if the first attempt fails */
  private async _removeJobWithLockFallback(
    job: any,
    queue: any,
    queueName: string,
    jobId: string
  ): Promise<void> {
    try {
      await job.remove()
    } catch {
      // Lock contention fallback: clear lock and retry once
      try {
        const client = await queue.client
        await client.del(`bull:${queueName}:${jobId}:lock`)
        const updatedJob = await queue.getJob(jobId)
        if (updatedJob) await updatedJob.remove()
      } catch {
        // Best effort - job will be cleaned up on next dismiss attempt
      }
    }
  }
 }
--- a/admin/app/services/drug_reference_service.ts
+++ b/admin/app/services/drug_reference_service.ts
@ -1,830 +0,0 @@
 import db from '@adonisjs/lucid/services/db'
 import logger from '@adonisjs/core/services/logger'
 import { rm } from 'node:fs/promises'
 import type { Job } from 'bullmq'
 import { QueueService } from './queue_service.js'
 import { DownloadDrugDataJob, STORAGE_BASE } from '#jobs/download_drug_data_job'
 import { IngestDrugDataJob } from '#jobs/ingest_drug_data_job'
 /**
 * Manifest resource id for the FDA drug dataset — the `installed_resources`
 * row's `resource_id`, matching the `medicine-standard` tier manifest entry.
 * Single source of truth so the install write-back, the tier-status math, and
 * uninstall all agree on the same id.
 */
 export const DRUG_DATASET_RESOURCE_ID = 'openfda-drug-labels'
 import {
  normalizeDrugName,
  parseDownloadState,
  deriveIngestPhase,
  resolveExpectedTotal,
  resolveIngestRecordsShown,
  summarizeJobError,
  isExportDateNewer,
 } from '../../util/drug_labels.js'
 import KVStore from '#models/kv_store'
 import { parseCompareIds, MAX_COMPARE } from '../../util/compare_ids.js'
 import type {
  DrugSearchResult,
  DrugLabelDetail,
  DrugIngestStatus,
  DrugPhaseState,
  DrugDownloadStatus,
  DrugIngestPhaseStatus,
  DrugInteractionEntry,
 } from '../../types/drug_reference.js'
 /**
 * Drug Reference v1 — service layer.
 *
 * Exposes search (collapsed by brand+generic), detail fetch, ingest trigger,
 * and ingest status. Mirrors the shape of DownloadService/ZimService.
 */
 export class DrugReferenceService {
  /**
   * Search for drug labels, collapsed by (brand_name, generic_name).
   *
   * Each distinct (brand_name, generic_name) pair returns ONE result — a
   * representative row id (MIN(id)) and a labelCount of how many set_ids
   * collapsed into it. This is the locked UX decision.
   *
   * Scope:
   *   'name'       (default) — existing path: MATCH(searchable_name) on brand+generic.
   *   'indication' — new path: MATCH(searchable_name, indications) on the combined
   *                  ft_drug_labels_name_indications index so users can search by
   *                  what a drug treats ("heartburn", "high blood pressure").
   *
   * Strategy (both scopes):
   *   1. FULLTEXT path: MATCH(cols) AGAINST(? IN NATURAL LANGUAGE MODE)
   *      — relevance-ranked, requires >= 3 chars (innodb_ft_min_token_size = 3).
   *   2. LIKE fallback: query < 3 chars OR FULLTEXT throws → LIKE '%term%'.
   *   3. Both paths apply the optional product_type filter and GROUP BY collapse.
   */
  async search(
    query: string,
    options: {
      productType?: string
      route?: string
      sort?: 'relevance' | 'name'
      limit?: number
      offset?: number
      scope?: 'name' | 'indication'
    }
  ): Promise<DrugSearchResult[]> {
    const limit = options.limit ?? 50
    const offset = options.offset ?? 0
    const scope = options.scope ?? 'name'
    const normalized = normalizeDrugName(query, null) ?? query.trim()
    if (!normalized || normalized.length === 0) return []
    const useLike = normalized.length < 3
    if (scope === 'indication') {
      if (!useLike) {
        try {
          return await this.searchIndicationFulltext(normalized, options.productType, limit, offset)
        } catch (err) {
          logger.warn(
            `[DrugReferenceService] FULLTEXT indication search failed, falling back to LIKE: ${
              err instanceof Error ? err.message : String(err)
            }`
          )
        }
      }
      return await this.searchIndicationLike(normalized, options.productType, limit, offset)
    }
    if (!useLike) {
      // FULLTEXT path
      try {
        return await this.searchFulltext(normalized, options, limit, offset)
      } catch (err) {
        logger.warn(
          `[DrugReferenceService] FULLTEXT search failed, falling back to LIKE: ${
            err instanceof Error ? err.message : String(err)
          }`
        )
      }
    }
    // LIKE fallback
    return await this.searchLike(normalized, options, limit, offset)
  }
  private async searchFulltext(
    normalized: string,
    opts: { productType?: string; route?: string; sort?: 'relevance' | 'name' },
    limit: number,
    offset: number
  ): Promise<DrugSearchResult[]> {
    let sql = `
      SELECT
        MIN(id) AS id,
        brand_name,
        generic_name,
        MIN(manufacturer) AS manufacturer,
        MIN(route) AS route,
        MIN(product_type) AS product_type,
        COUNT(*) AS labelCount,
        MAX(MATCH(searchable_name) AGAINST(? IN NATURAL LANGUAGE MODE)) AS relevance
      FROM drug_labels
      WHERE MATCH(searchable_name) AGAINST(? IN NATURAL LANGUAGE MODE)
    `
    const bindings: unknown[] = [normalized, normalized]
    if (opts.productType) {
      sql += ' AND product_type = ?'
      bindings.push(opts.productType)
    }
    if (opts.route) {
      // `route` is a comma-joined uppercase list (e.g. "ORAL, TOPICAL").
      sql += ' AND route LIKE ?'
      bindings.push(`%${opts.route.toUpperCase()}%`)
    }
    sql += `
      GROUP BY brand_name, generic_name
      ORDER BY ${opts.sort === 'name' ? 'COALESCE(brand_name, generic_name) ASC' : 'relevance DESC'}
      LIMIT ? OFFSET ?
    `
    bindings.push(limit, offset)
    const rows = await db.rawQuery(sql, bindings)
    return this.mapSearchRows(rows[0])
  }
  private async searchLike(
    normalized: string,
    opts: { productType?: string; route?: string; sort?: 'relevance' | 'name' },
    limit: number,
    offset: number
  ): Promise<DrugSearchResult[]> {
    const term = `%${normalized}%`
    let sql = `
      SELECT
        MIN(id) AS id,
        brand_name,
        generic_name,
        MIN(manufacturer) AS manufacturer,
        MIN(route) AS route,
        MIN(product_type) AS product_type,
        COUNT(*) AS labelCount
      FROM drug_labels
      WHERE (searchable_name LIKE ? OR brand_name LIKE ?)
    `
    const bindings: unknown[] = [term, term]
    if (opts.productType) {
      sql += ' AND product_type = ?'
      bindings.push(opts.productType)
    }
    if (opts.route) {
      sql += ' AND route LIKE ?'
      bindings.push(`%${opts.route.toUpperCase()}%`)
    }
    sql += `
      GROUP BY brand_name, generic_name
      ORDER BY brand_name ASC
      LIMIT ? OFFSET ?
    `
    bindings.push(limit, offset)
    const rows = await db.rawQuery(sql, bindings)
    return this.mapSearchRows(rows[0])
  }
  /**
   * FULLTEXT indication-scope search.
   *
   * MATCHes over (searchable_name, indications) — must exactly match the
   * ft_drug_labels_name_indications index column list. The MAX(MATCH …) pattern
   * is LOAD-BEARING: MySQL 8.0 ONLY_FULL_GROUP_BY rejects a bare MATCH() in
   * SELECT when GROUP BY is in effect; wrapping in MAX() makes it an aggregate
   * and satisfies the mode constraint.
   */
  private async searchIndicationFulltext(
    normalized: string,
    productType: string | undefined,
    limit: number,
    offset: number
  ): Promise<DrugSearchResult[]> {
    let sql = `
      SELECT
        MIN(id) AS id,
        brand_name,
        generic_name,
        MIN(manufacturer) AS manufacturer,
        MIN(route) AS route,
        MIN(product_type) AS product_type,
        COUNT(*) AS labelCount,
        MAX(MATCH(searchable_name, indications) AGAINST(? IN NATURAL LANGUAGE MODE)) AS relevance
      FROM drug_labels
      WHERE MATCH(searchable_name, indications) AGAINST(? IN NATURAL LANGUAGE MODE)
    `
    const bindings: unknown[] = [normalized, normalized]
    if (productType) {
      sql += ' AND product_type = ?'
      bindings.push(productType)
    }
    sql += `
      GROUP BY brand_name, generic_name
      ORDER BY relevance DESC
      LIMIT ? OFFSET ?
    `
    bindings.push(limit, offset)
    const rows = await db.rawQuery(sql, bindings)
    return this.mapSearchRows(rows[0])
  }
  /**
   * LIKE indication-scope fallback (query < 3 chars or FULLTEXT unavailable).
   *
   * Searches searchable_name OR indications so short queries still return
   * useful results without requiring the FULLTEXT index.
   */
  private async searchIndicationLike(
    normalized: string,
    productType: string | undefined,
    limit: number,
    offset: number
  ): Promise<DrugSearchResult[]> {
    const term = `%${normalized}%`
    let sql = `
      SELECT
        MIN(id) AS id,
        brand_name,
        generic_name,
        MIN(manufacturer) AS manufacturer,
        MIN(route) AS route,
        MIN(product_type) AS product_type,
        COUNT(*) AS labelCount
      FROM drug_labels
      WHERE (searchable_name LIKE ? OR indications LIKE ?)
    `
    const bindings: unknown[] = [term, term]
    if (productType) {
      sql += ' AND product_type = ?'
      bindings.push(productType)
    }
    sql += `
      GROUP BY brand_name, generic_name
      ORDER BY brand_name ASC
      LIMIT ? OFFSET ?
    `
    bindings.push(limit, offset)
    const rows = await db.rawQuery(sql, bindings)
    return this.mapSearchRows(rows[0])
  }
  private mapSearchRows(rows: any[]): DrugSearchResult[] {
    if (!Array.isArray(rows)) return []
    return rows.map((row) => ({
      id: Number(row.id),
      brand_name: row.brand_name ?? null,
      generic_name: row.generic_name ?? null,
      manufacturer: row.manufacturer ?? null,
      route: row.route ?? null,
      product_type: row.product_type ?? null,
      labelCount: Number(row.labelCount ?? row.labelcount ?? 1),
    }))
  }
  /**
   * Load the full detail for a single drug label row by its surrogate id.
   * Returns null if the row doesn't exist.
   */
  async find(id: number): Promise<DrugLabelDetail | null> {
    const { default: DrugLabel } = await import('#models/drug_label')
    const row = await DrugLabel.find(id)
    if (!row) return null
    return {
      id: row.id,
      set_id: row.set_id,
      spl_id: row.spl_id,
      version: row.version,
      brand_name: row.brand_name,
      generic_name: row.generic_name,
      manufacturer: row.manufacturer,
      product_ndc: row.product_ndc,
      route: row.route,
      product_type: row.product_type,
      indications: row.indications,
      dosage: row.dosage,
      warnings: row.warnings,
      boxed_warning: row.boxed_warning,
      drug_interactions: row.drug_interactions,
      contraindications: row.contraindications,
      when_using: row.when_using,
      stop_use: row.stop_use,
      source_updated_at: row.source_updated_at,
      ingested_at: row.ingested_at.toISO() ?? '',
    }
  }
  /**
   * Return the drug interaction text for a set of label ids.
   *
   * - Dedupes and caps the id list via parseCompareIds / MAX_COMPARE.
   * - One query: SELECT id, brand_name, generic_name, product_type,
   *   drug_interactions FROM drug_labels WHERE id IN (?).
   * - Re-orders the rows to match the requested id order so the caller's
   *   column positions are stable even if MySQL returns rows in a different
   *   order. Missing ids (non-existent in the table) are silently omitted.
   * - Returns [] for an empty or entirely-invalid id list.
   */
  async getInteractionsFor(ids: number[]): Promise<DrugInteractionEntry[]> {
    if (ids.length === 0) return []
    // Dedupe + cap (caller may already have done this, but be defensive).
    const safeIds = parseCompareIds(ids.join(',')).slice(0, MAX_COMPARE)
    if (safeIds.length === 0) return []
    const placeholders = safeIds.map(() => '?').join(', ')
    const sql = `
      SELECT id, brand_name, generic_name, product_type, drug_interactions
      FROM drug_labels
      WHERE id IN (${placeholders})
    `
    const rows = await db.rawQuery(sql, safeIds)
    const rawRows: Array<{
      id: number | string
      brand_name: string | null
      generic_name: string | null
      product_type: string | null
      drug_interactions: string | null
    }> = Array.isArray(rows[0]) ? rows[0] : []
    // Build a lookup by id so we can re-order to match the request order.
    const byId = new Map<number, DrugInteractionEntry>()
    for (const row of rawRows) {
      const entry: DrugInteractionEntry = {
        id: Number(row.id),
        brand_name: row.brand_name ?? null,
        generic_name: row.generic_name ?? null,
        product_type: row.product_type ?? null,
        drug_interactions: row.drug_interactions ?? null,
      }
      byId.set(entry.id, entry)
    }
    // Re-order: iterate safeIds, include only ids that exist in the table.
    const ordered: DrugInteractionEntry[] = []
    for (const id of safeIds) {
      const entry = byId.get(id)
      if (entry) ordered.push(entry)
    }
    return ordered
  }
  /**
   * Get current row count — what's searchable right now.
   */
  async rowCount(): Promise<number> {
    try {
      const result = await db.rawQuery('SELECT COUNT(*) AS cnt FROM drug_labels')
      const rows = result[0] as Array<{ cnt: number | string }>
      return Number(rows[0]?.cnt ?? 0)
    } catch {
      return 0
    }
  }
  /**
   * Dispatch the download phase (idempotent — deduped by deterministic jobId).
   * Auto-chains the ingest phase on completion. Returns "already running" if the
   * download job is active/waiting.
   */
  async triggerDownload() {
    return DownloadDrugDataJob.dispatch(true)
  }
  /**
   * Freshness check for the content-auto-update path: compare the live openFDA
   * manifest `export_date` against the last-ingested one (KV
   * `drugReference.lastUpdatedExportDate`). Reuses the job's single manifest
   * fetch (Maxim 4 — one openFDA call site) and the pure `isExportDateNewer`
   * compare (defensive about the unconfirmed date format; see its TODO).
   *
   * Returns `{ updateAvailable, latestExportDate, currentExportDate }`. Never
   * triggers a download itself — `attemptAutoUpdate` decides whether to, after
   * its own gating (only when installed, no active job).
   */
  async checkForUpdate(): Promise<{
    updateAvailable: boolean
    latestExportDate: string | null
    currentExportDate: string | null
  }> {
    const current = await KVStore.getValue('drugReference.lastUpdatedExportDate')
    const manifest = await DownloadDrugDataJob.fetchManifest()
    const latest = manifest.export_date ?? null
    return {
      updateAvailable: isExportDateNewer(latest ?? '', current),
      latestExportDate: latest,
      currentExportDate: current,
    }
  }
  /**
   * Freshness pass for the drug dataset, invoked from
   * `ContentAutoUpdateService.attemptDrugDataset()` inside the hourly content
   * loop — so the dataset refreshes alongside ZIMs and maps under the shared
   * `contentAutoUpdate.*` master switch and window, not on a rogue schedule.
   *
   * Gating (unchanged from the prior standalone job): only acts when the dataset
   * is installed, and never while a drug download or ingest is already in flight,
   * so it can't stack a refresh on a running install. A failed manifest fetch is
   * a transient miss (offline), reported rather than thrown.
   */
  async attemptAutoUpdate(): Promise<{
    started: boolean
    reason: string
    latestExportDate?: string | null
  }> {
    // Only act when the dataset is installed.
    const rowCount = await this.rowCount()
    if (rowCount === 0) {
      return { started: false, reason: 'not-installed' }
    }
    // Don't stack on top of an in-flight download/ingest.
    const [dlJob, ingJob] = await Promise.all([
      DownloadDrugDataJob.getJob(),
      IngestDrugDataJob.getJob(),
    ])
    const activeStates = ['active', 'waiting', 'delayed']
    const dlState = dlJob ? await dlJob.getState() : undefined
    const ingState = ingJob ? await ingJob.getState() : undefined
    if (
      (dlState && activeStates.includes(dlState)) ||
      (ingState && activeStates.includes(ingState))
    ) {
      return { started: false, reason: 'job-in-flight' }
    }
    let check: Awaited<ReturnType<DrugReferenceService['checkForUpdate']>>
    try {
      check = await this.checkForUpdate()
    } catch (err) {
      // Offline or manifest fetch failed — transient, retried next run.
      logger.warn(
        `[DrugReferenceService] Freshness check failed (will retry next run): ${
          err instanceof Error ? err.message : String(err)
        }`
      )
      return { started: false, reason: 'check-failed' }
    }
    if (!check.updateAvailable) {
      return {
        started: false,
        reason: `up-to-date (current=${check.currentExportDate ?? 'none'}, latest=${
          check.latestExportDate ?? 'unknown'
        })`,
        latestExportDate: check.latestExportDate,
      }
    }
    logger.info(
      `[DrugReferenceService] Newer export_date available ` +
        `(current=${check.currentExportDate ?? 'none'} → latest=${check.latestExportDate}); ` +
        'triggering re-download.'
    )
    const result = await this.triggerDownload()
    return {
      started: result.created,
      reason: result.created ? 'update-dispatched' : result.message,
      latestExportDate: check.latestExportDate,
    }
  }
  /**
   * Dispatch the ingest phase from the already-downloaded on-disk parts (the
   * manual "Ingest into search" path). Guards on the KV download-state marker so
   * it fails fast with a typed result when nothing has been downloaded, rather
   * than dispatching a job that would immediately fail in the worker.
   */
  async triggerIngestFromDisk() {
    const marker = parseDownloadState(await KVStore.getValue('drugReference.downloadState'))
    if (!marker) {
      return {
        job: undefined,
        created: false,
        message: 'Nothing downloaded — run Download FDA data first.',
        nothingDownloaded: true,
      }
    }
    const result = await IngestDrugDataJob.dispatch()
    return { ...result, nothingDownloaded: false }
  }
  /**
   * Force-clear a wedged ingest and restart it from the on-disk parts.
   *
   * A worker killed mid-ingest (e.g. during a `nomad upgrade`) leaves its job
   * 'active' holding a lock BullMQ won't reclaim until lockDuration elapses, so
   * the normal dispatch refuses to start a new ingest and the UI button stays
   * disabled ("Indexing…") with no way out. Obliterating the single-purpose
   * drug-ingest queue removes the stuck job (force = even active/locked); we then
   * re-dispatch from disk. The downloaded parts and the download-state marker are
   * left untouched, so this restarts ingest WITHOUT re-downloading.
   */
  async resetAndReingest() {
    const marker = parseDownloadState(await KVStore.getValue('drugReference.downloadState'))
    if (!marker) {
      return {
        job: undefined,
        created: false,
        message: 'Nothing downloaded — run Download FDA data first.',
        nothingDownloaded: true,
      }
    }
    const queue = QueueService.getInstance().getQueue(IngestDrugDataJob.queue)
    try {
      // force: true removes the active/locked stuck job too. Scoped to the
      // single-purpose ingest queue, so nothing else is affected.
      await queue.obliterate({ force: true })
      logger.info('[DrugReferenceService] drug-ingest queue obliterated for restart')
    } catch (err) {
      logger.warn(
        `[DrugReferenceService] ingest queue obliterate failed (continuing to dispatch): ${
          err instanceof Error ? err.message : String(err)
        }`
      )
    }
    const result = await IngestDrugDataJob.dispatch()
    return { ...result, nothingDownloaded: false }
  }
  /**
   * Uninstall the offline FDA drug dataset — the curated-tier "remove" path.
   *
   * Mirrors how ZimService.delete() reverses a ZIM install: stop anything that
   * could re-create the data, delete the on-disk artifacts, drop the data, and
   * remove the install-state row so the tier reads not-installed and the home
   * tiles auto-hide (the same cascade as a ZIM delete).
   *
   * Order matters:
   *   1. Obliterate BOTH drug queues (force = removes active/locked jobs too) so a
   *      running download/ingest can't write rows back in after we clear them.
   *      Scoped to the two single-purpose drug queues — nothing else is touched.
   *   2. Delete the on-disk parts dir (STORAGE_BASE is a FIXED constant, never a
   *      client value — no path-traversal surface). Usually empty after a full
   *      ingest; non-empty only when uninstalling a downloaded-not-yet-ingested
   *      state.
   *   3. TRUNCATE drug_labels (not DROP) — preserves the schema + FULLTEXT
   *      indexes so a later reinstall re-ingests without a migration.
   *   4. Clear the KV markers (download-state + last-updated export_date).
   *   5. Delete the `installed_resources` 'dataset' row.
   *
   * Every step is best-effort and logged: a partial failure still removes as much
   * as it can and reports what it did, rather than leaving a half-uninstalled
   * state with no signal.
   */
  async uninstall(): Promise<{
    success: boolean
    rowsDropped: number
    message: string
  }> {
    const rowsBefore = await this.rowCount()
    const errors: string[] = []
    // 1. Stop in-flight jobs on both drug queues (force removes locked/active).
    for (const queueName of [DownloadDrugDataJob.queue, IngestDrugDataJob.queue]) {
      try {
        await QueueService.getInstance().getQueue(queueName).obliterate({ force: true })
        logger.info(`[DrugReferenceService] obliterated ${queueName} for uninstall`)
      } catch (err) {
        const msg = err instanceof Error ? err.message : String(err)
        logger.warn(`[DrugReferenceService] obliterate ${queueName} failed: ${msg}`)
        errors.push(`queue ${queueName}: ${msg}`)
      }
    }
    // 2. Delete the on-disk parts directory. STORAGE_BASE is a fixed module const
    // (never a client filename), so this is not a path-traversal surface.
    try {
      await rm(STORAGE_BASE, { recursive: true, force: true })
      logger.info(`[DrugReferenceService] removed on-disk parts dir ${STORAGE_BASE}`)
    } catch (err) {
      const msg = err instanceof Error ? err.message : String(err)
      logger.warn(`[DrugReferenceService] could not remove ${STORAGE_BASE}: ${msg}`)
      errors.push(`storage: ${msg}`)
    }
    // 3. Drop the searchable data. TRUNCATE keeps schema + the FULLTEXT indexes
    // so reinstall re-ingests cleanly with no migration.
    try {
      await db.rawQuery('TRUNCATE TABLE drug_labels')
      logger.info('[DrugReferenceService] truncated drug_labels')
    } catch (err) {
      const msg = err instanceof Error ? err.message : String(err)
      logger.error(`[DrugReferenceService] TRUNCATE drug_labels failed: ${msg}`)
      errors.push(`truncate: ${msg}`)
    }
    // 4. Clear KV markers.
    try {
      await KVStore.clearValue('drugReference.downloadState')
      await KVStore.clearValue('drugReference.lastUpdatedExportDate')
    } catch (err) {
      const msg = err instanceof Error ? err.message : String(err)
      logger.warn(`[DrugReferenceService] clearing KV markers failed: ${msg}`)
      errors.push(`kv: ${msg}`)
    }
    // 5. Remove the install-state row so the tier reads not-installed.
    try {
      const { default: InstalledResource } = await import('#models/installed_resource')
      await InstalledResource.query()
        .where('resource_type', 'dataset')
        .where('resource_id', DRUG_DATASET_RESOURCE_ID)
        .delete()
      logger.info('[DrugReferenceService] deleted installed_resources dataset row')
    } catch (err) {
      const msg = err instanceof Error ? err.message : String(err)
      logger.warn(`[DrugReferenceService] deleting install row failed: ${msg}`)
      errors.push(`install-row: ${msg}`)
    }
    const rowsDropped = rowsBefore - (await this.rowCount())
    const success = errors.length === 0
    return {
      success,
      rowsDropped,
      message: success
        ? `Uninstalled FDA drug reference (${rowsDropped} labels removed).`
        : `Uninstall completed with ${errors.length} issue(s): ${errors.join('; ')}`,
    }
  }
  /**
   * Resolve the canonical deterministic job for a phase's queue, falling back to
   * the most-progressed auto-id continuation when the deterministic job is
   * absent or completed (passes > 0 use auto-generated ids). Each phase has its
   * own queue + jobId, so this is called once per queue with the matching ids.
   */
  private async resolvePhaseJob(
    queueName: string,
    deterministicJobId: string
  ): Promise<Job | undefined> {
    const queueService = QueueService.getInstance()
    const queue = queueService.getQueue(queueName)
    let job = await queue.getJob(deterministicJobId)
    if (!job || (await job.getState()) === 'completed') {
      const activeJobs = await queue.getJobs(['active', 'waiting', 'delayed'])
      const continuation = activeJobs
        .filter((j) => j.id !== deterministicJobId)
        .sort((a, b) => (b.data?.partIndex ?? 0) - (a.data?.partIndex ?? 0))[0]
      if (continuation) {
        job = continuation
      } else {
        // No live continuation. If the chain FAILED partway, surface the failed
        // job so the status reads 'failed' instead of falsely 'ready' with only
        // part 1's count. BUT removeOnFail keeps history: a STALE failure from a
        // prior run must not mask a newer successful one (seen live: a fixed
        // re-run completed, yet the panel stayed 'failed' on last night's
        // duplicate-key job). Compare finish times and only surface the failed
        // job when it is the most recent terminal outcome.
        const [failedJobs, completedJobs] = await Promise.all([
          queue.getJobs(['failed']),
          queue.getJobs(['completed']),
        ])
        const newestFailed = failedJobs.sort((a, b) => (b.finishedOn ?? 0) - (a.finishedOn ?? 0))[0]
        const newestCompletedFinish = completedJobs.reduce(
          (max, j) => Math.max(max, j.finishedOn ?? 0),
          0
        )
        if (newestFailed && (newestFailed.finishedOn ?? 0) > newestCompletedFinish) {
          job = newestFailed
        }
      }
    }
    return job
  }
  /** Map a BullMQ job state to the per-phase run state. */
  private phaseStateFor(state: string | undefined): DrugPhaseState {
    if (state === 'failed') return 'failed'
    if (state === 'completed') return 'completed'
    if (state === 'active' || state === 'waiting' || state === 'delayed') return 'running'
    return 'idle'
  }
  /**
   * Return the two-phase ingest status for the UI panel. Reads the download job
   * (drug-download queue) and the ingest job (drug-ingest queue) independently,
   * merges the KV download-state marker + last-updated marker + live row count,
   * and derives the top-level phase from the two sub-phases.
   */
  async getIngestStatus(): Promise<DrugIngestStatus> {
    const [downloadJob, ingestJob] = await Promise.all([
      this.resolvePhaseJob(DownloadDrugDataJob.queue, DownloadDrugDataJob.jobId),
      this.resolvePhaseJob(IngestDrugDataJob.queue, IngestDrugDataJob.jobId),
    ])
    const [lastUpdated, rawMarker, count] = await Promise.all([
      KVStore.getValue('drugReference.lastUpdatedExportDate'),
      KVStore.getValue('drugReference.downloadState'),
      this.rowCount(),
    ])
    const marker = parseDownloadState(rawMarker)
    // ── Download sub-status ─────────────────────────────────────────────────
    const dlState = downloadJob ? await downloadJob.getState() : undefined
    const dlData = downloadJob?.data ?? {}
    let downloadPhaseState = this.phaseStateFor(dlState)
    // A finished download job is pruned (removeOnComplete) but the marker proves
    // the parts are on disk — treat that as a completed download phase so the
    // manual "Ingest into search" button stays available.
    if (downloadPhaseState === 'idle' && marker) downloadPhaseState = 'completed'
    const download: DrugDownloadStatus = {
      state: downloadPhaseState,
      partsDone: downloadPhaseState === 'completed' ? (marker?.totalParts ?? dlData.totalParts ?? 0) : (dlData.partIndex ?? 0),
      totalParts: dlData.totalParts ?? marker?.totalParts ?? 0,
      bytesDownloaded: dlData.bytesDownloaded,
      currentPartName: dlData.currentPartName ?? null,
      failedReason:
        dlState === 'failed' ? summarizeJobError(downloadJob?.failedReason) : undefined,
    }
    // ── Ingest sub-status ───────────────────────────────────────────────────
    const ingState = ingestJob ? await ingestJob.getState() : undefined
    const ingData = ingestJob?.data ?? {}
    let ingestPhaseState = this.phaseStateFor(ingState)
    // A pruned-but-successful ingest leaves rows + the last-updated marker and
    // clears the download marker; reflect that as a completed ingest phase.
    if (ingestPhaseState === 'idle' && !marker && count > 0) ingestPhaseState = 'completed'
    // total_records 0 means "unknown" (e.g. a manifest rebuilt from an older
    // marker) — resolveExpectedTotal falls back to a parts estimate, then the
    // live row count, so the counter/%/ETA never silently vanish.
    const expectedTotal = resolveExpectedTotal(
      ingData.manifest?.total_records,
      ingData.totalParts,
      count
    )
    const jobRecords = ingData.recordsIngested ?? 0
    // While ingesting, the per-job recordsIngested lags across the per-part
    // continuation handoff (continuations run under auto jobIds). Drive the shown
    // count from max(jobRecords, live rowCount) so a first ingest tracks the table
    // filling 0 → ~259k, while a re-ingest into a populated table still rides the
    // per-job counter. Outside the running phase, trust the job's own total.
    // Always reconcile against the live row count (the per-job recordsIngested can
    // be a partial/stale total — a completed pass-0 job only counted part 1; a
    // failed continuation stops mid-run). WHILE RUNNING, additionally subtract the
    // run's start-row baseline so a re-ingest into a populated table shows THIS
    // run's progress (0 → ~259k) instead of reading ~100% from second zero.
    // Outside running, start=0 so 'ready'/'failed' reflect total searchable rows.
    const records =
      ingestPhaseState === 'running'
        ? resolveIngestRecordsShown(jobRecords, count, expectedTotal, ingData.startRowCount ?? 0)
        : resolveIngestRecordsShown(jobRecords, count, expectedTotal)
    const ingest: DrugIngestPhaseStatus = {
      state: ingestPhaseState,
      records,
      expectedTotal,
      partsDone: ingData.partIndex ?? 0,
      totalParts: ingData.totalParts ?? marker?.totalParts ?? 0,
      currentPartName: ingData.currentPartName ?? null,
      failedReason: ingState === 'failed' ? summarizeJobError(ingestJob?.failedReason) : undefined,
    }
    const phase = deriveIngestPhase(download, ingest, count)
    // The active phase drives elapsed/ETA: ingest start when ingesting, else the
    // download start.
    const startedAtMs =
      phase === 'ingesting'
        ? (ingData.startedAt ?? null)
        : phase === 'downloading'
          ? (dlData.startedAt ?? null)
          : null
    const error =
      phase === 'failed' ? (ingest.failedReason ?? download.failedReason) : undefined
    return {
      phase,
      download,
      ingest,
      startedAtMs,
      lastUpdated: lastUpdated ?? null,
      rowCount: count,
      error,
    }
  }
 }
--- a/admin/app/services/kiwix_catalog_service.ts
+++ b/admin/app/services/kiwix_catalog_service.ts
@ -1,338 +0,0 @@
 import axios from 'axios'
 import { XMLParser } from 'fast-xml-parser'
 import { DateTime } from 'luxon'
 import logger from '@adonisjs/core/services/logger'
 import InstalledResource from '#models/installed_resource'
 import { isRawListRemoteZimFilesResponse } from '../../util/zim.js'
 /**
 * Local, in-process freshness check for installed content (Kiwix ZIM files +
 * PMTiles maps). This replaces the former dependency on the external
 * project-nomad-api `/api/v1/resources/check-updates` endpoint — every NOMAD
 * instance now queries the upstream catalogs directly.
 *
 * Downloads have always gone straight to the Kiwix/GitHub mirrors regardless of
 * who performed the check, so moving the check in-process only shifts the
 * lightweight *catalog* lookup. To stay mirror-respectful the auto-updater gates
 * these calls behind the update window and bounds their concurrency; sizes come
 * from the catalog metadata so we avoid per-file HEAD requests.
 *
 * Robustness over the old API: ZIM lookups use the OPDS exact `name=` filter
 * (no lossy keyword stripping) and every returned link is still validated
 * against the authoritative `^<id>_YYYY-MM\.zim$` filename regex, so a substring
 * match in the catalog can never resolve to the wrong book. Parsing is fully
 * defensive — a malformed entry is skipped, never thrown.
 */
 const KIWIX_CATALOG_URL = 'https://browse.library.kiwix.org/catalog/v2/entries'
 const GITHUB_PMTILES_URL =
  'https://api.github.com/repos/Crosstalk-Solutions/project-nomad-maps/contents/pmtiles'
 const CATALOG_TIMEOUT_MS = 15000
 /** Bounded paginated fallback scan when the exact `name=` lookup comes up empty. */
 const KIWIX_PAGE_SIZE = 60
 const MAX_KIWIX_FETCHES = 5
 /** Concurrent ZIM catalog lookups — keep small to avoid hammering the mirror. */
 const ZIM_CHECK_CONCURRENCY = 4
 /** The newest available version of a single resource (a YYYY-MM date stamp). */
 export interface CatalogResult {
  version: string
  download_url: string
  size_bytes: number
 }
 interface CatalogZimEntry {
  name: string | null
  download_url: string
  file_name: string
  size_bytes: number
 }
 interface GithubContentEntry {
  name: string
  download_url: string | null
  size: number
 }
 function escapeRegex(input: string): string {
  return input.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
 }
 export class KiwixCatalogService {
  private parser = new XMLParser({
    ignoreAttributes: false,
    attributeNamePrefix: '',
    textNodeName: '#text',
  })
  /**
   * Resolve the newest available version for a batch of installed resources.
   * Returns a map keyed by `"<resource_type>:<resource_id>"`; resources with no
   * available newer version (or a failed lookup) are simply absent.
   *
   * ZIMs are checked one OPDS request each (bounded concurrency). All maps are
   * resolved from a single GitHub directory listing.
   */
  async getLatestForResources(
    resources: Array<{ resource_id: string; resource_type: 'zim' | 'map' }>
  ): Promise<Map<string, CatalogResult>> {
    const result = new Map<string, CatalogResult>()
    const zims = resources.filter((r) => r.resource_type === 'zim')
    const maps = resources.filter((r) => r.resource_type === 'map')
    await this.forEachWithConcurrency(zims, ZIM_CHECK_CONCURRENCY, async (r) => {
      try {
        const latest = await this.getLatestZim(r.resource_id)
        if (latest) result.set(`zim:${r.resource_id}`, latest)
      } catch (error) {
        logger.warn(
          `[KiwixCatalogService] ZIM check failed for ${r.resource_id}: ${error instanceof Error ? error.message : error}`
        )
      }
    })
    if (maps.length > 0) {
      try {
        const listing = await this.fetchMapListing()
        for (const r of maps) {
          const latest = this.pickNewestMap(listing, r.resource_id)
          if (latest) result.set(`map:${r.resource_id}`, latest)
        }
      } catch (error) {
        logger.warn(
          `[KiwixCatalogService] Map listing fetch failed: ${error instanceof Error ? error.message : error}`
        )
      }
    }
    return result
  }
  /** Newest catalog version of a single ZIM book, or null if none/older. */
  async getLatestZim(resourceId: string): Promise<CatalogResult | null> {
    const pattern = new RegExp(`^${escapeRegex(resourceId)}_(\\d{4}-\\d{2})\\.zim$`)
    // 1. Exact-name lookup (the robust path).
    const named = await this.fetchZimEntries({ name: resourceId, count: 50, start: 0 })
    const exact = this.pickNewestZim(named, pattern)
    if (exact) return exact
    // 2. Fallback: bounded keyword scan in case the catalog ignored `name=` or
    //    indexes the book under a slightly different name.
    return this.scanZimByQuery(resourceId, pattern)
  }
  /** Newest catalog version of a single PMTiles map, or null if none/older. */
  async getLatestMap(resourceId: string): Promise<CatalogResult | null> {
    const listing = await this.fetchMapListing()
    return this.pickNewestMap(listing, resourceId)
  }
  // ── ZIM internals ───────────────────────────────────────────────────────────
  private pickNewestZim(entries: CatalogZimEntry[], pattern: RegExp): CatalogResult | null {
    let latest: CatalogResult | null = null
    for (const entry of entries) {
      const match = entry.file_name.match(pattern)
      if (!match) continue
      const version = match[1]
      if (!latest || version > latest.version) {
        latest = { version, download_url: entry.download_url, size_bytes: entry.size_bytes }
      }
    }
    return latest
  }
  private async scanZimByQuery(
    resourceId: string,
    pattern: RegExp
  ): Promise<CatalogResult | null> {
    let start = 0
    let total = 0
    let latest: CatalogResult | null = null
    for (let i = 0; i < MAX_KIWIX_FETCHES; i++) {
      const { entries, totalResults } = await this.fetchZimEntriesPage({
        q: resourceId,
        count: KIWIX_PAGE_SIZE,
        start,
      })
      total = totalResults
      if (entries.length === 0) break
      start += entries.length
      const candidate = this.pickNewestZim(entries, pattern)
      if (candidate && (!latest || candidate.version > latest.version)) {
        latest = candidate
      }
      if (start >= total) break
    }
    return latest
  }
  private async fetchZimEntries(params: {
    name?: string
    q?: string
    count: number
    start: number
  }): Promise<CatalogZimEntry[]> {
    const { entries } = await this.fetchZimEntriesPage(params)
    return entries
  }
  private async fetchZimEntriesPage(params: {
    name?: string
    q?: string
    count: number
    start: number
  }): Promise<{ entries: CatalogZimEntry[]; totalResults: number }> {
    const res = await axios.get(KIWIX_CATALOG_URL, {
      params: {
        start: params.start,
        count: params.count,
        lang: 'eng',
        ...(params.name ? { name: params.name } : {}),
        ...(params.q ? { q: params.q } : {}),
      },
      responseType: 'text',
      timeout: CATALOG_TIMEOUT_MS,
    })
    return this.parseZimEntries(res.data)
  }
  private parseZimEntries(xml: string): { entries: CatalogZimEntry[]; totalResults: number } {
    let parsed: any
    try {
      parsed = this.parser.parse(xml)
    } catch {
      return { entries: [], totalResults: 0 }
    }
    if (!isRawListRemoteZimFilesResponse(parsed)) {
      return { entries: [], totalResults: 0 }
    }
    const feed = parsed.feed
    const totalResults = Number(feed?.totalResults)
    const rawEntries = feed?.entry
      ? Array.isArray(feed.entry)
        ? feed.entry
        : [feed.entry]
      : []
    const entries: CatalogZimEntry[] = []
    for (const raw of rawEntries) {
      if (!raw || typeof raw !== 'object') continue
      const links = Array.isArray(raw.link) ? raw.link : raw.link ? [raw.link] : []
      const downloadLink = links.find(
        (link: any) =>
          link &&
          typeof link === 'object' &&
          link.type === 'application/x-zim' &&
          typeof link.href === 'string'
      )
      if (!downloadLink) continue
      // The OPDS href ends with `.meta4`; strip it to get the real .zim URL.
      const href: string = downloadLink.href
      const download_url = href.endsWith('.meta4') ? href.slice(0, -'.meta4'.length) : href
      const file_name = download_url.split('/').pop() || ''
      if (!file_name) continue
      const size_bytes = Number.parseInt(downloadLink.length, 10) || 0
      entries.push({
        name: typeof raw.name === 'string' ? raw.name : null,
        download_url,
        file_name,
        size_bytes,
      })
    }
    return { entries, totalResults: Number.isFinite(totalResults) ? totalResults : 0 }
  }
  // ── Map internals ────────────────────────────────────────────────────────────
  private async fetchMapListing(): Promise<GithubContentEntry[]> {
    const res = await axios.get(GITHUB_PMTILES_URL, {
      headers: { Accept: 'application/vnd.github+json' },
      timeout: CATALOG_TIMEOUT_MS,
    })
    return Array.isArray(res.data) ? res.data : []
  }
  private pickNewestMap(listing: GithubContentEntry[], resourceId: string): CatalogResult | null {
    const pattern = new RegExp(`^${escapeRegex(resourceId)}_(\\d{4}-\\d{2})\\.pmtiles$`)
    let latest: CatalogResult | null = null
    for (const file of listing) {
      if (!file || typeof file.name !== 'string' || !file.download_url) continue
      const match = file.name.match(pattern)
      if (!match) continue
      const version = match[1]
      if (!latest || version > latest.version) {
        latest = {
          version,
          download_url: file.download_url,
          size_bytes: typeof file.size === 'number' ? file.size : 0,
        }
      }
    }
    return latest
  }
  // ── Shared ───────────────────────────────────────────────────────────────────
  private async forEachWithConcurrency<T>(
    items: T[],
    concurrency: number,
    worker: (item: T) => Promise<void>
  ): Promise<void> {
    let cursor = 0
    const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
      while (cursor < items.length) {
        const index = cursor++
        await worker(items[index])
      }
    })
    await Promise.all(runners)
  }
 }
 /**
 * Persist the latest-known available-update state onto an installed resource.
 * Shared by the manual check and the auto-updater so both keep the cool-off
 * anchor consistent.
 *
 * The first-seen anchor is reset **only** when the available version string
 * actually changes, so a manual "Check for updates" never resets the auto
 * cool-off clock. State is cleared entirely once the resource is current (the
 * update got installed, or the upstream release was withdrawn).
 */
 export async function reconcileResourceUpdateState(
  resource: InstalledResource,
  latest: CatalogResult | null,
  now: DateTime
 ): Promise<void> {
  const hasUpdate = latest !== null && latest.version > resource.version
  if (hasUpdate) {
    if (resource.available_update_version !== latest!.version) {
      resource.available_update_version = latest!.version
      resource.available_update_first_seen_at = now
    }
    // Keep the cached size fresh even when the version is unchanged (the catalog
    // may report a size it lacked on a previous check).
    const size = latest!.size_bytes || null
    if (resource.available_update_size_bytes !== size) {
      resource.available_update_size_bytes = size
    }
  } else if (resource.available_update_version !== null) {
    resource.available_update_version = null
    resource.available_update_size_bytes = null
    resource.available_update_first_seen_at = null
  }
  if (resource.$isDirty) {
    await resource.save()
  }
 }
--- a/admin/app/services/kiwix_library_service.ts
+++ b/admin/app/services/kiwix_library_service.ts
@ -1,364 +0,0 @@
 import { XMLBuilder, XMLParser } from 'fast-xml-parser'
 import { readFile, writeFile, rename, readdir } from 'fs/promises'
 import { join } from 'path'
 import { Archive } from '@openzim/libzim'
 import { KIWIX_LIBRARY_XML_PATH, ZIM_STORAGE_PATH, ensureDirectoryExists, isValidZimFile } from '../utils/fs.js'
 import logger from '@adonisjs/core/services/logger'
 import { randomUUID } from 'node:crypto'
 const CONTAINER_DATA_PATH = '/data'
 const XML_DECLARATION = '<?xml version="1.0" encoding="UTF-8"?>\n'
 interface KiwixBook {
  id: string
  path: string
  title: string
  description?: string
  language?: string
  creator?: string
  publisher?: string
  name?: string
  flavour?: string
  tags?: string
  faviconMimeType?: string
  favicon?: string
  date?: string
  articleCount?: number
  mediaCount?: number
  size?: number
 }
 export class KiwixLibraryService {
  getLibraryFilePath(): string {
    return join(process.cwd(), KIWIX_LIBRARY_XML_PATH)
  }
  containerLibraryPath(): string {
    return '/data/kiwix-library.xml'
  }
  private _filenameToTitle(filename: string): string {
    const withoutExt = filename.endsWith('.zim') ? filename.slice(0, -4) : filename
    const parts = withoutExt.split('_')
    // Drop last segment if it looks like a date (YYYY-MM)
    const lastPart = parts[parts.length - 1]
    const isDate = /^\d{4}-\d{2}$/.test(lastPart)
    const titleParts = isDate && parts.length > 1 ? parts.slice(0, -1) : parts
    return titleParts.map((p) => p.charAt(0).toUpperCase() + p.slice(1)).join(' ')
  }
  /**
   * Reads all kiwix-manage-compatible metadata from a ZIM file, including the internal UUID,
   * rich text fields, and the base64-encoded favicon. Kiwix-serve uses the UUID for OPDS
   * catalog entries and illustration URLs (/catalog/v2/illustration/{uuid}).
   *
   * Returns null on any error so callers can fall back gracefully.
   */
  private async _readZimMetadata(zimFilePath: string): Promise<Partial<KiwixBook> | null> {
    try {
      if (!(await isValidZimFile(zimFilePath))) {
        logger.warn(`[KiwixLibraryService] Skipping invalid/corrupted ZIM file: ${zimFilePath}`)
        return null
      }
      const archive = new Archive(zimFilePath)
      const getMeta = (key: string): string | undefined => {
        try {
          return archive.getMetadata(key) || undefined
        } catch {
          return undefined
        }
      }
      let favicon: string | undefined
      let faviconMimeType: string | undefined
      try {
        if (archive.illustrationSizes.size > 0) {
          const size = archive.illustrationSizes.has(48)
            ? 48
            : ([...archive.illustrationSizes][0] as number)
          const item = archive.getIllustrationItem(size)
          favicon = item.data.data.toString('base64')
          faviconMimeType = item.mimetype || undefined
        }
      } catch {
        // ZIM has no illustration — that's fine
      }
      const rawFilesize =
        typeof archive.filesize === 'bigint' ? Number(archive.filesize) : archive.filesize
      return {
        id: archive.uuid || undefined,
        title: getMeta('Title'),
        description: getMeta('Description'),
        language: getMeta('Language'),
        creator: getMeta('Creator'),
        publisher: getMeta('Publisher'),
        name: getMeta('Name'),
        flavour: getMeta('Flavour'),
        tags: getMeta('Tags'),
        date: getMeta('Date'),
        articleCount: archive.articleCount,
        mediaCount: archive.mediaCount,
        size: Math.floor(rawFilesize / 1024),
        favicon,
        faviconMimeType,
      }
    } catch {
      return null
    }
  }
  private _buildXml(books: KiwixBook[]): string {
    const builder = new XMLBuilder({
      ignoreAttributes: false,
      attributeNamePrefix: '@_',
      format: true,
      suppressEmptyNode: false,
    })
    const obj: Record<string, any> = {
      library: {
        '@_version': '20110515',
        ...(books.length > 0 && {
          book: books.map((b) => ({
            '@_id': b.id,
            '@_path': b.path,
            '@_title': b.title,
            ...(b.description !== undefined && { '@_description': b.description }),
            ...(b.language !== undefined && { '@_language': b.language }),
            ...(b.creator !== undefined && { '@_creator': b.creator }),
            ...(b.publisher !== undefined && { '@_publisher': b.publisher }),
            ...(b.name !== undefined && { '@_name': b.name }),
            ...(b.flavour !== undefined && { '@_flavour': b.flavour }),
            ...(b.tags !== undefined && { '@_tags': b.tags }),
            ...(b.faviconMimeType !== undefined && { '@_faviconMimeType': b.faviconMimeType }),
            ...(b.favicon !== undefined && { '@_favicon': b.favicon }),
            ...(b.date !== undefined && { '@_date': b.date }),
            ...(b.articleCount !== undefined && { '@_articleCount': b.articleCount }),
            ...(b.mediaCount !== undefined && { '@_mediaCount': b.mediaCount }),
            ...(b.size !== undefined && { '@_size': b.size }),
          })),
        }),
      },
    }
    return XML_DECLARATION + builder.build(obj)
  }
  private async _atomicWrite(content: string): Promise<void> {
    const filePath = this.getLibraryFilePath()
    const tmpPath = `${filePath}.tmp.${randomUUID()}`
    await writeFile(tmpPath, content, 'utf-8')
    await rename(tmpPath, filePath)
  }
  private _parseExistingBooks(xmlContent: string): KiwixBook[] {
    const parser = new XMLParser({
      ignoreAttributes: false,
      attributeNamePrefix: '@_',
      isArray: (name) => name === 'book',
    })
    const parsed = parser.parse(xmlContent)
    const books: any[] = parsed?.library?.book ?? []
    return books
      .map((b) => ({
        id: b['@_id'] ?? '',
        path: b['@_path'] ?? '',
        title: b['@_title'] ?? '',
        description: b['@_description'],
        language: b['@_language'],
        creator: b['@_creator'],
        publisher: b['@_publisher'],
        name: b['@_name'],
        flavour: b['@_flavour'],
        tags: b['@_tags'],
        faviconMimeType: b['@_faviconMimeType'],
        favicon: b['@_favicon'],
        date: b['@_date'],
        articleCount:
          b['@_articleCount'] !== undefined ? Number(b['@_articleCount']) : undefined,
        mediaCount: b['@_mediaCount'] !== undefined ? Number(b['@_mediaCount']) : undefined,
        size: b['@_size'] !== undefined ? Number(b['@_size']) : undefined,
      }))
      .filter((b) => b.id && b.path)
  }
  /**
   * Returns the number of books currently listed in the library XML, or 0 if the
   * file doesn't exist yet. Used to report a before/after delta on a manual rescan.
   */
  async getBookCount(): Promise<number> {
    try {
      const content = await readFile(this.getLibraryFilePath(), 'utf-8')
      return this._parseExistingBooks(content).length
    } catch (err: any) {
      if (err.code === 'ENOENT') return 0
      throw err
    }
  }
  /**
   * True if the library XML parses and has a <library> root. A truncated or
   * corrupt file (e.g. an interrupted write) fails this even though it exists,
   * so the caller can rebuild rather than leave Kiwix serving a broken library.
   * An empty-but-well-formed library is considered valid (nothing to repair).
   */
  private _isValidLibraryXml(xmlContent: string): boolean {
    try {
      const parser = new XMLParser({
        ignoreAttributes: false,
        attributeNamePrefix: '@_',
        isArray: (name) => name === 'book',
      })
      const parsed = parser.parse(xmlContent)
      return parsed?.library !== undefined && parsed?.library !== null
    } catch {
      return false
    }
  }
  /**
   * Boot-time safety net: if the library XML is missing or unparseable, rebuild
   * it from the ZIM files on disk so Kiwix (running in library mode with
   * --monitorLibrary) doesn't come up serving an empty/broken library with no
   * path to recovery. This covers files lost or corrupted outside the normal
   * download flow (storage relocation, interrupted write, manual deletion).
   *
   * Returns true if a rebuild was performed. Filesystem errors other than
   * "not found" are surfaced rather than masked by a rebuild.
   */
  async ensureLibraryXmlHealthy(): Promise<boolean> {
    let content: string
    try {
      content = await readFile(this.getLibraryFilePath(), 'utf-8')
    } catch (err: any) {
      if (err?.code === 'ENOENT') {
        logger.warn('[KiwixLibraryService] Library XML missing on startup; rebuilding from disk.')
        await this.rebuildFromDisk()
        return true
      }
      throw err
    }
    if (this._isValidLibraryXml(content)) return false
    logger.warn('[KiwixLibraryService] Library XML present but invalid; rebuilding from disk.')
    await this.rebuildFromDisk()
    return true
  }
  async rebuildFromDisk(opts?: { excludeFilenames?: string[] }): Promise<number> {
    const dirPath = join(process.cwd(), ZIM_STORAGE_PATH)
    await ensureDirectoryExists(dirPath)
    let entries: string[] = []
    try {
      entries = await readdir(dirPath)
    } catch {
      entries = []
    }
    const excludeSet = new Set(opts?.excludeFilenames ?? [])
    const zimFiles = entries.filter((name) => name.endsWith('.zim') && !excludeSet.has(name))
    const books: KiwixBook[] = []
    for (const filename of zimFiles) {
      const meta = await this._readZimMetadata(join(dirPath, filename))
      if (meta === null) {
        logger.warn(`[KiwixLibraryService] Skipping unreadable ZIM file: ${filename}`)
        continue
      }
      const containerPath = `${CONTAINER_DATA_PATH}/${filename}`
      books.push({
        ...meta,
        // Override fields that must be derived locally, not from ZIM metadata
        id: meta?.id ?? filename.slice(0, -4),
        path: containerPath,
        title: meta?.title ?? this._filenameToTitle(filename),
      })
    }
    const xml = this._buildXml(books)
    await this._atomicWrite(xml)
    logger.info(`[KiwixLibraryService] Rebuilt library XML with ${books.length} book(s).`)
    return books.length
  }
  async addBook(filename: string): Promise<void> {
    const zimFilename = filename.endsWith('.zim') ? filename : `${filename}.zim`
    const containerPath = `${CONTAINER_DATA_PATH}/${zimFilename}`
    const filePath = this.getLibraryFilePath()
    let existingBooks: KiwixBook[] = []
    try {
      const content = await readFile(filePath, 'utf-8')
      existingBooks = this._parseExistingBooks(content)
    } catch (err: any) {
      if (err.code === 'ENOENT') {
        // XML doesn't exist yet — rebuild from disk; the completed download is already there
        await this.rebuildFromDisk()
        return
      }
      throw err
    }
    if (existingBooks.some((b) => b.path === containerPath)) {
      logger.info(`[KiwixLibraryService] ${zimFilename} already in library, skipping.`)
      return
    }
    const fullPath = join(process.cwd(), ZIM_STORAGE_PATH, zimFilename)
    const meta = await this._readZimMetadata(fullPath)
    if (meta === null) {
      logger.error(`[KiwixLibraryService] Cannot add ${zimFilename}: file is invalid or corrupted.`)
      return
    }
    existingBooks.push({
      ...meta,
      id: meta?.id ?? zimFilename.slice(0, -4),
      path: containerPath,
      title: meta?.title ?? this._filenameToTitle(zimFilename),
    })
    const xml = this._buildXml(existingBooks)
    await this._atomicWrite(xml)
    logger.info(`[KiwixLibraryService] Added ${zimFilename} to library XML.`)
  }
  async removeBook(filename: string): Promise<void> {
    const zimFilename = filename.endsWith('.zim') ? filename : `${filename}.zim`
    const containerPath = `${CONTAINER_DATA_PATH}/${zimFilename}`
    const filePath = this.getLibraryFilePath()
    let existingBooks: KiwixBook[] = []
    try {
      const content = await readFile(filePath, 'utf-8')
      existingBooks = this._parseExistingBooks(content)
    } catch (err: any) {
      if (err.code === 'ENOENT') {
        logger.warn(`[KiwixLibraryService] Library XML not found, nothing to remove.`)
        return
      }
      throw err
    }
    const filtered = existingBooks.filter((b) => b.path !== containerPath)
    if (filtered.length === existingBooks.length) {
      logger.info(`[KiwixLibraryService] ${zimFilename} not found in library, nothing to remove.`)
      return
    }
    const xml = this._buildXml(filtered)
    await this._atomicWrite(xml)
    logger.info(`[KiwixLibraryService] Removed ${zimFilename} from library XML.`)
  }
 }
--- a/admin/app/services/map_service.ts
+++ b/admin/app/services/map_service.ts
@ -1,5 +1,6 @@
 import { BaseStylesFile, MapLayer } from '../../types/maps.js'
 import {
  DownloadCollectionOperation,
  DownloadRemoteSuccessCallback,
  FileEntry,
 } from '../../types/files.js'
@ -8,54 +9,22 @@ import { extract } from 'tar'
 import env from '#start/env'
 import {
  listDirectoryContentsRecursive,
  listDirectoryContents,
  getFileStatsIfExists,
  deleteFileIfExists,
  getFile,
  ensureDirectoryExists,
 } from '../utils/fs.js'
-import { join, resolve, sep } from 'path'
+import { join } from 'path'
 import urlJoin from 'url-join'
 import axios from 'axios'
 import { RunDownloadJob } from '#jobs/run_download_job'
 import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job'
 import logger from '@adonisjs/core/services/logger'
-import { assertNotPrivateUrl } from '#validators/common'
+import { CuratedCollectionsFile, CuratedCollectionWithStatus } from '../../types/downloads.js'
-import InstalledResource from '#models/installed_resource'
+import CuratedCollection from '#models/curated_collection'
-import { CollectionManifestService } from './collection_manifest_service.js'
+import vine from '@vinejs/vine'
-import { decideSupersededDeletion } from '../utils/superseded_resource.js'
+import { curatedCollectionsFileSchema } from '#validators/curated_collections'
-import type { CollectionWithStatus, MapsSpec } from '../../types/collections.js'
+import CuratedCollectionResource from '#models/curated_collection_resource'
 import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps.js'
 import {
  EXTRACT_DEFAULT_MAX_ZOOM,
  EXTRACT_MAX_ZOOM,
  EXTRACT_MIN_ZOOM,
  PMTILES_BINARY_PATH,
  WORLD_BASEMAP_FILENAME,
  WORLD_BASEMAP_MAX_ZOOM,
  WORLD_BASEMAP_SOURCE_NAME,
  buildPmtilesExtractArgs,
 } from '../../constants/map_regions.js'
 import { CountriesService } from './countries_service.js'
 import { execFile } from 'child_process'
 import { createHash, randomBytes } from 'crypto'
 import { tmpdir } from 'os'
 import { promisify } from 'util'
 const execFileAsync = promisify(execFile)
 const DRY_RUN_TIMEOUT_MS = 60_000
 const DRY_RUN_MAX_BUFFER = 256 * 1024
 // Real extract of z0-5 world tiles; generous to tolerate slow/metered links
 // since a failure leaves the map grey for uncovered regions.
 const WORLD_BASEMAP_EXTRACT_TIMEOUT_MS = 5 * 60_000
 const PROTOMAPS_BUILDS_METADATA_URL = 'https://build-metadata.protomaps.dev/builds.json'
 const PROTOMAPS_BUILD_BASE_URL = 'https://build.protomaps.com'
 export interface ProtomapsBuildInfo {
  url: string
  date: string
  size: number
  key: string
 }
 const BASE_ASSETS_MIME_TYPES = [
  'application/gzip',
@ -63,11 +32,15 @@ const BASE_ASSETS_MIME_TYPES = [
  'application/octet-stream',
 ]
 const COLLECTIONS_URL =
  'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/collections/maps.json'
 const PMTILES_ATTRIBUTION =
  '<a href="https://github.com/protomaps/basemaps">Protomaps</a> © <a href="https://openstreetmap.org">OpenStreetMap</a>'
 const PMTILES_MIME_TYPES = ['application/vnd.pmtiles', 'application/octet-stream']
 interface IMapService {
  downloadCollection: DownloadCollectionOperation
  downloadRemoteSuccessCallback: DownloadRemoteSuccessCallback
 }
@ -77,16 +50,10 @@ export class MapService implements IMapService {
  private readonly basemapsAssetsDir = 'basemaps-assets'
  private readonly baseAssetsTarFile = 'base-assets.tar.gz'
  private readonly baseDirPath = join(process.cwd(), this.mapStoragePath)
  private baseAssetsExistCache: boolean | null = null
  private worldBasemapReady = false
  private worldBasemapInFlight: Promise<void> | null = null
  async listRegions() {
    const files = (await this.listAllMapStorageItems()).filter(
-      (item) =>
+      (item) => item.type === 'file' && item.name.endsWith('.pmtiles')
        item.type === 'file' &&
        item.name.endsWith('.pmtiles') &&
        item.name !== WORLD_BASEMAP_FILENAME
    )
    return {
@ -126,46 +93,37 @@ export class MapService implements IMapService {
    await deleteFileIfExists(tempTarPath)
    // Invalidate cache since we just downloaded new assets
    this.baseAssetsExistCache = true
    return true
  }
-  async downloadCollection(slug: string): Promise<string[] | null> {
+  async downloadCollection(slug: string) {
-    const manifestService = new CollectionManifestService()
+    const collection = await CuratedCollection.query()
-    const spec = await manifestService.getSpecWithFallback<MapsSpec>('maps')
+      .where('slug', slug)
-    if (!spec) return null
+      .andWhere('type', 'map')
      .first()
    if (!collection) {
      return null
    }
-    const collection = spec.collections.find((c) => c.slug === slug)
+    const resources = await collection.related('resources').query().where('downloaded', false)
-    if (!collection) return null
+    if (resources.length === 0) {
-
+      return null
-    // Filter out already installed
+    }
    const installed = await InstalledResource.query().where('resource_type', 'map')
    const installedIds = new Set(installed.map((r) => r.resource_id))
    const toDownload = collection.resources.filter((r) => !installedIds.has(r.id))
    if (toDownload.length === 0) return null
    const downloadUrls = resources.map((res) => res.url)
    const downloadFilenames: string[] = []
-    for (const resource of toDownload) {
+    for (const url of downloadUrls) {
-      try {
+      const existing = await RunDownloadJob.getByUrl(url)
        assertNotPrivateUrl(resource.url)
      } catch {
        logger.warn(`[MapService] Blocked download from private/loopback URL: ${resource.url}`)
        continue
      }
      const existing = await RunDownloadJob.getActiveByUrl(resource.url)
      if (existing) {
-        logger.warn(`[MapService] Download already in progress for URL ${resource.url}, skipping.`)
+        logger.warn(`[MapService] Download already in progress for URL ${url}, skipping.`)
        continue
      }
-      const filename = resource.url.split('/').pop()
+      // Extract the filename from the URL
      const filename = url.split('/').pop()
      if (!filename) {
-        logger.warn(`[MapService] Could not determine filename from URL ${resource.url}, skipping.`)
+        logger.warn(`[MapService] Could not determine filename from URL ${url}, skipping.`)
        continue
      }
@ -173,18 +131,12 @@ export class MapService implements IMapService {
      const filepath = join(process.cwd(), this.mapStoragePath, 'pmtiles', filename)
      await RunDownloadJob.dispatch({
-        url: resource.url,
+        url,
        filepath,
        timeout: 30000,
        allowedMimeTypes: PMTILES_MIME_TYPES,
        forceNew: true,
        filetype: 'map',
        title: (resource as any).title || undefined,
        resourceMetadata: {
          resource_id: resource.id,
          version: resource.version,
          collection_ref: slug,
        },
      })
    }
@ -192,66 +144,11 @@ export class MapService implements IMapService {
  }
  async downloadRemoteSuccessCallback(urls: string[], _: boolean) {
-    // Create InstalledResource entries for downloaded map files
+    const resources = await CuratedCollectionResource.query().whereIn('url', urls)
-    for (const url of urls) {
+    for (const resource of resources) {
-      const filename = url.split('/').pop()
+      resource.downloaded = true
-      if (!filename) continue
+      await resource.save()
-
+      logger.info(`[MapService] Marked resource as downloaded: ${resource.url}`)
      const parsed = CollectionManifestService.parseMapFilename(filename)
      if (!parsed) continue
      const pmtilesDir = join(process.cwd(), this.mapStoragePath, 'pmtiles')
      const filepath = join(pmtilesDir, filename)
      const stats = await getFileStatsIfExists(filepath)
      try {
        // Capture the prior install for this resource_id before updateOrCreate
        // overwrites it, so we know the old file to clean up (#634).
        const prior = await InstalledResource.query()
          .where('resource_id', parsed.resource_id)
          .where('resource_type', 'map')
          .first()
        const { DateTime } = await import('luxon')
        await InstalledResource.updateOrCreate(
          { resource_id: parsed.resource_id, resource_type: 'map' },
          {
            version: parsed.version,
            url: url,
            file_path: filepath,
            file_size_bytes: stats ? Number(stats.size) : null,
            installed_at: DateTime.now(),
          }
        )
        logger.info(`[MapService] Created InstalledResource entry for: ${parsed.resource_id}`)
        // Remove the superseded prior version's pmtiles file if every safety
        // rail passes (see decideSupersededDeletion). Maps have no library index,
        // so a direct delete of the recorded old file is sufficient.
        const decision = decideSupersededDeletion({
          existing: prior ? { file_path: prior.file_path, version: prior.version } : null,
          newFilePath: filepath,
          newVersion: parsed.version,
          newFileExists: !!stats,
          storageBaseDir: pmtilesDir,
        })
        if (decision.delete && decision.path) {
          try {
            await deleteFileIfExists(decision.path)
            logger.info(
              `[MapService] Removed superseded ${parsed.resource_id} file: ${decision.path}`
            )
          } catch (err) {
            logger.warn(`[MapService] Failed to remove superseded file ${decision.path}:`, err)
          }
        } else if (decision.reason !== 'first_install' && decision.reason !== 'same_file') {
          logger.info(
            `[MapService] Kept prior ${parsed.resource_id} file (reason: ${decision.reason})`
          )
        }
      } catch (error) {
        logger.error(`[MapService] Failed to create InstalledResource for ${filename}:`, error)
      }
    }
  }
@ -261,7 +158,7 @@ export class MapService implements IMapService {
      throw new Error(`Invalid PMTiles file URL: ${url}. URL must end with .pmtiles`)
    }
-    const existing = await RunDownloadJob.getActiveByUrl(url)
+    const existing = await RunDownloadJob.getByUrl(url)
    if (existing) {
      throw new Error(`Download already in progress for URL ${url}`)
    }
@ -273,21 +170,6 @@ export class MapService implements IMapService {
    const filepath = join(process.cwd(), this.mapStoragePath, 'pmtiles', filename)
    // First, ensure base assets are present - regions depend on them
    const baseAssetsExist = await this.ensureBaseAssets()
    if (!baseAssetsExist) {
      throw new Error(
        'Base map assets are missing and could not be downloaded. Please check your connection and try again.'
      )
    }
    // Parse resource metadata
    const parsedFilename = CollectionManifestService.parseMapFilename(filename)
    const resourceMetadata = parsedFilename
      ? { resource_id: parsedFilename.resource_id, version: parsedFilename.version, collection_ref: null }
      : undefined
    // Dispatch background job
    const result = await RunDownloadJob.dispatch({
      url,
@ -296,7 +178,6 @@ export class MapService implements IMapService {
      allowedMimeTypes: PMTILES_MIME_TYPES,
      forceNew: true,
      filetype: 'map',
      resourceMetadata,
    })
    if (!result.job) {
@ -315,7 +196,6 @@ export class MapService implements IMapService {
    url: string
  ): Promise<{ filename: string; size: number } | { message: string }> {
    try {
      assertNotPrivateUrl(url)
      const parsed = new URL(url)
      if (!parsed.pathname.endsWith('.pmtiles')) {
        throw new Error(`Invalid PMTiles file URL: ${url}. URL must end with .pmtiles`)
@ -327,7 +207,6 @@ export class MapService implements IMapService {
      }
      // Perform a HEAD request to get the content length
      const { default: axios } = await import('axios')
      const response = await axios.head(url)
      if (response.status !== 200) {
@ -335,16 +214,15 @@ export class MapService implements IMapService {
      }
      const contentLength = response.headers['content-length']
-      const size = contentLength ? parseInt(contentLength.toString(), 10) : 0
+      const size = contentLength ? parseInt(contentLength, 10) : 0
      return { filename, size }
-    } catch (error: any) {
+    } catch (error) {
-      logger.error({ err: error }, '[MapService] Preflight check failed for URL')
+      return { message: `Preflight check failed: ${error.message}` }
      return { message: 'Preflight check failed. Please verify the URL is valid and accessible.' }
    }
  }
-  async generateStylesJSON(host: string | null = null, protocol: string = 'http'): Promise<BaseStylesFile> {
+  async generateStylesJSON() {
    if (!(await this.checkBaseAssetsExist())) {
      throw new Error('Base map assets are missing from storage/maps')
    }
@ -358,15 +236,12 @@ export class MapService implements IMapService {
    const rawStyles = JSON.parse(baseStyle.toString()) as BaseStylesFile
    const regions = (await this.listRegions()).files
    const sources = this.generateSourcesArray(regions)
-    /** If we have the host, use it to build public URLs, otherwise we'll fallback to defaults
+    const localUrl = env.get('URL')
-    * This is mainly useful because we need to know what host the user is accessing from in order to
+    const withProtocol = localUrl.startsWith('http') ? localUrl : `http://${localUrl}`
-    * properly generate URLs in the styles file
+    const baseUrlPath = urlJoin(this.mapStoragePath, this.basemapsAssetsDir)
-    * e.g. user is accessing from "example.com", but we would by default generate "localhost:8080/..." so maps would
+    const baseUrl = new URL(baseUrlPath, withProtocol).toString()
    * fail to load.
    */
    const sources = this.generateSourcesArray(host, regions, protocol)
    const baseUrl = this.getPublicFileBaseUrl(host, this.basemapsAssetsDir, protocol)
    const styles = await this.generateStylesFile(
      rawStyles,
@ -378,140 +253,62 @@ export class MapService implements IMapService {
    return styles
  }
-  async listCuratedCollections(): Promise<CollectionWithStatus[]> {
+  async checkBaseAssetsExist() {
-    const manifestService = new CollectionManifestService()
+    const storageContents = await this.listMapStorageItems()
-    return manifestService.getMapCollectionsWithStatus()
+    const baseStyleItem = storageContents.find(
      (item) => item.type === 'file' && item.name === this.baseStylesFile
    )
    const basemapsAssetsItem = storageContents.find(
      (item) => item.type === 'directory' && item.name === this.basemapsAssetsDir
    )
    return !!baseStyleItem && !!basemapsAssetsItem
  }
  async listCuratedCollections(): Promise<CuratedCollectionWithStatus[]> {
    const collections = await CuratedCollection.query().where('type', 'map').preload('resources')
    return collections.map((collection) => ({
      ...(collection.serialize() as CuratedCollection),
      all_downloaded: collection.resources.every((res) => res.downloaded),
    }))
  }
  async fetchLatestCollections(): Promise<boolean> {
    const manifestService = new CollectionManifestService()
    return manifestService.fetchAndCacheSpec('maps')
  }
  async ensureBaseAssets(): Promise<boolean> {
    const exists = await this.checkBaseAssetsExist()
    if (!exists) {
      const downloaded = await this.downloadBaseAssets()
      if (!downloaded) return false
    }
    try {
-      await this.ensureWorldBasemap()
+      const response = await axios.get<CuratedCollectionsFile>(COLLECTIONS_URL)
    } catch (err) {
      logger.warn(`[MapService] World basemap setup failed, continuing without it: ${err}`)
    }
-    return true
+      const validated = await vine.validate({
-  }
+        schema: curatedCollectionsFileSchema,
-
+        data: response.data,
  /**
   * Whether the low-zoom world basemap is present on disk. Checked directly (rather than trusting
   * the in-process `worldBasemapReady` flag) so callers get an accurate answer regardless of whether
   * `ensureWorldBasemap()` has run yet this process. Used to warn the user instead of showing a
   * silent grey map when the basemap was never provisioned (e.g. installed straight offline, #1030).
   */
  async checkWorldBasemapExists(): Promise<boolean> {
    const basePath = resolve(join(this.baseDirPath, 'pmtiles'))
    const filepath = resolve(join(basePath, WORLD_BASEMAP_FILENAME))
    if (!filepath.startsWith(basePath + sep)) return false
    const stats = await getFileStatsIfExists(filepath)
    const exists = !!stats && Number(stats.size) > 0
    if (exists) this.worldBasemapReady = true
    return exists
  }
  /**
   * Explicitly (re)provision the world basemap on demand. Ensures base assets exist, then extracts
   * the basemap if it isn't present yet. Requires internet — surfaced to the user as a "download the
   * base map" action so the one network dependency can be satisfied deliberately while online (#1030).
   */
  async provisionWorldBasemap(): Promise<boolean> {
    const baseAssetsExist = await this.ensureBaseAssets()
    if (!baseAssetsExist) return false
    await this.ensureWorldBasemap()
    return this.checkWorldBasemapExists()
  }
  /**
   * Extract a low-zoom global basemap once so the map isn't grey outside a
   * regional extract's polygon. Cheap (~15 MB, a handful of HTTP range
   * requests) and layered underneath regional sources at render time.
   *
   * Memoizes success in-process, and de-duplicates concurrent callers via a
   * shared in-flight promise so two simultaneous `/maps` requests on a cold
   * start don't both launch `pmtiles extract` against the same output path.
   */
  private async ensureWorldBasemap(): Promise<void> {
    if (this.worldBasemapReady) return
    if (this.worldBasemapInFlight) return this.worldBasemapInFlight
    this.worldBasemapInFlight = this._setupWorldBasemap().finally(() => {
      this.worldBasemapInFlight = null
    })
    return this.worldBasemapInFlight
  }
  private async _setupWorldBasemap(): Promise<void> {
    const basePath = resolve(join(this.baseDirPath, 'pmtiles'))
    const filepath = resolve(join(basePath, WORLD_BASEMAP_FILENAME))
    if (!filepath.startsWith(basePath + sep)) {
      throw new Error('Invalid world basemap path')
    }
    await ensureDirectoryExists(basePath)
    const existing = await getFileStatsIfExists(filepath)
    if (existing && Number(existing.size) > 0) {
      this.worldBasemapReady = true
      return
    }
    const info = await this.getGlobalMapInfo()
    const args = buildPmtilesExtractArgs({
      sourceUrl: info.url,
      outputFilepath: filepath,
      maxzoom: WORLD_BASEMAP_MAX_ZOOM,
      downloadThreads: 4,
    })
    logger.info(
      `[MapService] Extracting world basemap (z0-${WORLD_BASEMAP_MAX_ZOOM}) from ${info.url}`
    )
    try {
      await execFileAsync(PMTILES_BINARY_PATH, args, {
        timeout: WORLD_BASEMAP_EXTRACT_TIMEOUT_MS,
        maxBuffer: DRY_RUN_MAX_BUFFER,
      })
-      this.worldBasemapReady = true
+
-    } catch (err: any) {
+      for (const collection of validated.collections) {
-      await deleteFileIfExists(filepath)
+        const collectionResult = await CuratedCollection.updateOrCreate(
-      throw new Error(
+          { slug: collection.slug },
-        `pmtiles extract for world basemap failed: ${err.message}. stderr: ${err.stderr ?? ''}`
+          {
-      )
+            ...collection,
            type: 'map',
          }
        )
        logger.info(`[MapService] Upserted curated collection: ${collection.slug}`)
        await collectionResult.related('resources').createMany(collection.resources)
        logger.info(
          `[MapService] Upserted ${collection.resources.length} resources for collection: ${collection.slug}`
        )
      }
      return true
    } catch (error) {
      console.error(error)
      logger.error(`[MapService] Failed to download latest Kiwix collections:`, error)
      return false
    }
  }
-  private async checkBaseAssetsExist(useCache: boolean = true): Promise<boolean> {
+  private async listMapStorageItems(): Promise<FileEntry[]> {
    // Return cached result if available and caching is enabled
    if (useCache && this.baseAssetsExistCache !== null) {
      return this.baseAssetsExistCache
    }
    await ensureDirectoryExists(this.baseDirPath)
-
+    return await listDirectoryContents(this.baseDirPath)
    const baseStylePath = join(this.baseDirPath, this.baseStylesFile)
    const basemapsAssetsPath = join(this.baseDirPath, this.basemapsAssetsDir)
    const [baseStyleExists, basemapsAssetsExists] = await Promise.all([
      getFileStatsIfExists(baseStylePath),
      getFileStatsIfExists(basemapsAssetsPath),
    ])
    const exists = !!baseStyleExists && !!basemapsAssetsExists
    // update cache
    this.baseAssetsExistCache = exists
    return exists
  }
  private async listAllMapStorageItems(): Promise<FileEntry[]> {
@ -519,76 +316,28 @@ export class MapService implements IMapService {
    return await listDirectoryContentsRecursive(this.baseDirPath)
  }
-  /**
+  private generateSourcesArray(regions: FileEntry[]): BaseStylesFile['sources'][] {
-   * Compare two map-file versions (YYYY-MM, or null for an undated legacy file). Returns >0 if
+    const localUrl = env.get('URL')
   * `a` is newer than `b`, <0 if older, 0 if equal. A dated build is always newer than an undated
   * legacy file; two dated builds compare lexicographically (correct for zero-padded YYYY-MM).
   */
  private static compareMapVersions(a: string | null, b: string | null): number {
    if (a === b) return 0
    if (a === null) return -1
    if (b === null) return 1
    return a < b ? -1 : 1
  }
  private generateSourcesArray(host: string | null, regions: FileEntry[], protocol: string = 'http'): BaseStylesFile['sources'][] {
    const sources: BaseStylesFile['sources'][] = []
    const baseUrl = this.getPublicFileBaseUrl(host, 'pmtiles', protocol)
    // World basemap goes first so its layers render underneath regional extracts.
    // Only emitted when ensureWorldBasemap() succeeded — otherwise the style would
    // reference a file that doesn't exist and produce 404s on every tile request.
    if (this.worldBasemapReady) {
      const worldSource: BaseStylesFile['sources'] = {}
      worldSource[WORLD_BASEMAP_SOURCE_NAME] = {
        type: 'vector',
        attribution: PMTILES_ATTRIBUTION,
        url: `pmtiles://${urlJoin(baseUrl, WORLD_BASEMAP_FILENAME)}`,
      }
      sources.push(worldSource)
    }
    // Dedupe by region name, keeping only the newest file per region. The source name is the
    // date-stripped region (e.g. both "washington.pmtiles" and "washington_2025-12.pmtiles" map
    // to "washington"). Emitting both produces duplicate source keys and duplicate layer ids,
    // which MapLibre rejects outright — blanking the ENTIRE map, not just that region. Old copies
    // linger when a newer curated version installs (#634), so guard against it here so the style
    // stays valid even if cleanup hasn't run. A dated build beats an undated legacy file; between
    // two dated builds the later YYYY-MM wins (lexicographic compare is correct for that format).
    const bestByRegion = new Map<string, { region: FileEntry; version: string | null }>()
    for (const region of regions) {
      if (region.type === 'file' && region.name.endsWith('.pmtiles')) {
-        const parsed = CollectionManifestService.parseMapFilename(region.name)
+        const regionName = region.name.replace('.pmtiles', '')
-        const regionName = parsed ? parsed.resource_id : region.name.replace('.pmtiles', '')
+        const source: BaseStylesFile['sources'] = {}
-        const version = parsed?.version ?? null
+        const sourceUrl = new URL(
-        const existing = bestByRegion.get(regionName)
+          urlJoin(this.mapStoragePath, 'pmtiles', region.name),
-        if (!existing || MapService.compareMapVersions(version, existing.version) > 0) {
+          localUrl.startsWith('http') ? localUrl : `http://${localUrl}`
-          if (existing) {
+        ).toString()
-            logger.warn(
+
-              `[MapService] Duplicate map region "${regionName}": using "${region.name}" over "${existing.region.name}" (keeping newest)`
+        source[regionName] = {
-            )
+          type: 'vector',
-          }
+          attribution: PMTILES_ATTRIBUTION,
-          bestByRegion.set(regionName, { region, version })
+          url: `pmtiles://${sourceUrl}`,
        } else {
          logger.warn(
            `[MapService] Duplicate map region "${regionName}": skipping "${region.name}" in favor of "${existing.region.name}" (keeping newest)`
          )
        }
        sources.push(source)
      }
    }
    for (const [regionName, { region }] of bestByRegion) {
      const source: BaseStylesFile['sources'] = {}
      const sourceUrl = urlJoin(baseUrl, region.name)
      source[regionName] = {
        type: 'vector',
        attribution: PMTILES_ATTRIBUTION,
        url: `pmtiles://${sourceUrl}`,
      }
      sources.push(source)
    }
    return sources
  }
@ -624,283 +373,13 @@ export class MapService implements IMapService {
    return template
  }
  async getGlobalMapInfo(): Promise<ProtomapsBuildInfo> {
    const { default: axios } = await import('axios')
    const response = await axios.get(PROTOMAPS_BUILDS_METADATA_URL, { timeout: 15000 })
    const builds = response.data as Array<{ key: string; size: number }>
    if (!builds || builds.length === 0) {
      throw new Error('No protomaps builds found')
    }
    // Latest build first
    const sorted = builds.sort((a, b) => b.key.localeCompare(a.key))
    const latest = sorted[0]
    const dateStr = latest.key.replace('.pmtiles', '')
    const date = `${dateStr.slice(0, 4)}-${dateStr.slice(4, 6)}-${dateStr.slice(6, 8)}`
    return {
      url: `${PROTOMAPS_BUILD_BASE_URL}/${latest.key}`,
      date,
      size: latest.size,
      key: latest.key,
    }
  }
  async downloadGlobalMap(): Promise<{ filename: string; jobId?: string }> {
    const info = await this.getGlobalMapInfo()
    const existing = await RunDownloadJob.getByUrl(info.url)
    if (existing) {
      throw new Error(`Download already in progress for URL ${info.url}`)
    }
    const basePath = resolve(join(this.baseDirPath, 'pmtiles'))
    const filepath = resolve(join(basePath, info.key))
    // Prevent path traversal — resolved path must stay within the storage directory
    if (!filepath.startsWith(basePath + sep)) {
      throw new Error('Invalid filename')
    }
    // First, ensure base assets are present - the global map depends on them
    const baseAssetsExist = await this.ensureBaseAssets()
    if (!baseAssetsExist) {
      throw new Error(
        'Base map assets are missing and could not be downloaded. Please check your connection and try again.'
      )
    }
    // forceNew: false so retries resume partial downloads
    const result = await RunDownloadJob.dispatch({
      url: info.url,
      filepath,
      timeout: 30000,
      allowedMimeTypes: PMTILES_MIME_TYPES,
      forceNew: false,
      filetype: 'map',
    })
    if (!result.job) {
      throw new Error('Failed to dispatch download job')
    }
    logger.info(`[MapService] Dispatched global map download job ${result.job.id}`)
    return {
      filename: info.key,
      jobId: result.job?.id,
    }
  }
  async listCountries(): Promise<Country[]> {
    return CountriesService.getInstance().list()
  }
  async listCountryGroups(): Promise<CountryGroup[]> {
    return CountriesService.getInstance().listGroups()
  }
  async extractPreflight(params: {
    countries: CountryCode[]
    maxzoom?: number
  }): Promise<MapExtractPreflight> {
    this.validateMaxzoom(params.maxzoom)
    const countries = await CountriesService.getInstance().resolveCodes(params.countries)
    const regionFilepath = await CountriesService.getInstance().writeRegionFile(countries)
    const info = await this.getGlobalMapInfo()
    return this.runDryRun(info, regionFilepath, params.maxzoom)
  }
  private async runDryRun(
    info: { url: string; date: string; key: string },
    regionFilepath: string,
    maxzoom?: number
  ): Promise<MapExtractPreflight> {
    const dryRunOutput = join(tmpdir(), `pmtiles-dry-run-${randomBytes(6).toString('hex')}.pmtiles`)
    const args = buildPmtilesExtractArgs({
      sourceUrl: info.url,
      outputFilepath: dryRunOutput,
      regionFilepath,
      maxzoom,
      dryRun: true,
    })
    let stdout = ''
    let stderr = ''
    try {
      const result = await execFileAsync(PMTILES_BINARY_PATH, args, {
        timeout: DRY_RUN_TIMEOUT_MS,
        maxBuffer: DRY_RUN_MAX_BUFFER,
      })
      stdout = result.stdout
      stderr = result.stderr
    } catch (err: any) {
      throw new Error(
        `pmtiles extract --dry-run failed: ${err.message}. stderr: ${err.stderr ?? ''}`
      )
    }
    const parsed = this.parseDryRunOutput(stdout + '\n' + stderr)
    return {
      tiles: parsed.tiles,
      bytes: parsed.bytes,
      source: { url: info.url, date: info.date, key: info.key },
    }
  }
  async extractRegion(params: {
    countries: CountryCode[]
    maxzoom?: number
    label?: string
    estimatedBytes?: number
  }): Promise<{ filename: string; jobId?: string }> {
    this.validateMaxzoom(params.maxzoom)
    const countriesService = CountriesService.getInstance()
    const countries = await countriesService.resolveCodes(params.countries)
    const regionFilepath = await countriesService.writeRegionFile(countries)
    const maxzoom = params.maxzoom ?? EXTRACT_DEFAULT_MAX_ZOOM
    const [baseAssetsExist, info, groups] = await Promise.all([
      this.ensureBaseAssets(),
      this.getGlobalMapInfo(),
      countriesService.listGroups(),
    ])
    if (!baseAssetsExist) {
      throw new Error(
        'Base map assets are missing and could not be downloaded. Please check your connection and try again.'
      )
    }
    const groupMatch = findExactGroupMatch(countries, groups)
    const slug = this.buildRegionSlug(countries, groupMatch)
    const dateSlug = info.key.replace('.pmtiles', '')
    const filename = `${slug}_${dateSlug}_z${maxzoom}.pmtiles`
    const basePath = resolve(join(this.baseDirPath, 'pmtiles'))
    const filepath = resolve(join(basePath, filename))
    if (!filepath.startsWith(basePath + sep)) {
      throw new Error('Invalid filename')
    }
    let estimatedBytes = params.estimatedBytes ?? 0
    if (estimatedBytes === 0) {
      try {
        const preflight = await this.runDryRun(info, regionFilepath, maxzoom)
        estimatedBytes = preflight.bytes
      } catch (err) {
        logger.warn(`[MapService] extractRegion preflight failed, proceeding without estimate: ${err}`)
      }
    }
    const title = params.label ?? this.buildRegionTitle(countries, groupMatch)
    const result = await RunExtractPmtilesJob.dispatch({
      sourceUrl: info.url,
      outputFilepath: filepath,
      regionFilepath,
      maxzoom,
      estimatedBytes,
      filetype: 'map',
      title,
      resourceMetadata: {
        resource_id: slug,
        version: dateSlug,
        collection_ref: null,
      },
    })
    if (!result.job) {
      throw new Error('Failed to dispatch extract job')
    }
    logger.info(
      `[MapService] Dispatched extract job ${result.job.id} for ${filename} ` +
        `(countries=[${countries.join(',')}] maxzoom=${maxzoom} est=${estimatedBytes} bytes)`
    )
    return {
      filename,
      jobId: result.job.id,
    }
  }
  private buildRegionSlug(countries: CountryCode[], groupMatch: CountryGroup | null): string {
    if (groupMatch) return groupMatch.id
    if (countries.length === 1) return countries[0].toLowerCase()
    const hash = createHash('sha1').update(countries.join(',')).digest('hex').slice(0, 8)
    return `custom-${hash}`
  }
  private buildRegionTitle(countries: CountryCode[], groupMatch: CountryGroup | null): string {
    if (groupMatch) return groupMatch.name
    if (countries.length === 1) return countries[0]
    if (countries.length <= 3) return countries.join(', ')
    return `${countries.slice(0, 2).join(', ')} +${countries.length - 2} more`
  }
  private validateMaxzoom(maxzoom: number | undefined): void {
    if (typeof maxzoom !== 'number') return
    if (
      !Number.isInteger(maxzoom) ||
      maxzoom < EXTRACT_MIN_ZOOM ||
      maxzoom > EXTRACT_MAX_ZOOM
    ) {
      throw new Error(
        `maxzoom must be an integer in [${EXTRACT_MIN_ZOOM}, ${EXTRACT_MAX_ZOOM}]`
      )
    }
  }
  // go-pmtiles output format isn't stable across versions — parse loosely and
  // fall back to zeros. The extract can still proceed without an estimate.
  private parseDryRunOutput(output: string): { tiles: number; bytes: number } {
    let bytes = 0
    let tiles = 0
    const byteLine = output.match(/archive\s+size\s+of\s+([\d,.]+)\s*(B|KB|MB|GB|TB|bytes?)?/i)
    if (byteLine) {
      const raw = parseFloat(byteLine[1].replace(/,/g, ''))
      const unit = (byteLine[2] ?? 'B').toUpperCase()
      const multipliers: Record<string, number> = {
        B: 1,
        BYTE: 1,
        BYTES: 1,
        KB: 1_000,
        MB: 1_000_000,
        GB: 1_000_000_000,
        TB: 1_000_000_000_000,
      }
      bytes = Math.round(raw * (multipliers[unit] ?? 1))
    }
    const tileLine = output.match(/(?:tiles\s+to\s+extract|tiles)[^\d]*([\d,]+)/i)
    if (tileLine) {
      tiles = parseInt(tileLine[1].replace(/,/g, ''), 10) || 0
    }
    return { tiles, bytes }
  }
  async delete(file: string): Promise<void> {
    let fileName = file
-    if (!fileName.endsWith('.pmtiles')) {
+    if (!fileName.endsWith('.zim')) {
-      fileName += '.pmtiles'
+      fileName += '.zim'
    }
-    if (fileName === WORLD_BASEMAP_FILENAME) {
+    const fullPath = join(this.baseDirPath, fileName)
      throw new Error('The world basemap cannot be deleted')
    }
    const basePath = resolve(join(this.baseDirPath, 'pmtiles'))
    const fullPath = resolve(join(basePath, fileName))
    // Prevent path traversal — resolved path must stay within the storage directory
    if (!fullPath.startsWith(basePath + sep)) {
      throw new Error('Invalid filename')
    }
    const exists = await getFileStatsIfExists(fullPath)
    if (!exists) {
@ -908,80 +387,5 @@ export class MapService implements IMapService {
    }
    await deleteFileIfExists(fullPath)
    // Clean up InstalledResource entry
    const parsed = CollectionManifestService.parseMapFilename(fileName)
    if (parsed) {
      await InstalledResource.query()
        .where('resource_id', parsed.resource_id)
        .where('resource_type', 'map')
        .delete()
      logger.info(`[MapService] Deleted InstalledResource entry for: ${parsed.resource_id}`)
    }
  }
  /**
   * Gets the appropriate public URL for a map asset depending on environment. The host and protocol that the user
   * is accessing the maps from must match the host and protocol used in the generated URLs, otherwise maps will fail to load.
   * If you make changes to this function, you need to ensure it handles all the following cases correctly:
   * - No host provided (should default to localhost or env URL)
   * - Host provided as full URL (e.g. "http://example.com:8080")
   * - Host provided as host:port (e.g. "example.com:8080")
   * - Host provided as bare hostname (e.g. "example.com")
   * @param specifiedHost - the host as provided by the user/request, can be null or in various formats (full URL, host:port, bare hostname)
   * @param childPath - the path to append to the base URL (e.g. "basemaps-assets", "pmtiles")
   * @param protocol - the protocol to use in the generated URL (e.g. "http", "https"), defaults to "http"
   * @returns the public URL for the map asset
   */
  private getPublicFileBaseUrl(specifiedHost: string | null, childPath: string, protocol: string = 'http'): string {
    function getHost() {
      try {
        const localUrlRaw = env.get('URL')
        if (!localUrlRaw) return 'localhost'
        const localUrl = new URL(localUrlRaw)
        return localUrl.host
      } catch (error) {
        return 'localhost'
      }
    }
    function specifiedHostOrDefault() {
      if (specifiedHost === null) {
        return getHost()
      }
      // Try as a full URL first (e.g. "http://example.com:8080")
      try {
        const specifiedUrl = new URL(specifiedHost)
        if (specifiedUrl.host) return specifiedUrl.host
      } catch {}
      // Try as a bare host or host:port (e.g. "nomad-box:8080", "192.168.1.1:8080", "example.com")
      try {
        const specifiedUrl = new URL(`http://${specifiedHost}`)
        if (specifiedUrl.host) return specifiedUrl.host
      } catch {}
      return getHost()
    }
    const host = specifiedHostOrDefault();
    const withProtocol = `${protocol}://${host}`
    const baseUrlPath =
      process.env.NODE_ENV === 'production' ? childPath : urlJoin(this.mapStoragePath, childPath)
    const baseUrl = new URL(baseUrlPath, withProtocol).toString()
    return baseUrl
  }
 }
 function findExactGroupMatch(
  countries: CountryCode[],
  groups: CountryGroup[]
 ): CountryGroup | null {
  return (
    groups.find(
      (g) =>
        g.countries.length === countries.length &&
        g.countries.every((c, i) => c === countries[i])
    ) ?? null
  )
 }
--- a/admin/app/services/nomad_md_service.ts
+++ b/admin/app/services/nomad_md_service.ts
@ -1,51 +0,0 @@
 import { rename, writeFile } from 'node:fs/promises'
 import { dirname } from 'node:path'
 import { randomUUID } from 'node:crypto'
 import app from '@adonisjs/core/services/app'
 import { ensureDirectoryExists, getFile } from '../utils/fs.js'
 /**
 * Manages the user-editable `NOMAD.md` file that is injected as a system prompt
 * during chat completions. The file lives in the admin storage directory
 * (`/app/storage/NOMAD.md` in the container, `/opt/project-nomad/storage/NOMAD.md`
 * on the host), so an end user can also edit it directly on disk. The file is
 * created lazily on the first save — a missing or empty file simply means no
 * custom prompt is injected.
 */
 export class NomadMdService {
  static STORAGE_PATH = 'storage/NOMAD.md'
  private get filePath(): string {
    return app.makePath(NomadMdService.STORAGE_PATH)
  }
  /**
   * Read the raw file contents. Returns an empty string when the file does not
   * exist yet (the editor renders a template client-side in that case).
   */
  async read(): Promise<string> {
    const content = await getFile(this.filePath, 'string')
    return content ?? ''
  }
  /**
   * Persist the file. Written atomically (tmp file + rename) so a concurrent
   * on-disk edit never observes a half-written file.
   */
  async write(content: string): Promise<void> {
    const filePath = this.filePath
    await ensureDirectoryExists(dirname(filePath))
    const tmpPath = `${filePath}.tmp.${randomUUID()}`
    await writeFile(tmpPath, content, 'utf-8')
    await rename(tmpPath, filePath)
  }
  /**
   * The trimmed contents suitable for injection as a system message, or `null`
   * when the file is missing or blank (so chat behaviour is unchanged).
   */
  async getSystemPrompt(): Promise<string | null> {
    const content = (await this.read()).trim()
    return content.length > 0 ? content : null
  }
 }
--- a/admin/app/services/ollama_service.ts
+++ b/admin/app/services/ollama_service.ts
--- a/admin/app/services/openwebui_service.ts
+++ b/admin/app/services/openwebui_service.ts
@ -0,0 +1,646 @@
 import { inject } from '@adonisjs/core'
 import logger from '@adonisjs/core/services/logger'
 import { DockerService } from './docker_service.js'
 import axios from 'axios'
 import { NomadOllamaModel, OllamaModelListing } from '../../types/ollama.js'
 import fs from 'node:fs/promises'
 import path from 'node:path'
 import { PassThrough } from 'node:stream'
 import { DownloadModelJob } from '#jobs/download_model_job'
 const NOMAD_MODELS_API_BASE_URL = 'https://api.projectnomad.us/api/v1/ollama/models'
 const MODELS_CACHE_FILE = path.join(process.cwd(), 'storage', 'ollama-models-cache.json')
 const CACHE_MAX_AGE_MS = 24 * 60 * 60 * 1000 // 24 hours
@inject()
 export class OpenWebUIService {
  constructor(private dockerService: DockerService) {}
  /** We need to call this in the DownloadModelJob, so it can't be private,
   * but shouldn't be called directly (dispatch job instead)
   */
  async _downloadModel(
    model: string,
    onProgress?: (progress: {
      status: string
      completed?: number
      total?: number
      percent?: number
    }) => void
  ): Promise<{ success: boolean; message: string }> {
    return new Promise((resolve) => {
      try {
        const container = this.dockerService.docker.getContainer(DockerService.OLLAMA_SERVICE_NAME)
        if (!container) {
          logger.warn('[OpenWebUIService] Ollama container is not running. Cannot download model.')
          resolve({
            success: false,
            message: 'Ollama is not running. Please start Ollama and try again.',
          })
          return
        }
        container.exec(
          {
            Cmd: ['ollama', 'pull', model],
            AttachStdout: true,
            AttachStderr: true,
          },
          (err, exec) => {
            if (err) {
              logger.error(
                `[OpenWebUIService] Failed to execute model download command: ${
                  err instanceof Error ? err.message : err
                }`
              )
              resolve({ success: false, message: 'Failed to execute download command.' })
              return
            }
            if (!exec) {
              logger.error('[OpenWebUIService] No exec instance returned from exec command')
              resolve({ success: false, message: 'Failed to create exec instance.' })
              return
            }
            exec.start(
              {
                hijack: true,
                stdin: false,
              },
              (startErr, stream) => {
                if (startErr) {
                  logger.error(
                    `[OpenWebUIService] Failed to start exec stream: ${
                      startErr instanceof Error ? startErr.message : startErr
                    }`
                  )
                  resolve({ success: false, message: 'Failed to start download stream.' })
                  return
                }
                if (!stream) {
                  logger.error('[OpenWebUIService] No stream returned when starting exec')
                  resolve({ success: false, message: 'No stream available.' })
                  return
                }
                // Create PassThrough streams to capture output
                const stdout = new PassThrough()
                const stderr = new PassThrough()
                // Demultiplex the Docker stream
                this.dockerService.docker.modem.demuxStream(stream, stdout, stderr)
                // Capture and parse stdout (if any)
                stdout.on('data', (chunk) => {
                  const output = chunk.toString()
                  logger.info(`[OpenWebUIService] Model download (stdout): ${output}`)
                })
                // Capture stderr - ollama sends progress/status here (not necessarily errors)
                stderr.on('data', (chunk) => {
                  const output = chunk.toString()
                  // Check if this is an actual error message
                  if (
                    output.toLowerCase().includes('error') ||
                    output.toLowerCase().includes('failed')
                  ) {
                    logger.error(`[OpenWebUIService] Model download error: ${output}`)
                  } else {
                    // This is normal progress/status output from ollama
                    logger.info(`[OpenWebUIService] Model download progress: ${output}`)
                    // Parse JSON progress if available
                    try {
                      const lines = output
                        .split('\n')
                        .filter(
                          (line: any) => typeof line.trim() === 'string' && line.trim().length > 0
                        )
                      for (const line of lines) {
                        const parsed = JSON.parse(line)
                        if (parsed.status) {
                          const progressData: {
                            status: string
                            completed?: number
                            total?: number
                            percent?: number
                          } = {
                            status: parsed.status,
                          }
                          // Extract byte progress if available
                          if (parsed.completed !== undefined && parsed.total !== undefined) {
                            progressData.completed = parsed.completed
                            progressData.total = parsed.total
                            progressData.percent = Math.round(
                              (parsed.completed / parsed.total) * 100
                            )
                          }
                          // Call progress callback
                          if (onProgress) {
                            onProgress(progressData)
                          }
                          // Log structured progress
                          if (progressData.percent !== undefined) {
                            logger.info(
                              `[OpenWebUIService] ${progressData.status}: ${progressData.percent}% (${progressData.completed}/${progressData.total} bytes)`
                            )
                          } else {
                            logger.info(`[OpenWebUIService] ${progressData.status}`)
                          }
                        }
                      }
                    } catch {
                      // Not JSON, already logged above
                    }
                  }
                })
                // Handle stream end
                stream.on('end', () => {
                  logger.info(
                    `[OpenWebUIService] Model download process ended for model "${model}"`
                  )
                  resolve({
                    success: true,
                    message: 'Model download completed successfully.',
                  })
                })
                // Handle stream errors
                stream.on('error', (streamErr) => {
                  logger.error(
                    `[OpenWebUIService] Error during model download stream: ${
                      streamErr instanceof Error ? streamErr.message : streamErr
                    }`
                  )
                  resolve({
                    success: false,
                    message: 'Error occurred during model download.',
                  })
                })
              }
            )
          }
        )
      } catch (error) {
        logger.error(
          `[OpenWebUIService] Failed to download model "${model}": ${
            error instanceof Error ? error.message : error
          }`
        )
        resolve({ success: false, message: 'Failed to download model.' })
      }
    })
  }
  async deleteModel(model: string): Promise<{ success: boolean; message: string }> {
    return new Promise((resolve) => {
      try {
        const container = this.dockerService.docker.getContainer(DockerService.OLLAMA_SERVICE_NAME)
        if (!container) {
          logger.warn('[OpenWebUIService] Ollama container is not running. Cannot remove model.')
          resolve({
            success: false,
            message: 'Ollama is not running. Please start Ollama and try again.',
          })
          return
        }
        container.exec(
          {
            Cmd: ['ollama', 'rm', model],
            AttachStdout: true,
            AttachStderr: true,
          },
          (err, exec) => {
            if (err) {
              logger.error(
                `[OpenWebUIService] Failed to execute model remove command: ${
                  err instanceof Error ? err.message : err
                }`
              )
              resolve({ success: false, message: 'Failed to execute remove command.' })
              return
            }
            if (!exec) {
              logger.error('[OpenWebUIService] No exec instance returned from remove command')
              resolve({ success: false, message: 'Failed to create exec instance.' })
              return
            }
            exec.start(
              {
                hijack: true,
                stdin: false,
              },
              (startErr, stream) => {
                if (startErr) {
                  logger.error(
                    `[OpenWebUIService] Failed to start exec stream for remove: ${
                      startErr instanceof Error ? startErr.message : startErr
                    }`
                  )
                  resolve({ success: false, message: 'Failed to start remove command.' })
                  return
                }
                if (!stream) {
                  logger.error('[OpenWebUIService] No stream returned for remove command')
                  resolve({ success: false, message: 'No stream available.' })
                  return
                }
                const stdout = new PassThrough()
                const stderr = new PassThrough()
                let output = ''
                let errorOutput = ''
                this.dockerService.docker.modem.demuxStream(stream, stdout, stderr)
                stdout.on('data', (chunk) => {
                  output += chunk.toString()
                })
                stderr.on('data', (chunk) => {
                  errorOutput += chunk.toString()
                })
                stream.on('end', () => {
                  if (errorOutput) {
                    logger.error(`[OpenWebUIService] Error removing model: ${errorOutput}`)
                    resolve({
                      success: false,
                      message: errorOutput.trim() || 'Failed to remove model.',
                    })
                    return
                  }
                  logger.info(`[OpenWebUIService] Successfully removed model "${model}"`)
                  if (output) {
                    logger.info(`[OpenWebUIService] Remove output: ${output}`)
                  }
                  resolve({
                    success: true,
                    message: 'Model removed successfully.',
                  })
                })
                stream.on('error', (streamErr) => {
                  logger.error(
                    `[OpenWebUIService] Stream error during model remove: ${
                      streamErr instanceof Error ? streamErr.message : streamErr
                    }`
                  )
                  resolve({
                    success: false,
                    message: 'Error occurred while removing model.',
                  })
                })
              }
            )
          }
        )
      } catch (error) {
        logger.error(
          `[OpenWebUIService] Failed to remove model "${model}": ${
            error instanceof Error ? error.message : error
          }`
        )
        resolve({ success: false, message: 'Failed to remove model.' })
      }
    })
  }
  async dispatchModelDownload(modelName: string): Promise<{ success: boolean; message: string }> {
    try {
      logger.info(`[OpenWebUIService] Dispatching model download for ${modelName} via job queue`)
      await DownloadModelJob.dispatch({
        modelName,
      })
      return {
        success: true,
        message:
          'Model download has been queued successfully. It will start shortly after Ollama and Open WebUI are ready (if not already).',
      }
    } catch (error) {
      logger.error(
        `[OpenWebUIService] Failed to dispatch model download for ${modelName}: ${error instanceof Error ? error.message : error}`
      )
      return {
        success: false,
        message: 'Failed to queue model download. Please try again.',
      }
    }
  }
  async getAvailableModels(
    { sort, recommendedOnly }: { sort?: 'pulls' | 'name'; recommendedOnly?: boolean } = {
      sort: 'pulls',
      recommendedOnly: false,
    }
  ): Promise<NomadOllamaModel[] | null> {
    try {
      const models = await this.retrieveAndRefreshModels(sort)
      if (!models) {
        return null
      }
      if (!recommendedOnly) {
        return models
      }
      // If recommendedOnly is true, only return the first three models (if sorted by pulls, these will be the top 3)
      const sortedByPulls = sort === 'pulls' ? models : this.sortModels(models, 'pulls')
      const firstThree = sortedByPulls.slice(0, 3)
      // Only return the first tag of each of these models (should be the most lightweight variant)
      const recommendedModels = firstThree.map((model) => {
        return {
          ...model,
          tags: model.tags && model.tags.length > 0 ? [model.tags[0]] : [],
        }
      })
      return recommendedModels
    } catch (error) {
      logger.error(
        `[OpenWebUIService] Failed to get available models: ${error instanceof Error ? error.message : error}`
      )
      return null
    }
  }
  async getInstalledModels(): Promise<OllamaModelListing[] | null> {
    return new Promise((resolve) => {
      try {
        const container = this.dockerService.docker.getContainer(DockerService.OLLAMA_SERVICE_NAME)
        if (!container) {
          logger.warn('[OpenWebUIService] Ollama container is not running. Cannot list models.')
          resolve(null)
          return
        }
        container.exec(
          {
            Cmd: ['ollama', 'list'],
            AttachStdout: true,
            AttachStderr: true,
          },
          (err, exec) => {
            if (err) {
              logger.error(
                `[OpenWebUIService] Failed to execute ollama list command: ${
                  err instanceof Error ? err.message : err
                }`
              )
              resolve(null)
              return
            }
            if (!exec) {
              logger.error('[OpenWebUIService] No exec instance returned from ollama list')
              resolve(null)
              return
            }
            exec.start(
              {
                hijack: true,
                stdin: false,
              },
              (startErr, stream) => {
                if (startErr) {
                  logger.error(
                    `[OpenWebUIService] Failed to start exec stream for ollama list: ${
                      startErr instanceof Error ? startErr.message : startErr
                    }`
                  )
                  resolve(null)
                  return
                }
                if (!stream) {
                  logger.error('[OpenWebUIService] No stream returned for ollama list')
                  resolve(null)
                  return
                }
                const stdout = new PassThrough()
                const stderr = new PassThrough()
                let output = ''
                let errorOutput = ''
                this.dockerService.docker.modem.demuxStream(stream, stdout, stderr)
                stdout.on('data', (chunk) => {
                  output += chunk.toString()
                })
                stderr.on('data', (chunk) => {
                  errorOutput += chunk.toString()
                })
                stream.on('end', () => {
                  if (errorOutput) {
                    logger.error(
                      `[OpenWebUIService] Error from ollama list command: ${errorOutput}`
                    )
                  }
                  if (!output) {
                    logger.info('[OpenWebUIService] No models installed')
                    resolve([])
                    return
                  }
                  try {
                    // Parse the tabular output from ollama list
                    // Expected format:
                    // NAME                    ID              SIZE      MODIFIED
                    // llama2:latest          abc123def456    3.8 GB    2 days ago
                    const lines = output.split('\n').filter((line) => line.trim())
                    // Skip header line and parse model entries
                    const models: OllamaModelListing[] = []
                    for (let i = 1; i < lines.length; i++) {
                      const line = lines[i].trim()
                      if (!line) continue
                      // Split by whitespace (2+ spaces to handle columns with spaces)
                      const parts = line.split(/\s{2,}/)
                      if (parts.length >= 4) {
                        models.push({
                          name: parts[0].trim(),
                          id: parts[1].trim(),
                          size: parts[2].trim(),
                          modified: parts[3].trim(),
                        })
                      }
                    }
                    logger.info(`[OpenWebUIService] Found ${models.length} installed models`)
                    resolve(models)
                  } catch (parseError) {
                    logger.error(
                      `[OpenWebUIService] Failed to parse ollama list output: ${
                        parseError instanceof Error ? parseError.message : parseError
                      }`
                    )
                    logger.debug(`[OpenWebUIService] Raw output: ${output}`)
                    resolve(null)
                  }
                })
                stream.on('error', (streamErr) => {
                  logger.error(
                    `[OpenWebUIService] Stream error during ollama list: ${
                      streamErr instanceof Error ? streamErr.message : streamErr
                    }`
                  )
                  resolve(null)
                })
              }
            )
          }
        )
      } catch (error) {
        logger.error(
          `[OpenWebUIService] Failed to get installed models: ${
            error instanceof Error ? error.message : error
          }`
        )
        resolve(null)
      }
    })
  }
  private async retrieveAndRefreshModels(
    sort?: 'pulls' | 'name'
  ): Promise<NomadOllamaModel[] | null> {
    try {
      const cachedModels = await this.readModelsFromCache()
      if (cachedModels) {
        logger.info('[OpenWebUIService] Using cached available models data')
        return this.sortModels(cachedModels, sort)
      }
      logger.info('[OpenWebUIService] Fetching fresh available models from API')
      const response = await axios.get(NOMAD_MODELS_API_BASE_URL)
      if (!response.data || !Array.isArray(response.data.models)) {
        logger.warn(
          `[OpenWebUIService] Invalid response format when fetching available models: ${JSON.stringify(response.data)}`
        )
        return null
      }
      const models = response.data.models as NomadOllamaModel[]
      await this.writeModelsToCache(models)
      return this.sortModels(models, sort)
    } catch (error) {
      logger.error(
        `[OpenWebUIService] Failed to retrieve models from Nomad API: ${
          error instanceof Error ? error.message : error
        }`
      )
      return null
    }
  }
  private async readModelsFromCache(): Promise<NomadOllamaModel[] | null> {
    try {
      const stats = await fs.stat(MODELS_CACHE_FILE)
      const cacheAge = Date.now() - stats.mtimeMs
      if (cacheAge > CACHE_MAX_AGE_MS) {
        logger.info('[OpenWebUIService] Cache is stale, will fetch fresh data')
        return null
      }
      const cacheData = await fs.readFile(MODELS_CACHE_FILE, 'utf-8')
      const models = JSON.parse(cacheData) as NomadOllamaModel[]
      if (!Array.isArray(models)) {
        logger.warn('[OpenWebUIService] Invalid cache format, will fetch fresh data')
        return null
      }
      return models
    } catch (error) {
      // Cache doesn't exist or is invalid
      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
        logger.warn(
          `[OpenWebUIService] Error reading cache: ${error instanceof Error ? error.message : error}`
        )
      }
      return null
    }
  }
  private async writeModelsToCache(models: NomadOllamaModel[]): Promise<void> {
    try {
      await fs.mkdir(path.dirname(MODELS_CACHE_FILE), { recursive: true })
      await fs.writeFile(MODELS_CACHE_FILE, JSON.stringify(models, null, 2), 'utf-8')
      logger.info('[OpenWebUIService] Successfully cached available models')
    } catch (error) {
      logger.warn(
        `[OpenWebUIService] Failed to write models cache: ${error instanceof Error ? error.message : error}`
      )
    }
  }
  private sortModels(models: NomadOllamaModel[], sort?: 'pulls' | 'name'): NomadOllamaModel[] {
    if (sort === 'pulls') {
      // Sort by estimated pulls (it should be a string like "1.2K", "500", "4M" etc.)
      models.sort((a, b) => {
        const parsePulls = (pulls: string) => {
          const multiplier = pulls.endsWith('K')
            ? 1_000
            : pulls.endsWith('M')
              ? 1_000_000
              : pulls.endsWith('B')
                ? 1_000_000_000
                : 1
          return parseFloat(pulls) * multiplier
        }
        return parsePulls(b.estimated_pulls) - parsePulls(a.estimated_pulls)
      })
    } else if (sort === 'name') {
      models.sort((a, b) => a.name.localeCompare(b.name))
    }
    // Always sort model.tags by the size field in descending order
    // Size is a string like '75GB', '8.5GB', '2GB' etc. Smaller models first
    models.forEach((model) => {
      if (model.tags && Array.isArray(model.tags)) {
        model.tags.sort((a, b) => {
          const parseSize = (size: string) => {
            const multiplier = size.endsWith('KB')
              ? 1 / 1_000
              : size.endsWith('MB')
                ? 1 / 1_000_000
                : size.endsWith('GB')
                  ? 1
                  : size.endsWith('TB')
                    ? 1_000
                    : 0 // Unknown size format
            return parseFloat(size) * multiplier
          }
          return parseSize(a.size) - parseSize(b.size)
        })
      }
    })
    return models
  }
 }
--- a/admin/app/services/queue_service.ts
+++ b/admin/app/services/queue_service.ts
@ -1,25 +1,9 @@
 import { Queue } from 'bullmq'
 import queueConfig from '#config/queue'
 // Process-wide singleton. Instantiating a fresh QueueService per dispatch /
 // status lookup leaks connections, and under sustained job churn (e.g.
 // multi-batch ZIM ingestion enqueueing a continuation every few seconds) it
 // saturates Redis's maxclients within hours. All queues additionally reuse the
 // single shared ioredis instance exported from #config/queue (#885).
 export class QueueService {
  private queues: Map<string, Queue> = new Map()
  private static _instance: QueueService | null = null
  private constructor() {}
  static getInstance(): QueueService {
    if (!QueueService._instance) {
      QueueService._instance = new QueueService()
    }
    return QueueService._instance
  }
  getQueue(name: string): Queue {
    if (!this.queues.has(name)) {
      const queue = new Queue(name, {
@ -34,6 +18,5 @@ export class QueueService {
    for (const queue of this.queues.values()) {
      await queue.close()
    }
    this.queues.clear()
  }
 }
--- a/admin/app/services/rag_service.ts
+++ b/admin/app/services/rag_service.ts
--- a/admin/app/services/system_service.ts
+++ b/admin/app/services/system_service.ts
--- a/admin/app/services/system_update_service.ts
+++ b/admin/app/services/system_update_service.ts
@ -2,7 +2,6 @@ import logger from '@adonisjs/core/services/logger'
 import { readFileSync, existsSync } from 'fs'
 import { writeFile } from 'fs/promises'
 import { join } from 'path'
 import KVStore from '#models/kv_store'
 interface UpdateStatus {
  stage: 'idle' | 'starting' | 'pulling' | 'pulled' | 'recreating' | 'complete' | 'error'
@ -18,20 +17,11 @@ export class SystemUpdateService {
  private static LOG_FILE = join(SystemUpdateService.SHARED_DIR, 'update-log')
  /**
-   * Requests a system update by creating a request file that the sidecar will detect.
+   * Requests a system update by creating a request file that the sidecar will detect
   *
   * @param options.targetTag - Explicit Docker image tag to install (e.g. "v1.33.2").
   *   When omitted, falls back to the cached `system.latestVersion` (manual-update
   *   behavior). Auto-update passes an eligibility-vetted tag here, which may differ
   *   from `system.latestVersion` when the newest release is a major bump.
   * @param options.requester - Identifier recorded in the request file for auditing.
   */
-  async requestUpdate(options?: {
+  async requestUpdate(): Promise<{ success: boolean; message: string }> {
    targetTag?: string
    requester?: string
  }): Promise<{ success: boolean; message: string }> {
    try {
-      const currentStatus = this.getUpdateStatus()
+      const currentStatus = this.getUpdateStatus()      
      if (currentStatus && !['idle', 'complete', 'error'].includes(currentStatus.stage)) {
        return {
          success: false,
@ -39,32 +29,23 @@ export class SystemUpdateService {
        }
      }
      // Determine the Docker image tag to install. Prefer an explicit caller-supplied
      // tag; otherwise use the cached latest version.
      let targetTag = options?.targetTag
      if (!targetTag) {
        const latestVersion = await KVStore.getValue('system.latestVersion')
        targetTag = latestVersion ? `v${latestVersion}` : 'latest'
      }
      const requestData = {
        requested_at: new Date().toISOString(),
-        requester: options?.requester ?? 'admin-api',
+        requester: 'admin-api',
        target_tag: targetTag,
      }
      await writeFile(SystemUpdateService.REQUEST_FILE, JSON.stringify(requestData, null, 2))
-      logger.info(`[SystemUpdateService]: System update requested (target tag: ${requestData.target_tag}) - sidecar will process shortly`)
+      logger.info('[SystemUpdateService]: System update requested - sidecar will process shortly')
      return {
        success: true,
        message: 'System update initiated. The admin container will restart during the process.',
      }
    } catch (error) {
-      logger.error({ err: error }, '[SystemUpdateService] Failed to request system update')
+      logger.error('[SystemUpdateService]: Failed to request system update:', error)
      return {
        success: false,
-        message: 'Failed to request system update. Check server logs for details.',
+        message: `Failed to request update: ${error.message}`,
      }
    }
  }
--- a/Show More
+++ b/Show More