import {
  ListRemoteZimFilesResponse,
  RawRemoteZimFileEntry,
  RemoteZimFileEntry,
} from '../../types/zim.js'
import axios from 'axios'
import * as cheerio from 'cheerio'
import { XMLParser } from 'fast-xml-parser'
import { isRawListRemoteZimFilesResponse, isRawRemoteZimFileEntry } from '../../util/zim.js'
import { findReplacedWikipediaFiles } from '../utils/zim_filename.js'
import { decideSupersededDeletion } from '../utils/superseded_resource.js'
import logger from '@adonisjs/core/services/logger'
import { DockerService } from './docker_service.js'
import { inject } from '@adonisjs/core'
import {
  deleteFileIfExists,
  ensureDirectoryExists,
  getFileStatsIfExists,
  listDirectoryContents,
  ZIM_STORAGE_PATH,
} from '../utils/fs.js'
import { join, resolve, sep } from 'path'
import { WikipediaOption, WikipediaState } from '../../types/downloads.js'
import vine from '@vinejs/vine'
import { wikipediaOptionsFileSchema } from '#validators/curated_collections'
import WikipediaSelection from '#models/wikipedia_selection'
import InstalledResource from '#models/installed_resource'
import CollectionManifest from '#models/collection_manifest'
import { RunDownloadJob } from '#jobs/run_download_job'
import { SERVICE_NAMES } from '../../constants/service_names.js'
import { CollectionManifestService } from './collection_manifest_service.js'
import { KiwixLibraryService } from './kiwix_library_service.js'
import type { CategoryWithStatus } from '../../types/collections.js'
import CustomLibrarySource from '#models/custom_library_source'
import { assertNotPrivateUrl } from '#validators/common'

const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
const WIKIPEDIA_OPTIONS_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/wikipedia.json'

@inject()
export class ZimService {
  constructor(private dockerService: DockerService) { }

  async list() {
    const dirPath = join(process.cwd(), ZIM_STORAGE_PATH)
    await ensureDirectoryExists(dirPath)

    const all = await listDirectoryContents(dirPath)
    const zimEntries = all.filter((item) => item.name.endsWith('.zim'))

    const files = await Promise.all(
      zimEntries.map(async (entry) => {
        const filePath = entry.type === 'file' ? entry.key : join(dirPath, entry.name)
        const stats = await getFileStatsIfExists(filePath)
        return {
          ...entry,
          title: null,
          summary: null,
          author: null,
          size_bytes: stats ? Number(stats.size) : null,
        }
      })
    )

    return {
      files,
    }
  }

  async listRemote({
    start,
    count,
    query,
  }: {
    start: number
    count: number
    query?: string
  }): Promise<ListRemoteZimFilesResponse> {
    const LIBRARY_BASE_URL = 'https://browse.library.kiwix.org/catalog/v2/entries'
    // Kiwix returns pages of content unaware of what the user has installed locally. When
    // the installed set is large, a single 12-item Kiwix page can come back with everything
    // already installed → 0 post-filter items → frontend deadlock (#731). Accumulate across
    // upstream pages so we return a useful batch. Bounded by MAX_KIWIX_FETCHES so a heavily
    // saturated install doesn't hang a single request; the frontend scroll loop + auto-fetch
    // effect handle continuation.
    const KIWIX_PAGE_SIZE = 60
    const MAX_KIWIX_FETCHES = 5

    const parser = new XMLParser({
      ignoreAttributes: false,
      attributeNamePrefix: '',
      textNodeName: '#text',
    })

    // Snapshot locally-installed files once — the filesystem won't change mid-request.
    const existing = await this.list()
    const existingKeys = new Set(existing.files.map((file) => file.name))

    const accumulated: RemoteZimFileEntry[] = []
    const seenIds = new Set<string>()
    let currentStart = start
    let totalResults = 0

    for (let i = 0; i < MAX_KIWIX_FETCHES; i++) {
      const res = await axios.get(LIBRARY_BASE_URL, {
        params: {
          start: currentStart,
          count: KIWIX_PAGE_SIZE,
          lang: 'eng',
          ...(query ? { q: query } : {}),
        },
        responseType: 'text',
      })

      const parsed = parser.parse(res.data)
      if (!isRawListRemoteZimFilesResponse(parsed)) {
        throw new Error('Invalid response format from remote library')
      }
      totalResults = parsed.feed.totalResults

      const rawEntries = parsed.feed.entry
        ? Array.isArray(parsed.feed.entry)
          ? parsed.feed.entry
          : [parsed.feed.entry]
        : []

      // Empty upstream response — bail even if totalResults suggests more (transient Kiwix
      // hiccup or totalResults drift between pages). Prevents a pointless spin.
      if (rawEntries.length === 0) break

      // Advance by actual returned count, not requested count. Short pages at the tail
      // would otherwise cause us to skip entries on the next fetch.
      currentStart += rawEntries.length

      for (const raw of rawEntries) {
        if (!isRawRemoteZimFileEntry(raw)) continue
        const entry = raw as RawRemoteZimFileEntry

        const downloadLink = entry.link.find(
          (link: any) =>
            typeof link === 'object' &&
            'rel' in link &&
            'length' in link &&
            'href' in link &&
            'type' in link &&
            link.type === 'application/x-zim'
        )
        if (!downloadLink) continue

        // downloadLink['href'] ends with .meta4; strip that to get the actual .zim URL.
        const download_url = downloadLink['href'].substring(0, downloadLink['href'].length - 6)
        const file_name = download_url.split('/').pop() || `${entry.title}.zim`
        if (existingKeys.has(file_name)) continue
        if (seenIds.has(entry.id)) continue
        seenIds.add(entry.id)

        const sizeBytes = parseInt(downloadLink['length'], 10)
        accumulated.push({
          id: entry.id,
          title: entry.title,
          updated: entry.updated,
          summary: entry.summary,
          size_bytes: sizeBytes || 0,
          download_url,
          author: entry.author.name,
          file_name,
        })
      }

      if (accumulated.length >= count) break
      if (currentStart >= totalResults) break
    }

    return {
      items: accumulated,
      has_more: currentStart < totalResults,
      total_count: totalResults,
      next_start: currentStart,
    }
  }

  async downloadRemote(url: string, metadata?: { title?: string; summary?: string; author?: string; size_bytes?: number }): Promise<{ filename: string; jobId?: string }> {
    const parsed = new URL(url)
    if (!parsed.pathname.endsWith('.zim')) {
      throw new Error(`Invalid ZIM file URL: ${url}. URL must end with .zim`)
    }

    const existing = await RunDownloadJob.getActiveByUrl(url)
    if (existing) {
      throw new Error('A download for this URL is already in progress')
    }

    // Extract the filename from the URL
    const filename = url.split('/').pop()
    if (!filename) {
      throw new Error('Could not determine filename from URL')
    }

    const filepath = join(process.cwd(), ZIM_STORAGE_PATH, filename)

    // Parse resource metadata for the download job
    const parsedFilename = CollectionManifestService.parseZimFilename(filename)
    const resourceMetadata = parsedFilename
      ? { resource_id: parsedFilename.resource_id, version: parsedFilename.version, collection_ref: null }
      : undefined

    // Dispatch a background download job
    const result = await RunDownloadJob.dispatch({
      url,
      filepath,
      timeout: 30000,
      allowedMimeTypes: ZIM_MIME_TYPES,
      forceNew: true,
      filetype: 'zim',
      title: metadata?.title,
      totalBytes: metadata?.size_bytes,
      resourceMetadata,
    })

    if (!result || !result.job) {
      throw new Error('Failed to dispatch download job')
    }

    logger.info(`[ZimService] Dispatched background download job for ZIM file: ${filename}`)

    return {
      filename,
      jobId: result.job.id,
    }
  }

  async listCuratedCategories(): Promise<CategoryWithStatus[]> {
    const manifestService = new CollectionManifestService()
    return manifestService.getCategoriesWithStatus()
  }

  async downloadCategoryTier(categorySlug: string, tierSlug: string): Promise<string[] | null> {
    const manifestService = new CollectionManifestService()
    const spec = await manifestService.getSpecWithFallback<import('../../types/collections.js').ZimCategoriesSpec>('zim_categories')
    if (!spec) {
      throw new Error('Could not load ZIM categories spec')
    }

    const category = spec.categories.find((c) => c.slug === categorySlug)
    if (!category) {
      throw new Error(`Category not found: ${categorySlug}`)
    }

    const tier = category.tiers.find((t) => t.slug === tierSlug)
    if (!tier) {
      throw new Error(`Tier not found: ${tierSlug}`)
    }

    const allResources = CollectionManifestService.resolveTierResources(tier, category.tiers)

    // Filter out already installed
    const installed = await InstalledResource.query().where('resource_type', 'zim')
    const installedIds = new Set(installed.map((r) => r.resource_id))
    const toDownload = allResources.filter((r) => !installedIds.has(r.id))

    if (toDownload.length === 0) return null

    const downloadFilenames: string[] = []

    for (const resource of toDownload) {
      const existingJob = await RunDownloadJob.getActiveByUrl(resource.url)
      if (existingJob) {
        logger.warn(`[ZimService] Download already in progress for ${resource.url}, skipping.`)
        continue
      }

      const filename = resource.url.split('/').pop()
      if (!filename) continue

      downloadFilenames.push(filename)
      const filepath = join(process.cwd(), ZIM_STORAGE_PATH, filename)

      await RunDownloadJob.dispatch({
        url: resource.url,
        filepath,
        timeout: 30000,
        allowedMimeTypes: ZIM_MIME_TYPES,
        forceNew: true,
        filetype: 'zim',
        title: (resource as any).title || undefined,
        totalBytes: (resource as any).size_mb ? (resource as any).size_mb * 1024 * 1024 : undefined,
        resourceMetadata: {
          resource_id: resource.id,
          version: resource.version,
          collection_ref: categorySlug,
        },
      })
    }

    return downloadFilenames.length > 0 ? downloadFilenames : null
  }

  async downloadRemoteSuccessCallback(urls: string[], restart = true) {
    // Check if any URL is a Wikipedia download and handle it
    for (const url of urls) {
      if (url.includes('wikipedia_en_')) {
        await this.onWikipediaDownloadComplete(url, true)
      }
    }
    
    // Update the kiwix library XML after all downloaded ZIM files are in place.
    // This covers all ZIM types including Wikipedia. Rebuilding once from disk
    // avoids repeated XML parse/write cycles and reduces the chance of write races
    // when multiple download jobs complete concurrently.
    const kiwixLibraryService = new KiwixLibraryService()
    try {
      await kiwixLibraryService.rebuildFromDisk()
    } catch (err) {
      logger.error('[ZimService] Failed to rebuild kiwix library from disk:', err)
    }

    if (restart) {
      // Check if there are any remaining ZIM download jobs before restarting
      const { QueueService } = await import('./queue_service.js')
      const queueService = QueueService.getInstance()
      const queue = queueService.getQueue('downloads')

      // Get all active and waiting jobs
      const [activeJobs, waitingJobs] = await Promise.all([
        queue.getActive(),
        queue.getWaiting(),
      ])

      // Filter out completed jobs (progress === 100) to avoid race condition
      // where this job itself is still in the active queue
      const activeIncompleteJobs = activeJobs.filter((job) => {
        const progress = typeof job.progress === 'object' && job.progress !== null
          ? (job.progress as any).percent
          : typeof job.progress === 'number' ? job.progress : 0
        return progress < 100
      })

      // Check if any remaining incomplete jobs are ZIM downloads
      const allJobs = [...activeIncompleteJobs, ...waitingJobs]
      const hasRemainingZimJobs = allJobs.some((job) => job.data.filetype === 'zim')

      if (hasRemainingZimJobs) {
        logger.info('[ZimService] Skipping container restart - more ZIM downloads pending')
      } else {
        // If kiwix is already running in library mode, --monitorLibrary will pick up
        // the XML change automatically — no restart needed.
        const isLegacy = await this.dockerService.isKiwixOnLegacyConfig()
        if (!isLegacy) {
          logger.info('[ZimService] Kiwix is in library mode — XML updated, no container restart needed.')
        } else {
          // Legacy config: restart (affectContainer will trigger migration instead)
          logger.info('[ZimService] No more ZIM downloads pending - restarting KIWIX container')
          await this.dockerService
            .affectContainer(SERVICE_NAMES.KIWIX, 'restart')
            .catch((error) => {
              logger.error(`[ZimService] Failed to restart KIWIX container:`, error)
            })
        }
      }
    }

    // Create InstalledResource entries for downloaded files
    const zimStorageDir = join(process.cwd(), ZIM_STORAGE_PATH)
    let removedSupersededZim = false
    for (const url of urls) {
      // Skip Wikipedia files (managed separately)
      if (url.includes('wikipedia_en_')) continue

      const filename = url.split('/').pop()
      if (!filename) continue

      const parsed = CollectionManifestService.parseZimFilename(filename)
      if (!parsed) continue

      const filepath = join(zimStorageDir, filename)
      const stats = await getFileStatsIfExists(filepath)

      try {
        // Capture the prior install for this resource_id BEFORE updateOrCreate
        // overwrites it, so we know the old file path to clean up (#634).
        const prior = await InstalledResource.query()
          .where('resource_id', parsed.resource_id)
          .where('resource_type', 'zim')
          .first()

        const { DateTime } = await import('luxon')
        await InstalledResource.updateOrCreate(
          { resource_id: parsed.resource_id, resource_type: 'zim' },
          {
            version: parsed.version,
            url: url,
            file_path: filepath,
            file_size_bytes: stats ? Number(stats.size) : null,
            installed_at: DateTime.now(),
          }
        )
        logger.info(`[ZimService] Created InstalledResource entry for: ${parsed.resource_id}`)

        // Remove the superseded prior version's file if (and only if) every
        // safety rail passes — see decideSupersededDeletion. The InstalledResource
        // row already points at the new file, so we delete the old file directly
        // (NOT via this.delete(), which would drop the row by resource_id).
        const decision = decideSupersededDeletion({
          existing: prior ? { file_path: prior.file_path, version: prior.version } : null,
          newFilePath: filepath,
          newVersion: parsed.version,
          newFileExists: !!stats,
          storageBaseDir: zimStorageDir,
        })
        if (decision.delete && decision.path) {
          try {
            await deleteFileIfExists(decision.path)
            removedSupersededZim = true
            logger.info(
              `[ZimService] Removed superseded ${parsed.resource_id} file: ${decision.path}`
            )
          } catch (err) {
            logger.warn(`[ZimService] Failed to remove superseded file ${decision.path}:`, err)
          }
        } else if (decision.reason !== 'first_install' && decision.reason !== 'same_file') {
          logger.info(
            `[ZimService] Kept prior ${parsed.resource_id} file (reason: ${decision.reason})`
          )
        }
      } catch (error) {
        logger.error(`[ZimService] Failed to create InstalledResource for ${filename}:`, error)
      }
    }

    // If we removed any superseded ZIM, rebuild the Kiwix library so its XML no
    // longer references the deleted file. The earlier rebuild in this flow ran
    // while both versions were still on disk.
    if (removedSupersededZim) {
      try {
        await new KiwixLibraryService().rebuildFromDisk()
        logger.info('[ZimService] Rebuilt Kiwix library after removing superseded ZIM(s).')
      } catch (err) {
        logger.error('[ZimService] Failed to rebuild Kiwix library after cleanup:', err)
      }
    }
  }

  /**
   * Rebuilds the kiwix library XML from whatever ZIM files are currently on disk.
   *
   * This is the manual counterpart to the automatic rebuilds that run after a
   * download or delete. It exists for the sideload case: a user copies a .zim file
   * onto the box (USB, SSH, network share) outside the download flow, and kiwix has
   * no way to discover it without regenerating the library index.
   *
   * In library mode (--monitorLibrary) kiwix-serve hot-reloads the XML on its own, so
   * no restart is needed. Only legacy glob-mode containers are restarted to pick up
   * the change. Returns the book count before and after plus the number added.
   */
  async rescanLibrary(): Promise<{ before: number; after: number; added: number }> {
    const kiwixLibraryService = new KiwixLibraryService()
    const before = await kiwixLibraryService.getBookCount()
    const after = await kiwixLibraryService.rebuildFromDisk()

    const isLegacy = await this.dockerService.isKiwixOnLegacyConfig()
    if (isLegacy) {
      logger.info('[ZimService] Kiwix in legacy mode — restarting container after rescan.')
      await this.dockerService
        .affectContainer(SERVICE_NAMES.KIWIX, 'restart')
        .catch((error) => {
          logger.error('[ZimService] Failed to restart KIWIX container after rescan:', error)
        })
    }

    return { before, after, added: Math.max(0, after - before) }
  }

  async registerLocalUpload(filename: string): Promise<{ added: number }> {
    let added = 0
    try {
      const result = await this.rescanLibrary()
      added = result.added
    } catch (err) {
      logger.error('[ZimService] Failed to rebuild kiwix library after local upload:', err)
    }

    const parsed = CollectionManifestService.parseZimFilename(filename)
    if (parsed) {
      const filepath = join(process.cwd(), ZIM_STORAGE_PATH, filename)
      const stats = await getFileStatsIfExists(filepath)
      try {
        const { DateTime } = await import('luxon')
        await InstalledResource.updateOrCreate(
          { resource_id: parsed.resource_id, resource_type: 'zim' },
          {
            version: parsed.version,
            url: `local-upload://${filename}`,
            file_path: filepath,
            file_size_bytes: stats ? Number(stats.size) : null,
            installed_at: DateTime.now(),
          }
        )
      } catch (error) {
        logger.error(`[ZimService] Failed to create InstalledResource for ${filename}:`, error)
      }
    }

    // If the uploaded file matches a known Wikipedia option, mark it as installed
    try {
      const manifest = await CollectionManifest.find('wikipedia')
      if (manifest) {
        const spec = manifest.spec_data as { options: Array<{ id: string; url: string | null }> }
        const matchedOption = spec.options.find(
          (opt) => opt.url && opt.url.split('/').pop() === filename
        )
        if (matchedOption && matchedOption.url) {
          const existing = await WikipediaSelection.query().first()
          if (existing) {
            existing.option_id = matchedOption.id
            existing.url = matchedOption.url
            existing.filename = filename
            existing.status = 'installed'
            await existing.save()
          } else {
            await WikipediaSelection.create({
              option_id: matchedOption.id,
              url: matchedOption.url,
              filename,
              status: 'installed',
            })
          }
          logger.info(`[ZimService] Marked Wikipedia option '${matchedOption.id}' as installed from local upload`)

          // Remove any other wikipedia_en_*.zim files, same as the download flow
          const allFiles = await this.list()
          const staleWikipediaFiles = allFiles.files.filter(
            (f) => f.name.startsWith('wikipedia_en_') && f.name !== filename
          )
          for (const stale of staleWikipediaFiles) {
            try {
              await this.delete(stale.name)
              logger.info(`[ZimService] Deleted stale Wikipedia file after upload: ${stale.name}`)
            } catch (err) {
              logger.warn(`[ZimService] Could not delete stale Wikipedia file: ${stale.name}`, err)
            }
          }
        }
      }
    } catch (error) {
      logger.error(`[ZimService] Failed to update WikipediaSelection for ${filename}:`, error)
    }

    const ollamaUrl = await this.dockerService.getServiceURL('nomad_ollama')
    if (ollamaUrl) {
      try {
        const { EmbedFileJob } = await import('#jobs/embed_file_job')
        await EmbedFileJob.dispatch({
          fileName: filename,
          filePath: join(process.cwd(), ZIM_STORAGE_PATH, filename),
        })
      } catch (error) {
        logger.error(`[ZimService] EmbedFileJob dispatch failed after local upload:`, error)
      }
    }

    return { added }
  }

  async delete(file: string): Promise<void> {
    let fileName = file
    if (!fileName.endsWith('.zim')) {
      fileName += '.zim'
    }

    const basePath = resolve(join(process.cwd(), ZIM_STORAGE_PATH))
    const fullPath = resolve(join(basePath, fileName))

    // Prevent path traversal — resolved path must stay within the storage directory
    if (!fullPath.startsWith(basePath + sep)) {
      throw new Error('Invalid filename')
    }

    const exists = await getFileStatsIfExists(fullPath)
    if (!exists) {
      throw new Error('not_found')
    }

    await deleteFileIfExists(fullPath)

    // Remove from kiwix library XML so --monitorLibrary stops serving the deleted file
    const kiwixLibraryService = new KiwixLibraryService()
    await kiwixLibraryService.removeBook(fileName).catch((err) => {
      logger.error(`[ZimService] Failed to remove ${fileName} from kiwix library:`, err)
    })

    // Clean up InstalledResource entry
    const parsed = CollectionManifestService.parseZimFilename(fileName)
    if (parsed) {
      await InstalledResource.query()
        .where('resource_id', parsed.resource_id)
        .where('resource_type', 'zim')
        .delete()
      logger.info(`[ZimService] Deleted InstalledResource entry for: ${parsed.resource_id}`)
    }

    // If this file was the active Wikipedia selection, clear the selection
    try {
      const selection = await WikipediaSelection.query().first()
      if (selection && selection.filename === fileName) {
        selection.option_id = 'none'
        selection.status = 'none'
        selection.filename = null
        selection.url = null
        await selection.save()
        logger.info(`[ZimService] Cleared WikipediaSelection after deleting ${fileName}`)
      }
    } catch (error) {
      logger.error(`[ZimService] Failed to clear WikipediaSelection after deleting ${fileName}:`, error)
    }
  }

  // Wikipedia selector methods

  async getWikipediaOptions(): Promise<WikipediaOption[]> {
    try {
      const response = await axios.get(WIKIPEDIA_OPTIONS_URL)
      const data = response.data

      const validated = await vine.validate({
        schema: wikipediaOptionsFileSchema,
        data,
      })

      return validated.options
    } catch (error) {
      logger.error(`[ZimService] Failed to fetch Wikipedia options:`, error)
      throw new Error('Failed to fetch Wikipedia options')
    }
  }

  async getWikipediaSelection(): Promise<WikipediaSelection | null> {
    // Get the single row from wikipedia_selections (there should only ever be one)
    return WikipediaSelection.query().first()
  }

  async getWikipediaState(): Promise<WikipediaState> {
    const options = await this.getWikipediaOptions()
    const selection = await this.getWikipediaSelection()

    return {
      options,
      currentSelection: selection
        ? {
          optionId: selection.option_id,
          status: selection.status,
          filename: selection.filename,
          url: selection.url,
        }
        : null,
    }
  }

  async selectWikipedia(optionId: string): Promise<{ success: boolean; jobId?: string; message?: string }> {
    const options = await this.getWikipediaOptions()
    const selectedOption = options.find((opt) => opt.id === optionId)

    if (!selectedOption) {
      throw new Error(`Invalid Wikipedia option: ${optionId}`)
    }

    const currentSelection = await this.getWikipediaSelection()

    // If same as currently installed, no action needed
    if (currentSelection?.option_id === optionId && currentSelection.status === 'installed') {
      return { success: true, message: 'Already installed' }
    }

    // Handle "none" option - delete current Wikipedia file and update DB
    if (optionId === 'none') {
      if (currentSelection?.filename) {
        try {
          await this.delete(currentSelection.filename)
          logger.info(`[ZimService] Deleted Wikipedia file: ${currentSelection.filename}`)
        } catch (error) {
          // File might already be deleted, that's OK
          logger.warn(`[ZimService] Could not delete Wikipedia file (may already be gone): ${currentSelection.filename}`)
        }
      }

      // Update or create the selection record (always use first record)
      if (currentSelection) {
        currentSelection.option_id = 'none'
        currentSelection.url = null
        currentSelection.filename = null
        currentSelection.status = 'none'
        await currentSelection.save()
      } else {
        await WikipediaSelection.create({
          option_id: 'none',
          url: null,
          filename: null,
          status: 'none',
        })
      }

      // Restart Kiwix to reflect the change
      await this.dockerService
        .affectContainer(SERVICE_NAMES.KIWIX, 'restart')
        .catch((error) => {
          logger.error(`[ZimService] Failed to restart Kiwix after Wikipedia removal:`, error)
        })

      return { success: true, message: 'Wikipedia removed' }
    }

    // Start download for the new Wikipedia option
    if (!selectedOption.url) {
      throw new Error('Selected Wikipedia option has no download URL')
    }

    // Check if already downloading
    const existingJob = await RunDownloadJob.getActiveByUrl(selectedOption.url)
    if (existingJob) {
      return { success: false, message: 'Download already in progress' }
    }

    // Extract filename from URL
    const filename = selectedOption.url.split('/').pop()
    if (!filename) {
      throw new Error('Could not determine filename from URL')
    }

    const filepath = join(process.cwd(), ZIM_STORAGE_PATH, filename)

    // Update or create selection record to show downloading status
    let selection: WikipediaSelection
    if (currentSelection) {
      currentSelection.option_id = optionId
      currentSelection.url = selectedOption.url
      currentSelection.filename = filename
      currentSelection.status = 'downloading'
      await currentSelection.save()
      selection = currentSelection
    } else {
      selection = await WikipediaSelection.create({
        option_id: optionId,
        url: selectedOption.url,
        filename: filename,
        status: 'downloading',
      })
    }

    // Dispatch download job
    const result = await RunDownloadJob.dispatch({
      url: selectedOption.url,
      filepath,
      timeout: 30000,
      allowedMimeTypes: ZIM_MIME_TYPES,
      forceNew: true,
      filetype: 'zim',
      title: selectedOption.name,
      totalBytes: selectedOption.size_mb ? selectedOption.size_mb * 1024 * 1024 : undefined,
    })

    if (!result || !result.job) {
      // Revert status on failure to dispatch
      selection.option_id = currentSelection?.option_id || 'none'
      selection.url = currentSelection?.url || null
      selection.filename = currentSelection?.filename || null
      selection.status = currentSelection?.status || 'none'
      await selection.save()
      throw new Error('Failed to dispatch download job')
    }

    logger.info(`[ZimService] Started Wikipedia download for ${optionId}: ${filename}`)

    return {
      success: true,
      jobId: result.job.id,
      message: 'Download started',
    }
  }

  async onWikipediaDownloadComplete(url: string, success: boolean): Promise<void> {
    const filename = url.split('/').pop() || ''
    const selection = await this.getWikipediaSelection()

    // Determine which Wikipedia option this file belongs to by matching filename
    let matchedOptionId: string | null = null
    try {
      const options = await this.getWikipediaOptions()
      for (const opt of options) {
        if (opt.url && opt.url.split('/').pop() === filename) {
          matchedOptionId = opt.id
          break
        }
      }
    } catch {
      // If we can't fetch options, try to continue with existing selection
    }

    if (success) {
      // Update or create the selection record
      // Match by filename (not URL) so mirror downloads are recognized
      if (selection) {
        selection.option_id = matchedOptionId || selection.option_id
        selection.url = url
        selection.filename = filename
        selection.status = 'installed'
        await selection.save()
      } else {
        await WikipediaSelection.create({
          option_id: matchedOptionId || 'unknown',
          url: url,
          filename: filename,
          status: 'installed',
        })
      }

      logger.info(`[ZimService] Wikipedia download completed successfully: ${filename}`)

      // Delete prior versions of THIS specific Wikipedia variant only.
      // Earlier logic deleted anything starting with `wikipedia_en_`, which silently
      // wiped distinct corpora the user had installed independently (issue #884).
      const existingFiles = await this.list()
      const wikipediaFiles = findReplacedWikipediaFiles(
        filename,
        existingFiles.files.map((f) => f.name)
      )

      for (const oldFile of wikipediaFiles) {
        try {
          await this.delete(oldFile)
          logger.info(`[ZimService] Deleted old Wikipedia file: ${oldFile}`)
        } catch (error) {
          logger.warn(`[ZimService] Could not delete old Wikipedia file: ${oldFile}`, error)
        }
      }
    } else {
      // Download failed - update selection if it matches this file
      if (selection && (!selection.filename || selection.filename === filename)) {
        selection.status = 'failed'
        await selection.save()
        logger.error(`[ZimService] Wikipedia download failed for: ${filename}`)
      } else {
        logger.error(`[ZimService] Wikipedia download failed for: ${filename} (no matching selection)`)
      }
    }
  }

  // Custom library source management

  async listCustomLibraries(): Promise<CustomLibrarySource[]> {
    return CustomLibrarySource.all()
  }

  async addCustomLibrary(name: string, baseUrl: string): Promise<CustomLibrarySource> {
    const count = await CustomLibrarySource.query().count('* as total')
    const total = Number(count[0].$extras.total)
    if (total >= 10) {
      throw new Error('Maximum of 10 custom libraries allowed')
    }

    // Ensure URL ends with /
    const normalizedUrl = baseUrl.endsWith('/') ? baseUrl : baseUrl + '/'

    return CustomLibrarySource.create({
      name,
      base_url: normalizedUrl,
    })
  }

  async removeCustomLibrary(id: number): Promise<void> {
    const source = await CustomLibrarySource.find(id)
    if (!source) {
      throw new Error('Custom library not found')
    }
    if (source.is_default) {
      throw new Error('Cannot remove a built-in mirror')
    }
    await source.delete()
  }

  async browseLibraryUrl(url: string): Promise<{
    directories: { name: string; url: string }[]
    files: { name: string; url: string; size_bytes: number | null }[]
  }> {
    assertNotPrivateUrl(url)

    const normalizedUrl = url.endsWith('/') ? url : url + '/'

    const res = await axios.get(normalizedUrl, {
      responseType: 'text',
      timeout: 15000,
      headers: {
        'Accept': 'text/html',
      },
    })

    const html: string = res.data
    const directories: { name: string; url: string }[] = []
    const files: { name: string; url: string; size_bytes: number | null }[] = []

    const $ = cheerio.load(html)

    $('a').each((_, el) => {
      const href = el.attribs?.href
      if (!href || href === '../' || href === './' || href === '/' || href.startsWith('?') || href.startsWith('#')) {
        return
      }
      if (href.startsWith('/') || href.startsWith('http://') || href.startsWith('https://')) {
        return
      }

      if (href.endsWith('/')) {
        const dirName = decodeURIComponent(href.replace(/\/$/, ''))
        directories.push({
          name: dirName,
          url: new URL(href, normalizedUrl).toString(),
        })
        return
      }

      if (href.endsWith('.zim')) {
        const fileName = decodeURIComponent(href)

        // Apache/Nginx autoindex put the date + size in the text node directly
        // following </a> within a <pre>. Walk forward across text siblings until
        // we find a parseable size token.
        let trailingText = ''
        let sibling = el.next
        while (sibling && sibling.type === 'text') {
          trailingText += sibling.data
          if (/\n/.test(sibling.data)) break
          sibling = sibling.next
        }

        files.push({
          name: fileName,
          url: new URL(href, normalizedUrl).toString(),
          size_bytes: this._parseListingSize(trailingText),
        })
      }
    })

    directories.sort((a, b) => a.name.localeCompare(b.name))
    files.sort((a, b) => a.name.localeCompare(b.name))

    return { directories, files }
  }

  /**
   * Parse a directory-listing size token out of the text that follows an anchor.
   * Apache renders e.g. `   2024-01-15 10:30  5.1G`; Nginx renders raw bytes.
   * Returns bytes or null if no size token is found.
   */
  private _parseListingSize(text: string): number | null {
    // Skip the date/time columns; grab the last numeric token (with optional suffix)
    // before a newline. Matches `5.1G`, `5368709120`, `1.2T`, etc.
    const sizeMatch = /([\d.]+\s*[KMGT]?B?|\d+)\s*$/i.exec(text.split('\n')[0].trim())
    if (!sizeMatch) return null

    const sizeStr = sizeMatch[1].replace(/\s|B$/gi, '')
    const num = parseFloat(sizeStr)
    if (isNaN(num)) return null

    if (/^\d+$/.test(sizeStr)) return num

    const suffix = sizeStr.slice(-1).toUpperCase()
    const multipliers: Record<string, number> = { K: 1024, M: 1024 ** 2, G: 1024 ** 3, T: 1024 ** 4 }
    return multipliers[suffix] ? Math.round(num * multipliers[suffix]) : null
  }
}