1902 lines
75 KiB
TypeScript
1902 lines
75 KiB
TypeScript
import Service from '#models/service'
|
|
import Docker from 'dockerode'
|
|
import logger from '@adonisjs/core/services/logger'
|
|
import { inject } from '@adonisjs/core'
|
|
import transmit from '@adonisjs/transmit/services/main'
|
|
import { doResumableDownloadWithRetry } from '../utils/downloads.js'
|
|
import { join } from 'path'
|
|
import {
|
|
ZIM_STORAGE_PATH,
|
|
BOOKS_STORAGE_PATH,
|
|
CALIBRE_EMPTY_LIBRARY_ASSET_PATH,
|
|
VAULTWARDEN_STORAGE_PATH,
|
|
MEDIA_STORAGE_PATH,
|
|
JELLYFIN_MEDIA_SUBFOLDERS,
|
|
} from '../utils/fs.js'
|
|
import { KiwixLibraryService } from './kiwix_library_service.js'
|
|
import { SERVICE_NAMES } from '../../constants/service_names.js'
|
|
import { exec } from 'child_process'
|
|
import { promisify } from 'util'
|
|
import { readFile, mkdir, copyFile, chown, chmod, access } from 'node:fs/promises'
|
|
import KVStore from '#models/kv_store'
|
|
import { BROADCAST_CHANNELS } from '../../constants/broadcast.js'
|
|
import { KIWIX_LIBRARY_CMD } from '../../constants/kiwix.js'
|
|
|
|
@inject()
|
|
export class DockerService {
|
|
public docker: Docker
|
|
private activeInstallations: Set<string> = new Set()
|
|
public static NOMAD_NETWORK = 'project-nomad_default'
|
|
|
|
private _servicesStatusCache: { data: { service_name: string; status: string }[]; expiresAt: number } | null = null
|
|
private _servicesStatusInflight: Promise<{ service_name: string; status: string }[]> | null = null
|
|
|
|
constructor() {
|
|
// Support both Linux (production) and Windows (development with Docker Desktop)
|
|
const isWindows = process.platform === 'win32'
|
|
if (isWindows) {
|
|
// Windows Docker Desktop uses named pipe
|
|
this.docker = new Docker({ socketPath: '//./pipe/docker_engine' })
|
|
} else {
|
|
// Linux uses Unix socket
|
|
this.docker = new Docker({ socketPath: '/var/run/docker.sock' })
|
|
}
|
|
}
|
|
|
|
async affectContainer(
|
|
serviceName: string,
|
|
action: 'start' | 'stop' | 'restart'
|
|
): Promise<{ success: boolean; message: string }> {
|
|
try {
|
|
const service = await Service.query().where('service_name', serviceName).first()
|
|
if (!service || !service.installed) {
|
|
return {
|
|
success: false,
|
|
message: `Service ${serviceName} not found or not installed`,
|
|
}
|
|
}
|
|
|
|
const containers = await this.docker.listContainers({ all: true })
|
|
const container = containers.find((c) => c.Names.includes(`/${serviceName}`))
|
|
if (!container) {
|
|
return {
|
|
success: false,
|
|
message: `Container for service ${serviceName} not found`,
|
|
}
|
|
}
|
|
|
|
const dockerContainer = this.docker.getContainer(container.Id)
|
|
if (action === 'stop') {
|
|
await dockerContainer.stop()
|
|
this.invalidateServicesStatusCache()
|
|
return {
|
|
success: true,
|
|
message: `Service ${serviceName} stopped successfully`,
|
|
}
|
|
}
|
|
|
|
if (action === 'restart') {
|
|
if (serviceName === SERVICE_NAMES.KIWIX) {
|
|
const isLegacy = await this.isKiwixOnLegacyConfig()
|
|
if (isLegacy) {
|
|
logger.info('[DockerService] Kiwix on legacy glob config — running migration instead of restart.')
|
|
await this.migrateKiwixToLibraryMode()
|
|
this.invalidateServicesStatusCache()
|
|
return { success: true, message: 'Kiwix migrated to library mode successfully.' }
|
|
}
|
|
}
|
|
|
|
await dockerContainer.restart()
|
|
this.invalidateServicesStatusCache()
|
|
|
|
return {
|
|
success: true,
|
|
message: `Service ${serviceName} restarted successfully`,
|
|
}
|
|
}
|
|
|
|
if (action === 'start') {
|
|
if (container.State === 'running') {
|
|
return {
|
|
success: true,
|
|
message: `Service ${serviceName} is already running`,
|
|
}
|
|
}
|
|
|
|
await dockerContainer.start()
|
|
this.invalidateServicesStatusCache()
|
|
|
|
return {
|
|
success: true,
|
|
message: `Service ${serviceName} started successfully`,
|
|
}
|
|
}
|
|
|
|
return {
|
|
success: false,
|
|
message: `Invalid action: ${action}. Use 'start', 'stop', or 'restart'.`,
|
|
}
|
|
} catch (error: any) {
|
|
logger.error({ err: error }, `[DockerService] Error controlling service ${serviceName}`)
|
|
return {
|
|
success: false,
|
|
message: `Failed to ${action} service ${serviceName}. Check server logs for details.`,
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Fetches the status of all Docker containers related to Nomad services. (those prefixed with 'nomad_')
|
|
* Results are cached for 5 seconds and concurrent callers share a single in-flight request,
|
|
* preventing Docker socket congestion during rapid page navigation.
|
|
*/
|
|
async getServicesStatus(): Promise<{ service_name: string; status: string }[]> {
|
|
const now = Date.now()
|
|
if (this._servicesStatusCache && now < this._servicesStatusCache.expiresAt) {
|
|
return this._servicesStatusCache.data
|
|
}
|
|
if (this._servicesStatusInflight) return this._servicesStatusInflight
|
|
|
|
this._servicesStatusInflight = this._fetchServicesStatus().then((data) => {
|
|
this._servicesStatusCache = { data, expiresAt: Date.now() + 5000 }
|
|
this._servicesStatusInflight = null
|
|
return data
|
|
}).catch((err) => {
|
|
this._servicesStatusInflight = null
|
|
throw err
|
|
})
|
|
return this._servicesStatusInflight
|
|
}
|
|
|
|
/**
|
|
* Invalidates the services status cache. Call this after any container state change
|
|
* (start, stop, restart, install, uninstall) so the next read reflects reality.
|
|
*/
|
|
invalidateServicesStatusCache() {
|
|
this._servicesStatusCache = null
|
|
this._servicesStatusInflight = null
|
|
}
|
|
|
|
private async _fetchServicesStatus(): Promise<{ service_name: string; status: string }[]> {
|
|
try {
|
|
const containers = await this.docker.listContainers({ all: true })
|
|
const containerMap = new Map<string, Docker.ContainerInfo>()
|
|
containers.forEach((container) => {
|
|
const name = container.Names[0]?.replace('/', '')
|
|
if (name && name.startsWith('nomad_')) {
|
|
containerMap.set(name, container)
|
|
}
|
|
})
|
|
|
|
return Array.from(containerMap.entries()).map(([name, container]) => ({
|
|
service_name: name,
|
|
status: container.State,
|
|
}))
|
|
} catch (error: any) {
|
|
logger.error(`Error fetching services status: ${error.message}`)
|
|
return []
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get the URL to access a service based on its configuration.
|
|
* Attempts to return a docker-internal URL using the service name and exposed port.
|
|
* @param serviceName - The name of the service to get the URL for.
|
|
* @returns - The URL as a string, or null if it cannot be determined.
|
|
*/
|
|
async getServiceURL(serviceName: string): Promise<string | null> {
|
|
if (!serviceName || serviceName.trim() === '') {
|
|
return null
|
|
}
|
|
|
|
if (serviceName === SERVICE_NAMES.OLLAMA) {
|
|
const remoteUrl = await KVStore.getValue('ai.remoteOllamaUrl')
|
|
if (remoteUrl) return remoteUrl
|
|
}
|
|
|
|
const service = await Service.query()
|
|
.where('service_name', serviceName)
|
|
.andWhere('installed', true)
|
|
.first()
|
|
|
|
if (!service) {
|
|
return null
|
|
}
|
|
|
|
const hostname = process.env.NODE_ENV === 'production' ? serviceName : 'localhost'
|
|
|
|
// "https:8480" / "http:8480" — explicit scheme + port (e.g. an app serving its own TLS).
|
|
const schemePort = service.ui_location?.match(/^(https?):(\d+)$/)
|
|
if (schemePort) {
|
|
return `${schemePort[1]}://${hostname}:${schemePort[2]}`
|
|
}
|
|
|
|
// First, check if ui_location is set and is a valid port number
|
|
if (service.ui_location && parseInt(service.ui_location, 10)) {
|
|
return `http://${hostname}:${service.ui_location}`
|
|
}
|
|
|
|
// Next, try to extract a host port from container_config
|
|
const parsedConfig = this._parseContainerConfig(service.container_config)
|
|
if (parsedConfig?.HostConfig?.PortBindings) {
|
|
const portBindings = parsedConfig.HostConfig.PortBindings
|
|
const hostPorts = Object.values(portBindings)
|
|
if (!hostPorts || !Array.isArray(hostPorts) || hostPorts.length === 0) {
|
|
return null
|
|
}
|
|
|
|
const hostPortsArray = hostPorts.flat() as { HostPort: string }[]
|
|
const hostPortsStrings = hostPortsArray.map((binding) => binding.HostPort)
|
|
if (hostPortsStrings.length > 0) {
|
|
return `http://${hostname}:${hostPortsStrings[0]}`
|
|
}
|
|
}
|
|
|
|
// Otherwise, return null if we can't determine a URL
|
|
return null
|
|
}
|
|
|
|
async createContainerPreflight(
|
|
serviceName: string
|
|
): Promise<{ success: boolean; message: string }> {
|
|
const service = await Service.query().where('service_name', serviceName).first()
|
|
if (!service) {
|
|
return {
|
|
success: false,
|
|
message: `Service ${serviceName} not found`,
|
|
}
|
|
}
|
|
|
|
if (service.installed) {
|
|
return {
|
|
success: false,
|
|
message: `Service ${serviceName} is already installed`,
|
|
}
|
|
}
|
|
|
|
// Check if installation is already in progress (database-level)
|
|
if (service.installation_status === 'installing') {
|
|
return {
|
|
success: false,
|
|
message: `Service ${serviceName} installation is already in progress`,
|
|
}
|
|
}
|
|
|
|
// Double-check with in-memory tracking (race condition protection)
|
|
if (this.activeInstallations.has(serviceName)) {
|
|
return {
|
|
success: false,
|
|
message: `Service ${serviceName} installation is already in progress`,
|
|
}
|
|
}
|
|
|
|
// Mark installation as in progress
|
|
this.activeInstallations.add(serviceName)
|
|
service.installation_status = 'installing'
|
|
await service.save()
|
|
|
|
// Check if a service wasn't marked as installed but has an existing container
|
|
// This can happen if the service was created but not properly installed
|
|
// or if the container was removed manually without updating the service status.
|
|
// if (await this._checkIfServiceContainerExists(serviceName)) {
|
|
// const removeResult = await this._removeServiceContainer(serviceName);
|
|
// if (!removeResult.success) {
|
|
// return {
|
|
// success: false,
|
|
// message: `Failed to remove existing container for service ${serviceName}: ${removeResult.message}`,
|
|
// };
|
|
// }
|
|
// }
|
|
|
|
const containerConfig = this._parseContainerConfig(service.container_config)
|
|
|
|
// Execute installation asynchronously and handle cleanup
|
|
this._createContainer(service, containerConfig).catch(async (error) => {
|
|
logger.error(`Installation failed for ${serviceName}: ${error.message}`)
|
|
await this._cleanupFailedInstallation(serviceName)
|
|
})
|
|
|
|
return {
|
|
success: true,
|
|
message: `Service ${serviceName} installation initiated successfully. You can receive updates via server-sent events.`,
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Force reinstall a service by stopping, removing, and recreating its container.
|
|
*
|
|
* Volume handling: removes Docker-managed named volumes whose name equals
|
|
* `serviceName`, starts with `${serviceName}_`, or carries a `service=${serviceName}`
|
|
* label. Host bind mounts are NOT touched — any data living on a bind-mounted
|
|
* host path (ZIM stores, model caches, MySQL data dir, etc.) survives the reinstall.
|
|
* Anonymous volumes (random hash names) are also not matched.
|
|
*/
|
|
async forceReinstall(serviceName: string): Promise<{ success: boolean; message: string }> {
|
|
try {
|
|
const service = await Service.query().where('service_name', serviceName).first()
|
|
if (!service) {
|
|
return {
|
|
success: false,
|
|
message: `Service ${serviceName} not found`,
|
|
}
|
|
}
|
|
|
|
// Check if installation is already in progress
|
|
if (this.activeInstallations.has(serviceName)) {
|
|
return {
|
|
success: false,
|
|
message: `Service ${serviceName} installation is already in progress`,
|
|
}
|
|
}
|
|
|
|
// Mark as installing to prevent concurrent operations
|
|
this.activeInstallations.add(serviceName)
|
|
service.installation_status = 'installing'
|
|
await service.save()
|
|
|
|
this._broadcast(
|
|
serviceName,
|
|
'reinstall-starting',
|
|
`Starting force reinstall for ${serviceName}...`
|
|
)
|
|
|
|
// Step 1: Try to stop and remove the container if it exists
|
|
try {
|
|
const containers = await this.docker.listContainers({ all: true })
|
|
const container = containers.find((c) => c.Names.includes(`/${serviceName}`))
|
|
|
|
if (container) {
|
|
const dockerContainer = this.docker.getContainer(container.Id)
|
|
|
|
// Only try to stop if it's running
|
|
if (container.State === 'running') {
|
|
this._broadcast(serviceName, 'stopping', `Stopping container...`)
|
|
await dockerContainer.stop({ t: 10 }).catch((error) => {
|
|
// If already stopped, continue
|
|
if (!error.message.includes('already stopped')) {
|
|
logger.warn(`Error stopping container: ${error.message}`)
|
|
}
|
|
})
|
|
}
|
|
|
|
// Step 2: Remove the container
|
|
this._broadcast(serviceName, 'removing', `Removing container...`)
|
|
await dockerContainer.remove({ force: true }).catch((error) => {
|
|
logger.warn(`Error removing container: ${error.message}`)
|
|
})
|
|
} else {
|
|
this._broadcast(
|
|
serviceName,
|
|
'no-container',
|
|
`No existing container found, proceeding with installation...`
|
|
)
|
|
}
|
|
} catch (error: any) {
|
|
logger.warn({ err: error }, `[DockerService] Error during container cleanup for ${serviceName}`)
|
|
this._broadcast(serviceName, 'cleanup-warning', 'Warning during container cleanup. Check server logs for details.')
|
|
}
|
|
|
|
// Step 3: Clear volumes/data if needed
|
|
try {
|
|
this._broadcast(serviceName, 'clearing-volumes', `Checking for volumes to clear...`)
|
|
const volumes = await this.docker.listVolumes()
|
|
const serviceVolumes =
|
|
volumes.Volumes?.filter(
|
|
(v) =>
|
|
v.Name === serviceName ||
|
|
v.Name.startsWith(`${serviceName}_`) ||
|
|
v.Labels?.service === serviceName
|
|
) || []
|
|
|
|
for (const vol of serviceVolumes) {
|
|
try {
|
|
const volume = this.docker.getVolume(vol.Name)
|
|
await volume.remove({ force: true })
|
|
this._broadcast(serviceName, 'volume-removed', `Removed volume: ${vol.Name}`)
|
|
} catch (error: any) {
|
|
logger.warn(`Failed to remove volume ${vol.Name}: ${error.message}`)
|
|
}
|
|
}
|
|
|
|
if (serviceVolumes.length === 0) {
|
|
this._broadcast(serviceName, 'no-volumes', `No volumes found to clear`)
|
|
}
|
|
} catch (error: any) {
|
|
logger.warn({ err: error }, `[DockerService] Error during volume cleanup for ${serviceName}`)
|
|
this._broadcast(
|
|
serviceName,
|
|
'volume-cleanup-warning',
|
|
'Warning during volume cleanup. Check server logs for details.'
|
|
)
|
|
}
|
|
|
|
// Step 4: Mark service as uninstalled
|
|
service.installed = false
|
|
service.installation_status = 'installing'
|
|
await service.save()
|
|
this.invalidateServicesStatusCache()
|
|
|
|
// Step 5: Recreate the container
|
|
this._broadcast(serviceName, 'recreating', `Recreating container...`)
|
|
const containerConfig = this._parseContainerConfig(service.container_config)
|
|
|
|
// Execute installation asynchronously and handle cleanup
|
|
this._createContainer(service, containerConfig).catch(async (error) => {
|
|
logger.error(`Reinstallation failed for ${serviceName}: ${error.message}`)
|
|
await this._cleanupFailedInstallation(serviceName)
|
|
})
|
|
|
|
return {
|
|
success: true,
|
|
message: `Service ${serviceName} force reinstall initiated successfully. You can receive updates via server-sent events.`,
|
|
}
|
|
} catch (error: any) {
|
|
logger.error({ err: error }, `[DockerService] Force reinstall failed for ${serviceName}`)
|
|
await this._cleanupFailedInstallation(serviceName)
|
|
return {
|
|
success: false,
|
|
message: `Failed to force reinstall service ${serviceName}. Check server logs for details.`,
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Handles the long-running process of creating a Docker container for a service.
|
|
* NOTE: This method should not be called directly. Instead, use `createContainerPreflight` to check prerequisites first
|
|
* This method will also transmit server-sent events to the client to notify of progress.
|
|
* @param serviceName
|
|
* @returns
|
|
*/
|
|
async _createContainer(
|
|
service: Service & { dependencies?: Service[] },
|
|
containerConfig: any
|
|
): Promise<void> {
|
|
try {
|
|
this._broadcast(service.service_name, 'initializing', '')
|
|
|
|
let dependencies = []
|
|
if (service.depends_on) {
|
|
const dependency = await Service.query().where('service_name', service.depends_on).first()
|
|
if (dependency) {
|
|
dependencies.push(dependency)
|
|
}
|
|
}
|
|
|
|
// First, check if the service has any dependencies that need to be installed first
|
|
if (dependencies && dependencies.length > 0) {
|
|
this._broadcast(
|
|
service.service_name,
|
|
'checking-dependencies',
|
|
`Checking dependencies for service ${service.service_name}...`
|
|
)
|
|
for (const dependency of dependencies) {
|
|
if (!dependency.installed) {
|
|
this._broadcast(
|
|
service.service_name,
|
|
'dependency-not-installed',
|
|
`Dependency service ${dependency.service_name} is not installed. Installing it first...`
|
|
)
|
|
await this._createContainer(
|
|
dependency,
|
|
this._parseContainerConfig(dependency.container_config)
|
|
)
|
|
} else {
|
|
this._broadcast(
|
|
service.service_name,
|
|
'dependency-installed',
|
|
`Dependency service ${dependency.service_name} is already installed.`
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
const imageExists = await this._checkImageExists(service.container_image)
|
|
if (imageExists) {
|
|
this._broadcast(
|
|
service.service_name,
|
|
'image-exists',
|
|
`Docker image ${service.container_image} already exists locally. Skipping pull...`
|
|
)
|
|
} else {
|
|
// Start pulling the Docker image and wait for it to complete
|
|
const pullStream = await this.docker.pull(service.container_image)
|
|
this._broadcast(
|
|
service.service_name,
|
|
'pulling',
|
|
`Pulling Docker image ${service.container_image}...`
|
|
)
|
|
await new Promise((res) => this.docker.modem.followProgress(pullStream, res))
|
|
}
|
|
|
|
if (service.service_name === SERVICE_NAMES.KIWIX) {
|
|
await this._runPreinstallActions__KiwixServe()
|
|
this._broadcast(
|
|
service.service_name,
|
|
'preinstall-complete',
|
|
`Pre-install actions for Kiwix Serve completed successfully.`
|
|
)
|
|
}
|
|
|
|
if (service.service_name === SERVICE_NAMES.CALIBREWEB) {
|
|
await this._runPreinstallActions__CalibreWeb()
|
|
this._broadcast(
|
|
service.service_name,
|
|
'preinstall-complete',
|
|
`Pre-install actions for Calibre-Web completed successfully.`
|
|
)
|
|
}
|
|
|
|
if (service.service_name === SERVICE_NAMES.VAULTWARDEN) {
|
|
await this._runPreinstallActions__Vaultwarden()
|
|
this._broadcast(
|
|
service.service_name,
|
|
'preinstall-complete',
|
|
`Pre-install actions for Vaultwarden completed successfully.`
|
|
)
|
|
}
|
|
|
|
if (service.service_name === SERVICE_NAMES.JELLYFIN) {
|
|
await this._runPreinstallActions__Jellyfin()
|
|
this._broadcast(
|
|
service.service_name,
|
|
'preinstall-complete',
|
|
`Pre-install actions for Jellyfin completed successfully.`
|
|
)
|
|
}
|
|
|
|
// GPU-aware configuration for Ollama
|
|
let finalImage = service.container_image
|
|
let gpuHostConfig = containerConfig?.HostConfig || {}
|
|
let amdGpuConfigured = false
|
|
|
|
if (service.service_name === SERVICE_NAMES.OLLAMA) {
|
|
const gpuResult = await this._detectGPUType()
|
|
|
|
if (gpuResult.type === 'nvidia') {
|
|
this._broadcast(
|
|
service.service_name,
|
|
'gpu-config',
|
|
`NVIDIA container runtime detected. Configuring container with GPU support...`
|
|
)
|
|
|
|
// Add GPU support for NVIDIA
|
|
gpuHostConfig = {
|
|
...gpuHostConfig,
|
|
DeviceRequests: [
|
|
{
|
|
Driver: 'nvidia',
|
|
Count: -1, // -1 means all GPUs
|
|
Capabilities: [['gpu']],
|
|
},
|
|
],
|
|
}
|
|
} else if (gpuResult.type === 'amd') {
|
|
// AMD acceleration is opt-out via the 'ai.amdGpuAcceleration' KV key (default-on).
|
|
// Per memory feedback: KV values can be string or boolean — coerce explicitly.
|
|
const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration')
|
|
const amdAccelerationEnabled = String(amdEnabledRaw) !== 'false'
|
|
|
|
if (amdAccelerationEnabled) {
|
|
this._broadcast(
|
|
service.service_name,
|
|
'gpu-config',
|
|
`AMD GPU detected. Using ROCm image with /dev/kfd and /dev/dri passthrough...`
|
|
)
|
|
|
|
finalImage = 'ollama/ollama:rocm'
|
|
|
|
// The pull-if-missing earlier in this function used service.container_image
|
|
// (the DB-pinned tag, e.g. ollama/ollama:0.18.2). The AMD branch overrides
|
|
// to a different tag — so we need to pull :rocm separately if it's not local.
|
|
const rocmImageExists = await this._checkImageExists(finalImage)
|
|
if (!rocmImageExists) {
|
|
this._broadcast(
|
|
service.service_name,
|
|
'pulling',
|
|
`Pulling Docker image ${finalImage}...`
|
|
)
|
|
const rocmPullStream = await this.docker.pull(finalImage)
|
|
await new Promise((res) => this.docker.modem.followProgress(rocmPullStream, res))
|
|
}
|
|
|
|
const amdDevices = await this._discoverAMDDevices()
|
|
gpuHostConfig = {
|
|
...gpuHostConfig,
|
|
Devices: amdDevices,
|
|
}
|
|
amdGpuConfigured = true
|
|
logger.info(
|
|
`[DockerService] Configured ROCm image and ${amdDevices.length} AMD device entries for Ollama`
|
|
)
|
|
} else {
|
|
this._broadcast(
|
|
service.service_name,
|
|
'gpu-config',
|
|
`AMD GPU detected but acceleration is disabled via ai.amdGpuAcceleration. Using CPU-only configuration.`
|
|
)
|
|
logger.info('[DockerService] AMD GPU acceleration disabled by KV opt-out; using CPU-only configuration.')
|
|
}
|
|
} else if (gpuResult.toolkitMissing) {
|
|
this._broadcast(
|
|
service.service_name,
|
|
'gpu-config',
|
|
`NVIDIA GPU detected but NVIDIA Container Toolkit is not installed. Using CPU-only configuration. Install the toolkit and reinstall AI Assistant for GPU acceleration: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html`
|
|
)
|
|
} else {
|
|
this._broadcast(
|
|
service.service_name,
|
|
'gpu-config',
|
|
`No GPU detected. Using CPU-only configuration...`
|
|
)
|
|
}
|
|
}
|
|
|
|
const ollamaEnv: string[] = []
|
|
if (service.service_name === SERVICE_NAMES.OLLAMA) {
|
|
ollamaEnv.push('OLLAMA_NO_CLOUD=1')
|
|
const flashAttentionEnabled = await KVStore.getValue('ai.ollamaFlashAttention')
|
|
if (flashAttentionEnabled !== false) {
|
|
ollamaEnv.push('OLLAMA_FLASH_ATTENTION=1')
|
|
}
|
|
if (amdGpuConfigured) {
|
|
// gfx-aware HSA override — only set for cards that actually need it. See
|
|
// _resolveAmdHsaOverride() for the resolution order and gfx → version mapping.
|
|
const hsaOverride = await this._resolveAmdHsaOverride()
|
|
if (hsaOverride) {
|
|
ollamaEnv.push(`HSA_OVERRIDE_GFX_VERSION=${hsaOverride}`)
|
|
}
|
|
}
|
|
}
|
|
|
|
this._broadcast(
|
|
service.service_name,
|
|
'creating',
|
|
`Creating Docker container for service ${service.service_name}...`
|
|
)
|
|
const container = await this.docker.createContainer({
|
|
Image: finalImage,
|
|
name: service.service_name,
|
|
Labels: {
|
|
...(containerConfig?.Labels ?? {}),
|
|
'com.docker.compose.project': 'project-nomad-managed',
|
|
'io.project-nomad.managed': 'true',
|
|
},
|
|
...(containerConfig?.User && { User: containerConfig.User }),
|
|
HostConfig: gpuHostConfig,
|
|
...(containerConfig?.WorkingDir && { WorkingDir: containerConfig.WorkingDir }),
|
|
...(containerConfig?.ExposedPorts && { ExposedPorts: containerConfig.ExposedPorts }),
|
|
Env: [...(containerConfig?.Env ?? []), ...ollamaEnv],
|
|
...(service.container_command ? { Cmd: service.container_command.split(' ') } : {}),
|
|
// Ensure container is attached to the Nomad docker network in production
|
|
...(process.env.NODE_ENV === 'production' && {
|
|
NetworkingConfig: {
|
|
EndpointsConfig: {
|
|
[DockerService.NOMAD_NETWORK]: {},
|
|
},
|
|
},
|
|
}),
|
|
})
|
|
|
|
this._broadcast(
|
|
service.service_name,
|
|
'starting',
|
|
`Starting Docker container for service ${service.service_name}...`
|
|
)
|
|
await container.start()
|
|
|
|
this._broadcast(
|
|
service.service_name,
|
|
'finalizing',
|
|
`Finalizing installation of service ${service.service_name}...`
|
|
)
|
|
service.installed = true
|
|
service.installation_status = 'idle'
|
|
await service.save()
|
|
this.invalidateServicesStatusCache()
|
|
|
|
// Remove from active installs tracking
|
|
this.activeInstallations.delete(service.service_name)
|
|
|
|
// If Ollama was just installed, trigger Nomad docs discovery and embedding
|
|
if (service.service_name === SERVICE_NAMES.OLLAMA) {
|
|
logger.info('[DockerService] Ollama installation complete. Default behavior is to not enable chat suggestions.')
|
|
await KVStore.setValue('chat.suggestionsEnabled', false)
|
|
|
|
logger.info('[DockerService] Ollama installation complete. Triggering Nomad docs discovery...')
|
|
|
|
// Need to use dynamic imports here to avoid circular dependency
|
|
const ollamaService = new (await import('./ollama_service.js')).OllamaService()
|
|
const ragService = new (await import('./rag_service.js')).RagService(this, ollamaService)
|
|
|
|
ragService.discoverNomadDocs().catch((error) => {
|
|
logger.error('[DockerService] Failed to discover Nomad docs:', error)
|
|
})
|
|
}
|
|
|
|
this._broadcast(
|
|
service.service_name,
|
|
'completed',
|
|
`Service ${service.service_name} installation completed successfully.`
|
|
)
|
|
} catch (error: any) {
|
|
this._broadcast(
|
|
service.service_name,
|
|
'error',
|
|
`Error installing service ${service.service_name}: ${error.message}`
|
|
)
|
|
// Mark install as failed and cleanup
|
|
await this._cleanupFailedInstallation(service.service_name)
|
|
throw new Error(`Failed to install service ${service.service_name}: ${error.message}`)
|
|
}
|
|
}
|
|
|
|
async _checkIfServiceContainerExists(serviceName: string): Promise<boolean> {
|
|
try {
|
|
const containers = await this.docker.listContainers({ all: true })
|
|
return containers.some((container) => container.Names.includes(`/${serviceName}`))
|
|
} catch (error: any) {
|
|
logger.error(`Error checking if service container exists: ${error.message}`)
|
|
return false
|
|
}
|
|
}
|
|
|
|
async _removeServiceContainer(
|
|
serviceName: string
|
|
): Promise<{ success: boolean; message: string }> {
|
|
try {
|
|
const containers = await this.docker.listContainers({ all: true })
|
|
const container = containers.find((c) => c.Names.includes(`/${serviceName}`))
|
|
if (!container) {
|
|
return { success: false, message: `Container for service ${serviceName} not found` }
|
|
}
|
|
|
|
const dockerContainer = this.docker.getContainer(container.Id)
|
|
await dockerContainer.remove({ force: true })
|
|
|
|
return { success: true, message: `Service ${serviceName} container removed successfully` }
|
|
} catch (error: any) {
|
|
logger.error({ err: error }, `[DockerService] Error removing service container ${serviceName}`)
|
|
return {
|
|
success: false,
|
|
message: `Failed to remove service ${serviceName} container. Check server logs for details.`,
|
|
}
|
|
}
|
|
}
|
|
|
|
private async _runPreinstallActions__KiwixServe(): Promise<void> {
|
|
/**
|
|
* At least one .zim file must be available before we can start the kiwix container.
|
|
* We'll download the lightweight mini Wikipedia Top 100 zim file for this purpose.
|
|
**/
|
|
const WIKIPEDIA_ZIM_URL =
|
|
'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/main/install/wikipedia_en_100_mini_2026-01.zim'
|
|
const filename = 'wikipedia_en_100_mini_2026-01.zim'
|
|
const filepath = join(process.cwd(), ZIM_STORAGE_PATH, filename)
|
|
logger.info(`[DockerService] Kiwix Serve pre-install: Downloading ZIM file to ${filepath}`)
|
|
|
|
this._broadcast(
|
|
SERVICE_NAMES.KIWIX,
|
|
'preinstall',
|
|
`Running pre-install actions for Kiwix Serve...`
|
|
)
|
|
this._broadcast(
|
|
SERVICE_NAMES.KIWIX,
|
|
'preinstall',
|
|
`Downloading Wikipedia ZIM file from ${WIKIPEDIA_ZIM_URL}. This may take some time...`
|
|
)
|
|
|
|
try {
|
|
await doResumableDownloadWithRetry({
|
|
url: WIKIPEDIA_ZIM_URL,
|
|
filepath,
|
|
timeout: 60000,
|
|
allowedMimeTypes: [
|
|
'application/x-zim',
|
|
'application/x-openzim',
|
|
'application/octet-stream',
|
|
],
|
|
})
|
|
|
|
this._broadcast(
|
|
SERVICE_NAMES.KIWIX,
|
|
'preinstall',
|
|
`Downloaded Wikipedia ZIM file to ${filepath}`
|
|
)
|
|
|
|
// Generate the initial kiwix library XML before the container is created
|
|
const kiwixLibraryService = new KiwixLibraryService()
|
|
await kiwixLibraryService.rebuildFromDisk()
|
|
this._broadcast(SERVICE_NAMES.KIWIX, 'preinstall', 'Generated kiwix library XML.')
|
|
} catch (error: any) {
|
|
this._broadcast(
|
|
SERVICE_NAMES.KIWIX,
|
|
'preinstall-error',
|
|
`Failed to download Wikipedia ZIM file: ${error.message}`
|
|
)
|
|
throw new Error(`Pre-install action failed: ${error.message}`)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Calibre-Web cannot create a library from scratch — without an existing Calibre database it
|
|
* dead-ends at the "Database Configuration" page. Seed an empty library (bundled in the admin
|
|
* image) into storage/books so the user just points Calibre-Web at /books and starts adding
|
|
* books. Only seeds when no metadata.db already exists, so an existing library is never clobbered.
|
|
* Ownership is handed to the container user (PUID/PGID 1000 in the seeder) so Calibre-Web can
|
|
* write to the library and create book folders on upload.
|
|
*/
|
|
private async _runPreinstallActions__CalibreWeb(): Promise<void> {
|
|
// Keep in sync with the PUID/PGID set on the Calibre-Web service in the seeder.
|
|
const CALIBRE_WEB_UID = 1000
|
|
const CALIBRE_WEB_GID = 1000
|
|
|
|
const booksDir = join(process.cwd(), BOOKS_STORAGE_PATH)
|
|
const metadataPath = join(booksDir, 'metadata.db')
|
|
const assetPath = join(process.cwd(), CALIBRE_EMPTY_LIBRARY_ASSET_PATH)
|
|
|
|
this._broadcast(
|
|
SERVICE_NAMES.CALIBREWEB,
|
|
'preinstall',
|
|
`Running pre-install actions for Calibre-Web...`
|
|
)
|
|
|
|
try {
|
|
await mkdir(booksDir, { recursive: true })
|
|
|
|
// Don't clobber an existing library — only seed when there's no metadata.db yet.
|
|
const alreadyHasLibrary = await access(metadataPath)
|
|
.then(() => true)
|
|
.catch(() => false)
|
|
|
|
if (alreadyHasLibrary) {
|
|
this._broadcast(
|
|
SERVICE_NAMES.CALIBREWEB,
|
|
'preinstall',
|
|
`Existing Calibre library found in books folder — leaving it as-is.`
|
|
)
|
|
} else {
|
|
await copyFile(assetPath, metadataPath)
|
|
this._broadcast(
|
|
SERVICE_NAMES.CALIBREWEB,
|
|
'preinstall',
|
|
`Seeded an empty Calibre library into the books folder.`
|
|
)
|
|
}
|
|
|
|
// Hand the books folder + library to the Calibre-Web container user so it can read/write
|
|
// the library and create book folders on upload (Docker creates bind dirs as root otherwise).
|
|
await chown(booksDir, CALIBRE_WEB_UID, CALIBRE_WEB_GID)
|
|
await chown(metadataPath, CALIBRE_WEB_UID, CALIBRE_WEB_GID)
|
|
} catch (error: any) {
|
|
this._broadcast(
|
|
SERVICE_NAMES.CALIBREWEB,
|
|
'preinstall-error',
|
|
`Failed to prepare the Calibre library: ${error.message}`
|
|
)
|
|
throw new Error(`Pre-install action failed: ${error.message}`)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Vaultwarden's web vault refuses to run without a secure context — over plain HTTP it shows
|
|
* "You need to enable HTTPS!" and you can't register or unlock. We give it a self-signed TLS
|
|
* cert in its /data volume so it serves HTTPS out of the box (paired with the ROCKET_TLS env and
|
|
* the `https:8480` ui_location set in the seeder). Users click through a one-time browser warning;
|
|
* after that the vault is fully functional offline. Self-signed is the only option on a LAN
|
|
* appliance with no public DNS to get a trusted cert for.
|
|
*/
|
|
private async _runPreinstallActions__Vaultwarden(): Promise<void> {
|
|
const dataDir = join(process.cwd(), VAULTWARDEN_STORAGE_PATH)
|
|
const certPath = join(dataDir, 'cert.pem')
|
|
const keyPath = join(dataDir, 'key.pem')
|
|
|
|
this._broadcast(
|
|
SERVICE_NAMES.VAULTWARDEN,
|
|
'preinstall',
|
|
`Running pre-install actions for Vaultwarden...`
|
|
)
|
|
|
|
try {
|
|
await mkdir(dataDir, { recursive: true })
|
|
|
|
// Don't regenerate if a cert is already present — keeps the same cert across reinstalls so
|
|
// users don't get a fresh browser warning every time, and never clobbers a cert an admin
|
|
// may have swapped in themselves.
|
|
const alreadyHasCert = await Promise.all([
|
|
access(certPath)
|
|
.then(() => true)
|
|
.catch(() => false),
|
|
access(keyPath)
|
|
.then(() => true)
|
|
.catch(() => false),
|
|
]).then(([c, k]) => c && k)
|
|
|
|
if (alreadyHasCert) {
|
|
this._broadcast(
|
|
SERVICE_NAMES.VAULTWARDEN,
|
|
'preinstall',
|
|
`Existing Vaultwarden TLS certificate found — leaving it as-is.`
|
|
)
|
|
return
|
|
}
|
|
|
|
// 10-year self-signed cert. CN/SAN are cosmetic for a self-signed cert (the browser warns
|
|
// regardless), but a SAN keeps it structurally valid for clients that require one.
|
|
const execAsync = promisify(exec)
|
|
await execAsync(
|
|
`openssl req -x509 -newkey rsa:2048 -nodes ` +
|
|
`-keyout "${keyPath}" -out "${certPath}" -days 3650 ` +
|
|
`-subj "/CN=Project NOMAD Vaultwarden" ` +
|
|
`-addext "subjectAltName=DNS:nomad,DNS:localhost"`
|
|
)
|
|
|
|
// Lock the private key down; the cert can stay world-readable. Vaultwarden runs as root and
|
|
// reads both from /data, so no chown is needed (admin writes them as root too).
|
|
await chmod(keyPath, 0o600)
|
|
await chmod(certPath, 0o644)
|
|
|
|
this._broadcast(
|
|
SERVICE_NAMES.VAULTWARDEN,
|
|
'preinstall',
|
|
`Generated a self-signed HTTPS certificate for Vaultwarden.`
|
|
)
|
|
} catch (error: any) {
|
|
this._broadcast(
|
|
SERVICE_NAMES.VAULTWARDEN,
|
|
'preinstall-error',
|
|
`Failed to prepare the Vaultwarden certificate: ${error.message}`
|
|
)
|
|
throw new Error(`Pre-install action failed: ${error.message}`)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Jellyfin works best when each library points at its own subfolder. Pointing one library at the
|
|
* whole media root and another at a subfolder inside it makes Jellyfin report a "duplicate path"
|
|
* and silently drop the nested library's content. To steer users toward the one-folder-per-type
|
|
* layout (and match the documentation), we pre-create the suggested subfolders under the shared
|
|
* media root so they're ready to pick in the setup wizard. Idempotent: only missing folders are
|
|
* created (mkdir is a no-op on existing ones) and nothing the user added is ever touched.
|
|
*/
|
|
private async _runPreinstallActions__Jellyfin(): Promise<void> {
|
|
const mediaDir = join(process.cwd(), MEDIA_STORAGE_PATH)
|
|
|
|
this._broadcast(
|
|
SERVICE_NAMES.JELLYFIN,
|
|
'preinstall',
|
|
`Running pre-install actions for Jellyfin...`
|
|
)
|
|
|
|
try {
|
|
await mkdir(mediaDir, { recursive: true })
|
|
for (const name of JELLYFIN_MEDIA_SUBFOLDERS) {
|
|
await mkdir(join(mediaDir, name), { recursive: true })
|
|
}
|
|
this._broadcast(
|
|
SERVICE_NAMES.JELLYFIN,
|
|
'preinstall',
|
|
`Prepared media folders: ${JELLYFIN_MEDIA_SUBFOLDERS.join(', ')}.`
|
|
)
|
|
} catch (error: any) {
|
|
this._broadcast(
|
|
SERVICE_NAMES.JELLYFIN,
|
|
'preinstall-error',
|
|
`Failed to prepare the Jellyfin media folders: ${error.message}`
|
|
)
|
|
throw new Error(`Pre-install action failed: ${error.message}`)
|
|
}
|
|
}
|
|
|
|
private async _cleanupFailedInstallation(serviceName: string): Promise<void> {
|
|
try {
|
|
const service = await Service.query().where('service_name', serviceName).first()
|
|
if (service) {
|
|
if (service.is_custom) {
|
|
// Custom apps have no seeder definition to fall back to — leaving the row would
|
|
// surface a phantom, un-installable card. Remove the record entirely so a failed
|
|
// install cleanly disappears. (The 'error' broadcast has already fired upstream.)
|
|
await service.delete()
|
|
} else {
|
|
service.installation_status = 'error'
|
|
await service.save()
|
|
}
|
|
}
|
|
this.activeInstallations.delete(serviceName)
|
|
|
|
// Ensure any partially created container is removed
|
|
await this._removeServiceContainer(serviceName)
|
|
|
|
logger.info(`[DockerService] Cleaned up failed installation for ${serviceName}`)
|
|
} catch (error: any) {
|
|
logger.error(
|
|
`[DockerService] Failed to cleanup installation for ${serviceName}: ${error.message}`
|
|
)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Checks whether the running kiwix container is using the legacy glob-pattern command
|
|
* (`*.zim --address=all`) rather than the library-file command. Used to detect containers
|
|
* that need to be migrated to library mode.
|
|
*/
|
|
async isKiwixOnLegacyConfig(): Promise<boolean> {
|
|
try {
|
|
const containers = await this.docker.listContainers({ all: true })
|
|
const info = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.KIWIX}`))
|
|
if (!info) return false
|
|
|
|
const inspected = await this.docker.getContainer(info.Id).inspect()
|
|
const cmd: string[] = inspected.Config?.Cmd ?? []
|
|
return cmd.some((arg) => arg.includes('*.zim'))
|
|
} catch (err: any) {
|
|
logger.warn(`[DockerService] Could not inspect kiwix container: ${err.message}`)
|
|
return false
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Migrates the kiwix container from legacy glob mode (`*.zim`) to library mode
|
|
* (`--library /data/kiwix-library.xml --monitorLibrary`).
|
|
*
|
|
* This is a non-destructive recreation: ZIM files and volumes are preserved.
|
|
* The container is stopped, removed, and recreated with the correct library-mode command.
|
|
* This function is authoritative: it writes the correct command to the DB itself rather than
|
|
* trusting the DB to have been pre-updated by a separate migration.
|
|
*/
|
|
async migrateKiwixToLibraryMode(): Promise<void> {
|
|
if (this.activeInstallations.has(SERVICE_NAMES.KIWIX)) {
|
|
logger.warn('[DockerService] Kiwix migration already in progress, skipping duplicate call.')
|
|
return
|
|
}
|
|
|
|
this.activeInstallations.add(SERVICE_NAMES.KIWIX)
|
|
|
|
try {
|
|
// Step 1: Build/update the XML from current disk state
|
|
this._broadcast(SERVICE_NAMES.KIWIX, 'migrating', 'Migrating kiwix to library mode...')
|
|
const kiwixLibraryService = new KiwixLibraryService()
|
|
await kiwixLibraryService.rebuildFromDisk()
|
|
this._broadcast(SERVICE_NAMES.KIWIX, 'migrating', 'Built kiwix library XML from existing ZIM files.')
|
|
|
|
// Step 2: Stop and remove old container (leave ZIM volumes intact)
|
|
const containers = await this.docker.listContainers({ all: true })
|
|
const containerInfo = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.KIWIX}`))
|
|
if (containerInfo) {
|
|
const oldContainer = this.docker.getContainer(containerInfo.Id)
|
|
if (containerInfo.State === 'running') {
|
|
await oldContainer.stop({ t: 10 }).catch((e: any) =>
|
|
logger.warn(`[DockerService] Kiwix stop warning during migration: ${e.message}`)
|
|
)
|
|
}
|
|
await oldContainer.remove({ force: true }).catch((e: any) =>
|
|
logger.warn(`[DockerService] Kiwix remove warning during migration: ${e.message}`)
|
|
)
|
|
}
|
|
|
|
// Step 3: Read the service record and authoritatively set the correct command.
|
|
// Do NOT rely on prior DB state — we write container_command here so the record
|
|
// stays consistent regardless of whether the DB migration ran.
|
|
const service = await Service.query().where('service_name', SERVICE_NAMES.KIWIX).first()
|
|
if (!service) {
|
|
throw new Error('Kiwix service record not found in DB during migration')
|
|
}
|
|
|
|
service.container_command = KIWIX_LIBRARY_CMD
|
|
service.installed = false
|
|
service.installation_status = 'installing'
|
|
await service.save()
|
|
|
|
const containerConfig = this._parseContainerConfig(service.container_config)
|
|
|
|
// Step 4: Recreate container directly (skipping _createContainer to avoid re-downloading
|
|
// the bootstrap ZIM — ZIM files already exist on disk)
|
|
this._broadcast(SERVICE_NAMES.KIWIX, 'migrating', 'Recreating kiwix container with library mode config...')
|
|
const newContainer = await this.docker.createContainer({
|
|
Image: service.container_image,
|
|
name: service.service_name,
|
|
HostConfig: containerConfig?.HostConfig ?? {},
|
|
...(containerConfig?.ExposedPorts && { ExposedPorts: containerConfig.ExposedPorts }),
|
|
Cmd: KIWIX_LIBRARY_CMD.split(' '),
|
|
...(process.env.NODE_ENV === 'production' && {
|
|
NetworkingConfig: {
|
|
EndpointsConfig: {
|
|
[DockerService.NOMAD_NETWORK]: {},
|
|
},
|
|
},
|
|
}),
|
|
})
|
|
|
|
await newContainer.start()
|
|
|
|
service.installed = true
|
|
service.installation_status = 'idle'
|
|
await service.save()
|
|
this.activeInstallations.delete(SERVICE_NAMES.KIWIX)
|
|
|
|
this._broadcast(SERVICE_NAMES.KIWIX, 'migrated', 'Kiwix successfully migrated to library mode.')
|
|
logger.info('[DockerService] Kiwix migration to library mode complete.')
|
|
} catch (error: any) {
|
|
logger.error(`[DockerService] Kiwix migration failed: ${error.message}`)
|
|
await this._cleanupFailedInstallation(SERVICE_NAMES.KIWIX)
|
|
throw error
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Detect GPU type and toolkit availability.
|
|
* Primary: Check Docker runtimes via docker.info() (works from inside containers).
|
|
* Secondary: Read /app/storage/.nomad-gpu-type written by install_nomad.sh — needed
|
|
* for AMD detection because lspci isn't available inside the admin container and
|
|
* AMD has no Docker runtime registration to query.
|
|
* Fallback: lspci for host-based installs.
|
|
*/
|
|
private async _detectGPUType(): Promise<{ type: 'nvidia' | 'amd' | 'none'; toolkitMissing?: boolean }> {
|
|
try {
|
|
// Primary: Check Docker daemon for nvidia runtime (works from inside containers)
|
|
try {
|
|
const dockerInfo = await this.docker.info()
|
|
const runtimes = dockerInfo.Runtimes || {}
|
|
if ('nvidia' in runtimes) {
|
|
logger.info('[DockerService] NVIDIA container runtime detected via Docker API')
|
|
await this._persistGPUType('nvidia')
|
|
return { type: 'nvidia' }
|
|
}
|
|
} catch (error: any) {
|
|
logger.warn(`[DockerService] Could not query Docker info for GPU runtimes: ${error.message}`)
|
|
}
|
|
|
|
// Secondary: install_nomad.sh writes the host-detected GPU type to a marker file in
|
|
// the storage volume so the admin container (which lacks lspci) can read it.
|
|
try {
|
|
const marker = (await readFile('/app/storage/.nomad-gpu-type', 'utf8')).trim()
|
|
if (marker === 'nvidia') {
|
|
// Hardware present but Docker doesn't have nvidia runtime → toolkit missing
|
|
logger.warn('[DockerService] NVIDIA GPU recorded in marker file but NVIDIA Container Toolkit is not installed')
|
|
return { type: 'none', toolkitMissing: true }
|
|
}
|
|
if (marker === 'amd') {
|
|
logger.info('[DockerService] AMD GPU detected via install-time marker file')
|
|
await this._persistGPUType('amd')
|
|
return { type: 'amd' }
|
|
}
|
|
} catch {
|
|
// No marker file — fall through to lspci attempt for host-based installs
|
|
}
|
|
|
|
// Fallback: lspci for host-based installs (not available inside Docker)
|
|
const execAsync = promisify(exec)
|
|
|
|
// Check for NVIDIA GPU via lspci
|
|
try {
|
|
const { stdout: nvidiaCheck } = await execAsync(
|
|
'lspci 2>/dev/null | grep -i nvidia || true'
|
|
)
|
|
if (nvidiaCheck.trim()) {
|
|
// GPU hardware found but no nvidia runtime — toolkit not installed
|
|
logger.warn('[DockerService] NVIDIA GPU detected via lspci but NVIDIA Container Toolkit is not installed')
|
|
return { type: 'none', toolkitMissing: true }
|
|
}
|
|
} catch (error: any) {
|
|
// lspci not available (likely inside Docker container), continue
|
|
}
|
|
|
|
// Check for AMD GPU via lspci — restrict to display controller classes to avoid
|
|
// false positives from AMD CPU host bridges, PCI bridges, and chipset devices.
|
|
try {
|
|
const { stdout: amdCheck } = await execAsync(
|
|
'lspci 2>/dev/null | grep -iE "VGA|3D controller|Display" | grep -iE "amd|radeon" || true'
|
|
)
|
|
if (amdCheck.trim()) {
|
|
logger.info('[DockerService] AMD GPU detected via lspci')
|
|
await this._persistGPUType('amd')
|
|
return { type: 'amd' }
|
|
}
|
|
} catch (error: any) {
|
|
// lspci not available, continue
|
|
}
|
|
|
|
// Last resort: check if we previously detected a GPU and it's likely still present.
|
|
// This handles cases where live detection fails transiently (e.g., Docker daemon
|
|
// hiccup, runtime temporarily unavailable) but the hardware hasn't changed.
|
|
try {
|
|
const savedType = await KVStore.getValue('gpu.type')
|
|
if (savedType === 'nvidia' || savedType === 'amd') {
|
|
logger.info(`[DockerService] No GPU detected live, but KV store has '${savedType}' from previous detection. Using saved value.`)
|
|
return { type: savedType as 'nvidia' | 'amd' }
|
|
}
|
|
} catch {
|
|
// KV store not available, continue
|
|
}
|
|
|
|
logger.info('[DockerService] No GPU detected')
|
|
return { type: 'none' }
|
|
} catch (error: any) {
|
|
logger.warn(`[DockerService] Error detecting GPU type: ${error.message}`)
|
|
return { type: 'none' }
|
|
}
|
|
}
|
|
|
|
private async _persistGPUType(type: 'nvidia' | 'amd'): Promise<void> {
|
|
try {
|
|
await KVStore.setValue('gpu.type', type)
|
|
logger.info(`[DockerService] Persisted GPU type '${type}' to KV store`)
|
|
} catch (error: any) {
|
|
logger.warn(`[DockerService] Failed to persist GPU type: ${error.message}`)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Resolve the HSA_OVERRIDE_GFX_VERSION value for the host's AMD GPU.
|
|
*
|
|
* gfx1030 (RX 6800/6700/etc.), gfx1100/1101/1102 (RX 7900/7800/7600) are on AMD's
|
|
* official ROCm allowlist — forcing an override on these breaks GPU discovery.
|
|
* gfx1035 / gfx1036 (RDNA 2 iGPUs like 680M) need 10.3.0 to coerce to gfx1030.
|
|
* gfx1103 / gfx1150 / gfx1151 (RDNA 3/3.5 iGPUs like 780M / 890M / Strix Halo) need 11.0.0.
|
|
*
|
|
* Resolution order:
|
|
* 1. KV `ai.amdHsaOverride` — manual user override; accepts 'none' (disable) or a semver-style value.
|
|
* 2. Marker file `/app/storage/.nomad-amd-gfx` written by install_nomad.sh.
|
|
* 3. Default: '11.0.0' — preserves prior behavior so existing iGPU users don't regress on
|
|
* upgrade. Discrete-card users on existing installs can opt out via the KV.
|
|
*
|
|
* Returns null when no override should be applied.
|
|
*/
|
|
private async _resolveAmdHsaOverride(): Promise<string | null> {
|
|
const manualRaw = await KVStore.getValue('ai.amdHsaOverride')
|
|
if (manualRaw !== null && manualRaw !== undefined && String(manualRaw).trim() !== '') {
|
|
const manual = String(manualRaw).trim().toLowerCase()
|
|
if (manual === 'none' || manual === 'off' || manual === 'false') {
|
|
logger.info('[DockerService] HSA override disabled via ai.amdHsaOverride')
|
|
return null
|
|
}
|
|
if (/^\d+\.\d+\.\d+$/.test(manual)) {
|
|
logger.info(`[DockerService] HSA override forced to ${manual} via ai.amdHsaOverride`)
|
|
return manual
|
|
}
|
|
logger.warn(`[DockerService] Ignoring invalid ai.amdHsaOverride value: ${manualRaw}`)
|
|
}
|
|
|
|
try {
|
|
const gfx = (await readFile('/app/storage/.nomad-amd-gfx', 'utf8')).trim()
|
|
const mapped = this._mapGfxToHsaOverride(gfx)
|
|
logger.info(`[DockerService] AMD gfx marker '${gfx}' → HSA override ${mapped ?? 'none'}`)
|
|
return mapped
|
|
} catch {
|
|
// Marker absent — most likely an existing install upgraded without re-running
|
|
// install_nomad.sh. Fall through to the default.
|
|
}
|
|
|
|
logger.info('[DockerService] No AMD gfx marker; defaulting HSA override to 11.0.0 for backward compatibility')
|
|
return '11.0.0'
|
|
}
|
|
|
|
private _mapGfxToHsaOverride(gfx: string): string | null {
|
|
// Officially supported by ROCm — no override needed
|
|
if (gfx === 'gfx1030' || gfx === 'gfx1100' || gfx === 'gfx1101' || gfx === 'gfx1102') {
|
|
return null
|
|
}
|
|
// RDNA 2 variants + iGPUs (gfx1031..gfx1036, e.g. Rembrandt 680M)
|
|
if (/^gfx103[1-6]$/.test(gfx)) {
|
|
return '10.3.0'
|
|
}
|
|
// RDNA 3 / 3.5 mobile parts (Phoenix 780M = gfx1103, Strix 890M = gfx1150, Strix Halo = gfx1151)
|
|
if (gfx === 'gfx1103' || gfx === 'gfx1150' || gfx === 'gfx1151') {
|
|
return '11.0.0'
|
|
}
|
|
return '11.0.0'
|
|
}
|
|
|
|
/**
|
|
* Build the Docker Devices array for AMD GPU passthrough.
|
|
*
|
|
* Returns /dev/kfd (Kernel Fusion Driver, required by ROCm) and /dev/dri (the DRM
|
|
* device tree). Passing /dev/dri as a single directory entry mirrors Docker CLI
|
|
* --device behavior — the daemon expands it to all child devices (card*, renderD*)
|
|
* regardless of how the host enumerates them. This avoids the brittle hardcoded
|
|
* fallback (card0/renderD128) the prior implementation used, which was wrong on
|
|
* systems where the AMD GPU enumerates as card1+ (e.g. UM890 Pro 780M iGPU).
|
|
*/
|
|
private async _discoverAMDDevices(): Promise<
|
|
Array<{ PathOnHost: string; PathInContainer: string; CgroupPermissions: string }>
|
|
> {
|
|
return [
|
|
{ PathOnHost: '/dev/kfd', PathInContainer: '/dev/kfd', CgroupPermissions: 'rwm' },
|
|
{ PathOnHost: '/dev/dri', PathInContainer: '/dev/dri', CgroupPermissions: 'rwm' },
|
|
]
|
|
}
|
|
|
|
/**
|
|
* Update a service container to a new image version while preserving volumes and data.
|
|
* Includes automatic rollback if the new container fails health checks.
|
|
*/
|
|
async updateContainer(
|
|
serviceName: string,
|
|
targetVersion: string
|
|
): Promise<{ success: boolean; message: string }> {
|
|
try {
|
|
const service = await Service.query().where('service_name', serviceName).first()
|
|
if (!service) {
|
|
return { success: false, message: `Service ${serviceName} not found` }
|
|
}
|
|
if (!service.installed) {
|
|
return { success: false, message: `Service ${serviceName} is not installed` }
|
|
}
|
|
if (this.activeInstallations.has(serviceName)) {
|
|
return { success: false, message: `Service ${serviceName} already has an operation in progress` }
|
|
}
|
|
|
|
this.activeInstallations.add(serviceName)
|
|
|
|
// newImage = the semver tag we record in the DB after the update (e.g. ollama/ollama:0.23.2).
|
|
// runtimeImage = the tag we actually pull and run. For AMD-on-Ollama these diverge: we run
|
|
// the rolling :rocm tag because per-version ROCm tags aren't always published, but the DB
|
|
// must keep the semver tag so the Apps page shows the actual version (not literally "rocm")
|
|
// and the registry update-check parses a valid tag (instead of looping on the same update).
|
|
const currentImage = service.container_image
|
|
const imageBase = currentImage.includes(':')
|
|
? currentImage.substring(0, currentImage.lastIndexOf(':'))
|
|
: currentImage
|
|
const newImage = `${imageBase}:${targetVersion}`
|
|
let runtimeImage = newImage
|
|
|
|
// GPU detection runs before the pull so AMD updates pull ollama/ollama:rocm rather
|
|
// than the standard tag. Detection result is reused below when building the new
|
|
// container config (devices, env). Non-Ollama services skip this entirely.
|
|
let updatedDeviceRequests: any[] | undefined = undefined
|
|
let updatedAmdDevices: any[] | undefined = undefined
|
|
let updatedAmdGpuConfigured = false
|
|
if (serviceName === SERVICE_NAMES.OLLAMA) {
|
|
const gpuResult = await this._detectGPUType()
|
|
if (gpuResult.type === 'nvidia') {
|
|
this._broadcast(
|
|
serviceName,
|
|
'update-gpu-config',
|
|
`NVIDIA container runtime detected. Configuring updated container with GPU support...`
|
|
)
|
|
updatedDeviceRequests = [
|
|
{ Driver: 'nvidia', Count: -1, Capabilities: [['gpu']] },
|
|
]
|
|
} else if (gpuResult.type === 'amd') {
|
|
const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration')
|
|
const amdAccelerationEnabled = String(amdEnabledRaw) !== 'false'
|
|
if (amdAccelerationEnabled) {
|
|
this._broadcast(
|
|
serviceName,
|
|
'update-gpu-config',
|
|
`AMD GPU detected. Using ROCm image with /dev/kfd and /dev/dri passthrough...`
|
|
)
|
|
runtimeImage = 'ollama/ollama:rocm'
|
|
updatedAmdDevices = await this._discoverAMDDevices()
|
|
updatedAmdGpuConfigured = true
|
|
} else {
|
|
this._broadcast(
|
|
serviceName,
|
|
'update-gpu-config',
|
|
`AMD GPU detected but acceleration is disabled via ai.amdGpuAcceleration. Using CPU-only configuration.`
|
|
)
|
|
}
|
|
} else if (gpuResult.toolkitMissing) {
|
|
this._broadcast(
|
|
serviceName,
|
|
'update-gpu-config',
|
|
`NVIDIA GPU detected but NVIDIA Container Toolkit is not installed. Using CPU-only configuration. Install the toolkit and reinstall AI Assistant for GPU acceleration: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html`
|
|
)
|
|
} else {
|
|
this._broadcast(serviceName, 'update-gpu-config', `No GPU detected. Using CPU-only configuration.`)
|
|
}
|
|
}
|
|
|
|
// Step 1: Pull new image (runtimeImage diverges from newImage for AMD, see above)
|
|
this._broadcast(serviceName, 'update-pulling', `Pulling image ${runtimeImage}...`)
|
|
const pullStream = await this.docker.pull(runtimeImage)
|
|
await new Promise((res) => this.docker.modem.followProgress(pullStream, res))
|
|
|
|
// Step 2: Find and stop existing container
|
|
this._broadcast(serviceName, 'update-stopping', `Stopping current container...`)
|
|
const containers = await this.docker.listContainers({ all: true })
|
|
const existingContainer = containers.find((c) => c.Names.includes(`/${serviceName}`))
|
|
|
|
if (!existingContainer) {
|
|
this.activeInstallations.delete(serviceName)
|
|
return { success: false, message: `Container for ${serviceName} not found` }
|
|
}
|
|
|
|
const oldContainer = this.docker.getContainer(existingContainer.Id)
|
|
|
|
// Inspect to capture full config before stopping
|
|
const inspectData = await oldContainer.inspect()
|
|
|
|
if (existingContainer.State === 'running') {
|
|
await oldContainer.stop({ t: 15 })
|
|
}
|
|
|
|
// Step 3: Rename old container as safety net
|
|
const oldName = `${serviceName}_old`
|
|
await oldContainer.rename({ name: oldName })
|
|
|
|
// Step 4: Create new container with inspected config + new image
|
|
this._broadcast(serviceName, 'update-creating', `Creating updated container...`)
|
|
|
|
const hostConfig = inspectData.HostConfig || {}
|
|
|
|
// GPU detection already ran above (before the pull) so we know the right image, devices,
|
|
// and whether HSA_OVERRIDE needs injection. For AMD, replace any prior HSA_OVERRIDE in
|
|
// the inspect-captured env so updates from older containers pick up the current value.
|
|
const baseEnv = inspectData.Config?.Env || []
|
|
let finalEnv = baseEnv
|
|
if (updatedAmdGpuConfigured) {
|
|
const hsaOverride = await this._resolveAmdHsaOverride()
|
|
finalEnv = baseEnv.filter((e: string) => !e.startsWith('HSA_OVERRIDE_GFX_VERSION='))
|
|
if (hsaOverride) {
|
|
finalEnv.push(`HSA_OVERRIDE_GFX_VERSION=${hsaOverride}`)
|
|
}
|
|
}
|
|
|
|
const newContainerConfig: any = {
|
|
Image: runtimeImage,
|
|
name: serviceName,
|
|
Env: finalEnv.length > 0 ? finalEnv : undefined,
|
|
Cmd: inspectData.Config?.Cmd || undefined,
|
|
ExposedPorts: inspectData.Config?.ExposedPorts || undefined,
|
|
WorkingDir: inspectData.Config?.WorkingDir || undefined,
|
|
User: inspectData.Config?.User || undefined,
|
|
HostConfig: {
|
|
Binds: hostConfig.Binds || undefined,
|
|
PortBindings: hostConfig.PortBindings || undefined,
|
|
RestartPolicy: hostConfig.RestartPolicy || undefined,
|
|
DeviceRequests: serviceName === SERVICE_NAMES.OLLAMA ? updatedDeviceRequests : (hostConfig.DeviceRequests || undefined),
|
|
Devices: serviceName === SERVICE_NAMES.OLLAMA && updatedAmdDevices ? updatedAmdDevices : (hostConfig.Devices || undefined),
|
|
},
|
|
NetworkingConfig: inspectData.NetworkSettings?.Networks
|
|
? {
|
|
EndpointsConfig: Object.fromEntries(
|
|
Object.keys(inspectData.NetworkSettings.Networks).map((net) => [net, {}])
|
|
),
|
|
}
|
|
: undefined,
|
|
}
|
|
|
|
// Remove undefined values from HostConfig
|
|
Object.keys(newContainerConfig.HostConfig).forEach((key) => {
|
|
if (newContainerConfig.HostConfig[key] === undefined) {
|
|
delete newContainerConfig.HostConfig[key]
|
|
}
|
|
})
|
|
|
|
let newContainer: any
|
|
try {
|
|
newContainer = await this.docker.createContainer(newContainerConfig)
|
|
} catch (createError: any) {
|
|
// Rollback: rename old container back
|
|
this._broadcast(serviceName, 'update-rollback', `Failed to create new container: ${createError.message}. Rolling back...`)
|
|
const rollbackContainer = this.docker.getContainer((await this.docker.listContainers({ all: true })).find((c) => c.Names.includes(`/${oldName}`))!.Id)
|
|
await rollbackContainer.rename({ name: serviceName })
|
|
await rollbackContainer.start()
|
|
this.activeInstallations.delete(serviceName)
|
|
return { success: false, message: `Failed to create updated container: ${createError.message}` }
|
|
}
|
|
|
|
// Step 5: Start new container
|
|
this._broadcast(serviceName, 'update-starting', `Starting updated container...`)
|
|
await newContainer.start()
|
|
|
|
// Step 6: Health check — verify container stays running for 5 seconds
|
|
await new Promise((resolve) => setTimeout(resolve, 5000))
|
|
const newContainerInfo = await newContainer.inspect()
|
|
|
|
if (newContainerInfo.State?.Running) {
|
|
// Healthy — clean up old container
|
|
try {
|
|
const oldContainerRef = this.docker.getContainer(
|
|
(await this.docker.listContainers({ all: true })).find((c) =>
|
|
c.Names.includes(`/${oldName}`)
|
|
)?.Id || ''
|
|
)
|
|
await oldContainerRef.remove({ force: true })
|
|
} catch {
|
|
// Old container may already be gone
|
|
}
|
|
|
|
// Update DB
|
|
service.container_image = newImage
|
|
service.available_update_version = null
|
|
await service.save()
|
|
|
|
this.activeInstallations.delete(serviceName)
|
|
this._broadcast(
|
|
serviceName,
|
|
'update-complete',
|
|
`Successfully updated ${serviceName} to ${targetVersion}`
|
|
)
|
|
return { success: true, message: `Service ${serviceName} updated to ${targetVersion}` }
|
|
} else {
|
|
// Unhealthy — rollback
|
|
this._broadcast(
|
|
serviceName,
|
|
'update-rollback',
|
|
`New container failed health check. Rolling back to previous version...`
|
|
)
|
|
|
|
try {
|
|
await newContainer.stop({ t: 5 }).catch(() => {})
|
|
await newContainer.remove({ force: true })
|
|
} catch {
|
|
// Best effort cleanup
|
|
}
|
|
|
|
// Restore old container
|
|
const oldContainers = await this.docker.listContainers({ all: true })
|
|
const oldRef = oldContainers.find((c) => c.Names.includes(`/${oldName}`))
|
|
if (oldRef) {
|
|
const rollbackContainer = this.docker.getContainer(oldRef.Id)
|
|
await rollbackContainer.rename({ name: serviceName })
|
|
await rollbackContainer.start()
|
|
}
|
|
|
|
this.activeInstallations.delete(serviceName)
|
|
return {
|
|
success: false,
|
|
message: `Update failed: new container did not stay running. Rolled back to previous version.`,
|
|
}
|
|
}
|
|
} catch (error: any) {
|
|
this.activeInstallations.delete(serviceName)
|
|
this._broadcast(
|
|
serviceName,
|
|
'update-rollback',
|
|
'Update failed. Check server logs for details.'
|
|
)
|
|
logger.error({ err: error }, `[DockerService] Update failed for ${serviceName}`)
|
|
return { success: false, message: 'Update failed. Check server logs for details.' }
|
|
}
|
|
}
|
|
|
|
private _broadcast(service: string, status: string, message: string) {
|
|
transmit.broadcast(BROADCAST_CHANNELS.SERVICE_INSTALLATION, {
|
|
service_name: service,
|
|
timestamp: new Date().toISOString(),
|
|
status,
|
|
message,
|
|
})
|
|
logger.info(`[DockerService] [${service}] ${status}: ${message}`)
|
|
}
|
|
|
|
private _parseContainerConfig(containerConfig: any): any {
|
|
if (!containerConfig) {
|
|
return {}
|
|
}
|
|
|
|
try {
|
|
// Handle the case where containerConfig is returned as an object by DB instead of a string
|
|
let toParse = containerConfig
|
|
if (typeof containerConfig === 'object') {
|
|
toParse = JSON.stringify(containerConfig)
|
|
}
|
|
|
|
return JSON.parse(toParse)
|
|
} catch (error: any) {
|
|
logger.error(`Failed to parse container configuration: ${error.message}`)
|
|
throw new Error(`Invalid container configuration: ${error.message}`)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check whether any of the supplied host ports are already bound by a running or stopped
|
|
* Docker container. Uses the Docker API exclusively — probing ports via net.createServer()
|
|
* would only test the admin container's own network namespace (DooD pattern), not the host.
|
|
*/
|
|
async checkPortConflicts(
|
|
ports: number[]
|
|
): Promise<{ conflicts: { port: number; usedBy: string }[] }> {
|
|
if (!ports.length) return { conflicts: [] }
|
|
|
|
try {
|
|
const containers = await this.docker.listContainers({ all: true })
|
|
const bound = new Map<number, string>()
|
|
|
|
for (const c of containers) {
|
|
const name = (c.Names[0] || '').replace('/', '')
|
|
for (const p of c.Ports) {
|
|
if (p.PublicPort) bound.set(p.PublicPort, name || c.Id.slice(0, 12))
|
|
}
|
|
}
|
|
|
|
const conflicts = ports
|
|
.filter((p) => bound.has(p))
|
|
.map((p) => ({ port: p, usedBy: bound.get(p)! }))
|
|
|
|
return { conflicts }
|
|
} catch (error: any) {
|
|
logger.warn(`[DockerService] checkPortConflicts failed: ${error.message}`)
|
|
return { conflicts: [] }
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Remove a custom-app container and, when `removeImage` is set, its backing image too. Called
|
|
* before deleting the DB record. Image removal is best-effort: a shared/in-use image is left alone.
|
|
*/
|
|
async removeCustomAppContainer(
|
|
serviceName: string,
|
|
removeImage = false
|
|
): Promise<{ success: boolean; message: string }> {
|
|
try {
|
|
const containers = await this.docker.listContainers({ all: true })
|
|
const container = containers.find((c) => c.Names.includes(`/${serviceName}`))
|
|
|
|
if (!container) return { success: true, message: 'No container found — nothing to remove' }
|
|
|
|
const imageRef = container.Image
|
|
const c = this.docker.getContainer(container.Id)
|
|
if (container.State === 'running') await c.stop()
|
|
await c.remove({ force: true })
|
|
|
|
if (removeImage && imageRef) {
|
|
try {
|
|
await this.docker.getImage(imageRef).remove()
|
|
} catch (imgErr: any) {
|
|
// Non-fatal: the image may be shared with another container or already gone.
|
|
logger.warn(`[DockerService] Could not remove image ${imageRef} for ${serviceName}: ${imgErr.message}`)
|
|
}
|
|
}
|
|
|
|
this.invalidateServicesStatusCache()
|
|
return { success: true, message: `Container ${serviceName} removed` }
|
|
} catch (error: any) {
|
|
logger.error({ err: error }, `[DockerService] removeCustomAppContainer failed for ${serviceName}`)
|
|
return { success: false, message: error.message }
|
|
}
|
|
}
|
|
|
|
/** Find a container by its managed service name (`/serviceName`), or null. */
|
|
private async _findContainerByName(serviceName: string) {
|
|
const containers = await this.docker.listContainers({ all: true })
|
|
return containers.find((c) => c.Names.includes(`/${serviceName}`)) ?? null
|
|
}
|
|
|
|
/**
|
|
* Decode the multiplexed stream Docker returns for non-TTY container logs. Each frame is an
|
|
* 8-byte header ([streamType, 0,0,0, big-endian payloadSize]) followed by the payload.
|
|
*/
|
|
private _demuxDockerLog(buf: Buffer): string {
|
|
let out = ''
|
|
let offset = 0
|
|
while (offset + 8 <= buf.length) {
|
|
const size = buf.readUInt32BE(offset + 4)
|
|
offset += 8
|
|
if (offset + size > buf.length) {
|
|
out += buf.toString('utf8', offset)
|
|
break
|
|
}
|
|
out += buf.toString('utf8', offset, offset + size)
|
|
offset += size
|
|
}
|
|
return out
|
|
}
|
|
|
|
/** Return the last `tail` lines of a service container's combined stdout/stderr. */
|
|
async getContainerLogs(
|
|
serviceName: string,
|
|
tail = 200
|
|
): Promise<{ success: boolean; logs?: string; message?: string }> {
|
|
try {
|
|
const info = await this._findContainerByName(serviceName)
|
|
if (!info) return { success: false, message: `No container found for ${serviceName}` }
|
|
|
|
const container = this.docker.getContainer(info.Id)
|
|
const inspect = await container.inspect()
|
|
const tty = inspect.Config?.Tty ?? false
|
|
|
|
const buf = (await container.logs({
|
|
stdout: true,
|
|
stderr: true,
|
|
follow: false,
|
|
tail,
|
|
timestamps: false,
|
|
})) as unknown as Buffer
|
|
|
|
const logs = tty ? buf.toString('utf8') : this._demuxDockerLog(buf)
|
|
return { success: true, logs }
|
|
} catch (error: any) {
|
|
logger.error({ err: error }, `[DockerService] getContainerLogs failed for ${serviceName}`)
|
|
return { success: false, message: error.message }
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Return a single resource-usage snapshot (CPU %, memory) for a running service container.
|
|
* Uses Docker's non-streaming stats, which include precpu_stats so CPU % is computable.
|
|
*/
|
|
async getContainerStats(serviceName: string): Promise<{
|
|
success: boolean
|
|
running?: boolean
|
|
stats?: { cpuPercent: number; memUsageBytes: number; memLimitBytes: number; memPercent: number }
|
|
message?: string
|
|
}> {
|
|
try {
|
|
const info = await this._findContainerByName(serviceName)
|
|
if (!info) return { success: false, message: `No container found for ${serviceName}` }
|
|
if (info.State !== 'running') return { success: true, running: false }
|
|
|
|
const container = this.docker.getContainer(info.Id)
|
|
const s: any = await container.stats({ stream: false })
|
|
|
|
const cpuDelta =
|
|
(s.cpu_stats?.cpu_usage?.total_usage ?? 0) - (s.precpu_stats?.cpu_usage?.total_usage ?? 0)
|
|
const systemDelta =
|
|
(s.cpu_stats?.system_cpu_usage ?? 0) - (s.precpu_stats?.system_cpu_usage ?? 0)
|
|
const numCpus =
|
|
s.cpu_stats?.online_cpus ?? s.cpu_stats?.cpu_usage?.percpu_usage?.length ?? 1
|
|
const cpuPercent =
|
|
systemDelta > 0 && cpuDelta > 0 ? (cpuDelta / systemDelta) * numCpus * 100 : 0
|
|
|
|
// Subtract page cache from usage to better reflect the container's working set.
|
|
const cache = s.memory_stats?.stats?.cache ?? s.memory_stats?.stats?.inactive_file ?? 0
|
|
const memUsageBytes = Math.max(0, (s.memory_stats?.usage ?? 0) - cache)
|
|
const memLimitBytes = s.memory_stats?.limit ?? 0
|
|
const memPercent = memLimitBytes > 0 ? (memUsageBytes / memLimitBytes) * 100 : 0
|
|
|
|
return {
|
|
success: true,
|
|
running: true,
|
|
stats: {
|
|
cpuPercent: Math.round(cpuPercent * 10) / 10,
|
|
memUsageBytes,
|
|
memLimitBytes,
|
|
memPercent: Math.round(memPercent * 10) / 10,
|
|
},
|
|
}
|
|
} catch (error: any) {
|
|
logger.error({ err: error }, `[DockerService] getContainerStats failed for ${serviceName}`)
|
|
return { success: false, message: error.message }
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Wait for a freshly started container to be "ready". If the image declares a HEALTHCHECK we poll
|
|
* its health until healthy/unhealthy (up to timeoutMs); otherwise we fall back to a 5s settle and
|
|
* a plain Running check. Returns whether it's ready plus a reason when not.
|
|
*/
|
|
private async _awaitContainerReady(
|
|
container: any,
|
|
timeoutMs = 30000
|
|
): Promise<{ ready: boolean; reason?: string }> {
|
|
let inspect = await container.inspect()
|
|
const hasHealthcheck = !!inspect.State?.Health
|
|
|
|
if (!hasHealthcheck) {
|
|
await new Promise((r) => setTimeout(r, 5000))
|
|
inspect = await container.inspect()
|
|
return inspect.State?.Running
|
|
? { ready: true }
|
|
: { ready: false, reason: 'container did not stay running' }
|
|
}
|
|
|
|
const deadline = Date.now() + timeoutMs
|
|
while (Date.now() < deadline) {
|
|
inspect = await container.inspect()
|
|
if (!inspect.State?.Running) return { ready: false, reason: 'container exited' }
|
|
const status = inspect.State?.Health?.Status
|
|
if (status === 'healthy') return { ready: true }
|
|
if (status === 'unhealthy') return { ready: false, reason: 'failed its health check' }
|
|
await new Promise((r) => setTimeout(r, 2000))
|
|
}
|
|
// Still in "starting" at timeout — accept it if it's at least running rather than roll back a slow boot.
|
|
return inspect.State?.Running ? { ready: true } : { ready: false, reason: 'health check timed out' }
|
|
}
|
|
|
|
/**
|
|
* Recreate a custom app's container from its (already-updated) Service record, preserving data.
|
|
* Uses the same rename-and-rollback safety net as the update flow: the live container is renamed
|
|
* aside, a new one is created from the new config/image, health-gated, and only then is the old one
|
|
* removed — otherwise we roll back to it. Bind-mounted data is untouched throughout. Pass
|
|
* `forcePull` to always re-pull the image first (used by the "update" action for moving tags).
|
|
*/
|
|
async recreateCustomAppContainer(
|
|
serviceName: string,
|
|
opts: { forcePull?: boolean } = {}
|
|
): Promise<{ success: boolean; message: string }> {
|
|
const service = await Service.query().where('service_name', serviceName).first()
|
|
if (!service) return { success: false, message: `Service ${serviceName} not found` }
|
|
|
|
const containerConfig = this._parseContainerConfig(service.container_config)
|
|
const oldInfo = await this._findContainerByName(serviceName)
|
|
const oldName = `${serviceName}_old`
|
|
|
|
// Clear any stale `_old` left behind by a previous recreate that died mid-flight. Without this,
|
|
// the rename below would fail (name in use) and the rollback path would then destroy the live
|
|
// container and resurrect the stale one in its place.
|
|
const staleOld = await this._findContainerByName(oldName)
|
|
if (staleOld) {
|
|
await this.docker.getContainer(staleOld.Id).remove({ force: true }).catch(() => {})
|
|
}
|
|
|
|
try {
|
|
// Stop + rename the existing container aside as a rollback safety net.
|
|
if (oldInfo) {
|
|
const oldContainer = this.docker.getContainer(oldInfo.Id)
|
|
if (oldInfo.State === 'running') await oldContainer.stop({ t: 10 }).catch(() => {})
|
|
await oldContainer.rename({ name: oldName })
|
|
}
|
|
|
|
// Pull the image if it's missing locally, or always when forcePull (e.g. :latest updates).
|
|
if (opts.forcePull || !(await this._checkImageExists(service.container_image))) {
|
|
const pullStream = await this.docker.pull(service.container_image)
|
|
await new Promise((res) => this.docker.modem.followProgress(pullStream, res))
|
|
}
|
|
|
|
const newContainer = await this.docker.createContainer({
|
|
Image: service.container_image,
|
|
name: serviceName,
|
|
Labels: {
|
|
...(containerConfig?.Labels ?? {}),
|
|
'com.docker.compose.project': 'project-nomad-managed',
|
|
'io.project-nomad.managed': 'true',
|
|
},
|
|
...(containerConfig?.User && { User: containerConfig.User }),
|
|
HostConfig: containerConfig?.HostConfig ?? {},
|
|
...(containerConfig?.ExposedPorts && { ExposedPorts: containerConfig.ExposedPorts }),
|
|
...(containerConfig?.Env && { Env: containerConfig.Env }),
|
|
...(service.container_command ? { Cmd: service.container_command.split(' ') } : {}),
|
|
...(process.env.NODE_ENV === 'production' && {
|
|
NetworkingConfig: { EndpointsConfig: { [DockerService.NOMAD_NETWORK]: {} } },
|
|
}),
|
|
})
|
|
await newContainer.start()
|
|
|
|
// Health gate before discarding the old container.
|
|
const readiness = await this._awaitContainerReady(newContainer)
|
|
if (!readiness.ready) throw new Error(`recreated container ${readiness.reason}`)
|
|
|
|
if (oldInfo) {
|
|
const oldRef = await this._findContainerByName(oldName)
|
|
if (oldRef) await this.docker.getContainer(oldRef.Id).remove({ force: true })
|
|
}
|
|
service.installed = true
|
|
service.installation_status = 'idle'
|
|
await service.save()
|
|
this.invalidateServicesStatusCache()
|
|
return { success: true, message: `Service ${serviceName} reconfigured successfully` }
|
|
} catch (error: any) {
|
|
logger.error({ err: error }, `[DockerService] recreateCustomAppContainer failed for ${serviceName}`)
|
|
// Roll back: discard the failed new container and restore the renamed original.
|
|
try {
|
|
const failedNew = await this._findContainerByName(serviceName)
|
|
if (failedNew) {
|
|
const c = this.docker.getContainer(failedNew.Id)
|
|
await c.stop({ t: 5 }).catch(() => {})
|
|
await c.remove({ force: true }).catch(() => {})
|
|
}
|
|
const renamed = await this._findContainerByName(oldName)
|
|
if (renamed) {
|
|
const c = this.docker.getContainer(renamed.Id)
|
|
await c.rename({ name: serviceName })
|
|
await c.start().catch(() => {})
|
|
}
|
|
} catch (rollbackError: any) {
|
|
logger.error({ err: rollbackError }, `[DockerService] rollback failed for ${serviceName}`)
|
|
}
|
|
this.invalidateServicesStatusCache()
|
|
return { success: false, message: `Reconfigure failed and was rolled back: ${error.message}` }
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check if a Docker image exists locally.
|
|
* @param imageName - The name and tag of the image (e.g., "nginx:latest")
|
|
* @returns - True if the image exists locally, false otherwise
|
|
*/
|
|
private async _checkImageExists(imageName: string): Promise<boolean> {
|
|
try {
|
|
const images = await this.docker.listImages()
|
|
|
|
// Check if any image has a RepoTag that matches the requested image
|
|
return images.some((image) => image.RepoTags && image.RepoTags.includes(imageName))
|
|
} catch (error: any) {
|
|
logger.warn(`Error checking if image exists: ${error.message}`)
|
|
// If run into an error, assume the image does not exist
|
|
return false
|
|
}
|
|
}
|
|
}
|