RomanNum3ral
/
project-nomad
mirror of https://github.com/Crosstalk-Solutions/project-nomad.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge 7f989b917b into cb66faacbc

This commit is contained in:
just-jbc 2026-07-18 23:34:34 +02:00 committed by GitHub
parent cb66faacbc 7f989b917b
commit 84da627ae7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 130 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
admin/app/services/rag_service.ts
View File

@ -10,6 +10,7 @@ import { createWorker } from 'tesseract.js'
import { fromBuffer } from 'pdf2pic'
import JSZip from 'jszip'
import * as cheerio from 'cheerio'
import mammoth from 'mammoth'
import { OllamaService } from './ollama_service.js'
import { SERVICE_NAMES } from '../../constants/service_names.js'
import { removeStopwords } from 'stopword'
@ -611,6 +612,16 @@ export class RagService {
return await this.extractTXTText(fileBuffer)
}
/**
* Extract text content from a DOCX file using mammoth. DOCX is a ZIP-based
* XML format, so raw-text extraction (extractTXTText) would return garbage —
* this parses the document XML properly and returns clean plain text.
*/
private async processDocxFile(fileBuffer: Buffer): Promise<string> {
const { value: text } = await mammoth.extractRawText({ buffer: fileBuffer })
return text
}
/**
* Extract text content from an EPUB file.
* EPUBs are ZIP archives containing XHTML content files.
@ -765,6 +776,9 @@ export class RagService {
case 'pdf':
extractedText = await this.processPDFFile(fileBuffer!)
break
case 'docx':
extractedText = await this.processDocxFile(fileBuffer!)
break
case 'epub':
extractedText = await this.processEPUBFile(fileBuffer!)
break

6
admin/app/utils/fs.ts
View File

@ -190,13 +190,15 @@ export function matchesDevice(fsPath: string, deviceName: string): boolean {
return false
}
export function determineFileType(filename: string): 'image' | 'pdf' | 'text' | 'epub' | 'zim' | 'unknown' {
export function determineFileType(filename: string): 'image' | 'pdf' | 'text' | 'docx' | 'epub' | 'zim' | 'unknown' {
const ext = path.extname(filename).toLowerCase()
if (['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp'].includes(ext)) {
return 'image'
} else if (ext === '.pdf') {
return 'pdf'
} else if (['.txt', '.md', '.docx', '.rtf'].includes(ext)) {
} else if (ext === '.docx') {
return 'docx'
} else if (['.txt', '.md', '.rtf'].includes(ext)) {
return 'text'
} else if (ext === '.epub') {
return 'epub'

111
admin/package-lock.json generated
View File

@ -52,6 +52,7 @@
"ipaddr.js": "2.4.0",
"jszip": "3.10.1",
"luxon": "3.7.2",
"mammoth": "^1.12.0",
"maplibre-gl": "4.7.1",
"mysql2": "3.22.5",
"ollama": "0.6.3",
@ -6347,6 +6348,15 @@
"vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0"
}
},
"node_modules/@xmldom/xmldom": {
"version": "0.8.13",
"resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.13.tgz",
"integrity": "sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==",
"license": "MIT",
"engines": {
"node": ">=10.0.0"
}
},
"node_modules/abbrev": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/abbrev/-/abbrev-3.0.1.tgz",
@ -6806,6 +6816,12 @@
"readable-stream": "^3.4.0"
}
},
"node_modules/bluebird": {
"version": "3.4.7",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz",
"integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==",
"license": "MIT"
},
"node_modules/bmp-js": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz",
@ -8069,6 +8085,12 @@
"node": ">=0.3.1"
}
},
"node_modules/dingbat-to-unicode": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dingbat-to-unicode/-/dingbat-to-unicode-1.0.1.tgz",
"integrity": "sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w==",
"license": "BSD-2-Clause"
},
"node_modules/dlv": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz",
@ -8175,6 +8197,15 @@
"url": "https://dotenvx.com"
}
},
"node_modules/duck": {
"version": "0.1.12",
"resolved": "https://registry.npmjs.org/duck/-/duck-0.1.12.tgz",
"integrity": "sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg==",
"license": "BSD",
"dependencies": {
"underscore": "^1.13.1"
}
},
"node_modules/dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
@ -11491,6 +11522,17 @@
"loose-envify": "cli.js"
}
},
"node_modules/lop": {
"version": "0.4.2",
"resolved": "https://registry.npmjs.org/lop/-/lop-0.4.2.tgz",
"integrity": "sha512-RefILVDQ4DKoRZsJ4Pj22TxE3omDO47yFpkIBoDKzkqPRISs5U1cnAdg/5583YPkWPaLIYHOKRMQSvjFsO26cw==",
"license": "BSD-2-Clause",
"dependencies": {
"duck": "^0.1.12",
"option": "~0.2.1",
"underscore": "^1.13.1"
}
},
"node_modules/lru-cache": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
@ -11570,6 +11612,39 @@
"node": ">= 0.6"
}
},
"node_modules/mammoth": {
"version": "1.12.0",
"resolved": "https://registry.npmjs.org/mammoth/-/mammoth-1.12.0.tgz",
"integrity": "sha512-cwnK1RIcRdDMi2HRx2EXGYlxqIEh0Oo3bLhorgnsVJi2UkbX1+jKxuBNR9PC5+JaX7EkmJxFPmo6mjLpqShI2w==",
"license": "BSD-2-Clause",
"dependencies": {
"@xmldom/xmldom": "^0.8.6",
"argparse": "~1.0.3",
"base64-js": "^1.5.1",
"bluebird": "~3.4.0",
"dingbat-to-unicode": "^1.0.1",
"jszip": "^3.7.1",
"lop": "^0.4.2",
"path-is-absolute": "^1.0.0",
"underscore": "^1.13.1",
"xmlbuilder": "^10.0.0"
},
"bin": {
"mammoth": "bin/mammoth"
},
"engines": {
"node": ">=12.0.0"
}
},
"node_modules/mammoth/node_modules/argparse": {
"version": "1.0.10",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
"integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
"license": "MIT",
"dependencies": {
"sprintf-js": "~1.0.2"
}
},
"node_modules/maplibre-gl": {
"version": "4.7.1",
"resolved": "https://registry.npmjs.org/maplibre-gl/-/maplibre-gl-4.7.1.tgz",
@ -13284,6 +13359,12 @@
"opencollective-postinstall": "index.js"
}
},
"node_modules/option": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/option/-/option-0.2.4.tgz",
"integrity": "sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==",
"license": "BSD-2-Clause"
},
"node_modules/optionator": {
"version": "0.9.4",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz",
@ -13628,6 +13709,15 @@
"node": ">=14.0.0"
}
},
"node_modules/path-is-absolute": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
"integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/path-key": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
@ -15641,6 +15731,12 @@
"node": ">= 10.x"
}
},
"node_modules/sprintf-js": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
"integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==",
"license": "BSD-3-Clause"
},
"node_modules/sql-escaper": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/sql-escaper/-/sql-escaper-1.3.3.tgz",
@ -16627,6 +16723,12 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/underscore": {
"version": "1.13.8",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.8.tgz",
"integrity": "sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==",
"license": "MIT"
},
"node_modules/undici": {
"version": "6.26.0",
"resolved": "https://registry.npmjs.org/undici/-/undici-6.26.0.tgz",
@ -17258,6 +17360,15 @@
"node": ">=16.0.0"
}
},
"node_modules/xmlbuilder": {
"version": "10.1.1",
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-10.1.1.tgz",
"integrity": "sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==",
"license": "MIT",
"engines": {
"node": ">=4.0"
}
},
"node_modules/y18n": {
"version": "5.0.8",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",

1
admin/package.json
View File

@ -105,6 +105,7 @@
"ipaddr.js": "2.4.0",
"jszip": "3.10.1",
"luxon": "3.7.2",
"mammoth": "^1.12.0",
"maplibre-gl": "4.7.1",
"mysql2": "3.22.5",
"ollama": "0.6.3",