RomanNum3ral
/
project-nomad
mirror of https://github.com/Crosstalk-Solutions/project-nomad.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix(rag): add proper .docx text extraction via mammoth 

.docx files were classified as plain text and routed through raw-text
extraction (extractTXTText). Since .docx is a ZIP archive containing XML,
this produced garbage content in the Knowledge Base — XML tags and binary
noise instead of the actual document text.

Adds a dedicated 'docx' file type (split out of the generic 'text' bucket
in determineFileType) and a mammoth-based extractor that parses the
document XML properly.
This commit is contained in:
John Cortright 2026-07-13 18:22:34 -07:00
parent 6a4f02dd46
commit ff9695fda5
4 changed files with 140 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
admin/app/services/rag_service.ts
View File

@ -10,6 +10,7 @@ import { createWorker } from 'tesseract.js'
import { fromBuffer } from 'pdf2pic'
import JSZip from 'jszip'
import * as cheerio from 'cheerio'
import mammoth from 'mammoth'
import { OllamaService } from './ollama_service.js'
import { SERVICE_NAMES } from '../../constants/service_names.js'
import { removeStopwords } from 'stopword'
@ -611,6 +612,16 @@ export class RagService {
return await this.extractTXTText(fileBuffer)
}
/**
* Extract text content from a DOCX file using mammoth. DOCX is a ZIP-based
* XML format, so raw-text extraction (extractTXTText) would return garbage —
* this parses the document XML properly and returns clean plain text.
*/
private async processDocxFile(fileBuffer: Buffer): Promise<string> {
const { value: text } = await mammoth.extractRawText({ buffer: fileBuffer })
return text
}
/**
* Extract text content from an EPUB file.
* EPUBs are ZIP archives containing XHTML content files.
@ -765,6 +776,9 @@ export class RagService {
case 'pdf':
extractedText = await this.processPDFFile(fileBuffer!)
break
case 'docx':
extractedText = await this.processDocxFile(fileBuffer!)
break
case 'epub':
extractedText = await this.processEPUBFile(fileBuffer!)
break

6
admin/app/utils/fs.ts
View File

@ -190,13 +190,15 @@ export function matchesDevice(fsPath: string, deviceName: string): boolean {
return false
}
export function determineFileType(filename: string): 'image' | 'pdf' | 'text' | 'epub' | 'zim' | 'unknown' {
export function determineFileType(filename: string): 'image' | 'pdf' | 'text' | 'docx' | 'epub' | 'zim' | 'unknown' {
const ext = path.extname(filename).toLowerCase()
if (['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp'].includes(ext)) {
return 'image'
} else if (ext === '.pdf') {
return 'pdf'
} else if (['.txt', '.md', '.docx', '.rtf'].includes(ext)) {
} else if (ext === '.docx') {
return 'docx'
} else if (['.txt', '.md', '.rtf'].includes(ext)) {
return 'text'
} else if (ext === '.epub') {
return 'epub'

259
admin/package-lock.json generated
View File

@ -52,6 +52,7 @@
"ipaddr.js": "2.4.0",
"jszip": "3.10.1",
"luxon": "3.7.2",
"mammoth": "^1.12.0",
"maplibre-gl": "4.7.1",
"mysql2": "3.22.5",
"ollama": "0.6.3",
@ -2230,9 +2231,6 @@
"cpu": [
"arm"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@ -2249,9 +2247,6 @@
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@ -2268,9 +2263,6 @@
"cpu": [
"ppc64"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@ -2287,9 +2279,6 @@
"cpu": [
"riscv64"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@ -2306,9 +2295,6 @@
"cpu": [
"s390x"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@ -2325,9 +2311,6 @@
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@ -2344,9 +2327,6 @@
"cpu": [
"arm64"
],
"libc": [
"musl"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@ -2363,9 +2343,6 @@
"cpu": [
"x64"
],
"libc": [
"musl"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@ -2382,9 +2359,6 @@
"cpu": [
"arm"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@ -2407,9 +2381,6 @@
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@ -2432,9 +2403,6 @@
"cpu": [
"ppc64"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@ -2457,9 +2425,6 @@
"cpu": [
"riscv64"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@ -2482,9 +2447,6 @@
"cpu": [
"s390x"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@ -2507,9 +2469,6 @@
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@ -2532,9 +2491,6 @@
"cpu": [
"arm64"
],
"libc": [
"musl"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@ -2557,9 +2513,6 @@
"cpu": [
"x64"
],
"libc": [
"musl"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@ -3435,9 +3388,6 @@
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -3454,9 +3404,6 @@
"cpu": [
"arm64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@ -3473,9 +3420,6 @@
"cpu": [
"riscv64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -3492,9 +3436,6 @@
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -3511,9 +3452,6 @@
"cpu": [
"x64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@ -4199,9 +4137,6 @@
"cpu": [
"arm"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -4215,9 +4150,6 @@
"cpu": [
"arm"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@ -4231,9 +4163,6 @@
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -4247,9 +4176,6 @@
"cpu": [
"arm64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@ -4263,9 +4189,6 @@
"cpu": [
"loong64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -4279,9 +4202,6 @@
"cpu": [
"loong64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@ -4295,9 +4215,6 @@
"cpu": [
"ppc64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -4311,9 +4228,6 @@
"cpu": [
"ppc64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@ -4327,9 +4241,6 @@
"cpu": [
"riscv64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -4343,9 +4254,6 @@
"cpu": [
"riscv64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@ -4359,9 +4267,6 @@
"cpu": [
"s390x"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -4375,9 +4280,6 @@
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -4391,9 +4293,6 @@
"cpu": [
"x64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@ -4590,6 +4489,7 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4606,6 +4506,7 @@
"cpu": [
"x64"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4622,6 +4523,7 @@
"cpu": [
"arm"
],
"dev": true,
"license": "Apache-2.0",
"optional": true,
"os": [
@ -4638,9 +4540,7 @@
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4657,9 +4557,7 @@
"cpu": [
"arm64"
],
"libc": [
"musl"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4676,9 +4574,7 @@
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4695,9 +4591,7 @@
"cpu": [
"x64"
],
"libc": [
"musl"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4714,6 +4608,7 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4730,6 +4625,7 @@
"cpu": [
"ia32"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4746,6 +4642,7 @@
"cpu": [
"x64"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4938,9 +4835,6 @@
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -4957,9 +4851,6 @@
"cpu": [
"arm64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@ -4976,9 +4867,6 @@
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -4995,9 +4883,6 @@
"cpu": [
"x64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@ -6347,6 +6232,15 @@
"vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0"
}
},
"node_modules/@xmldom/xmldom": {
"version": "0.8.13",
"resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.13.tgz",
"integrity": "sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==",
"license": "MIT",
"engines": {
"node": ">=10.0.0"
}
},
"node_modules/abbrev": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/abbrev/-/abbrev-3.0.1.tgz",
@ -6806,6 +6700,12 @@
"readable-stream": "^3.4.0"
}
},
"node_modules/bluebird": {
"version": "3.4.7",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz",
"integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==",
"license": "MIT"
},
"node_modules/bmp-js": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz",
@ -8069,6 +7969,12 @@
"node": ">=0.3.1"
}
},
"node_modules/dingbat-to-unicode": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dingbat-to-unicode/-/dingbat-to-unicode-1.0.1.tgz",
"integrity": "sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w==",
"license": "BSD-2-Clause"
},
"node_modules/dlv": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz",
@ -8175,6 +8081,15 @@
"url": "https://dotenvx.com"
}
},
"node_modules/duck": {
"version": "0.1.12",
"resolved": "https://registry.npmjs.org/duck/-/duck-0.1.12.tgz",
"integrity": "sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg==",
"license": "BSD",
"dependencies": {
"underscore": "^1.13.1"
}
},
"node_modules/dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
@ -11240,9 +11155,6 @@
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"license": "MPL-2.0",
"optional": true,
"os": [
@ -11263,9 +11175,6 @@
"cpu": [
"arm64"
],
"libc": [
"musl"
],
"license": "MPL-2.0",
"optional": true,
"os": [
@ -11286,9 +11195,6 @@
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"license": "MPL-2.0",
"optional": true,
"os": [
@ -11309,9 +11215,6 @@
"cpu": [
"x64"
],
"libc": [
"musl"
],
"license": "MPL-2.0",
"optional": true,
"os": [
@ -11491,6 +11394,17 @@
"loose-envify": "cli.js"
}
},
"node_modules/lop": {
"version": "0.4.2",
"resolved": "https://registry.npmjs.org/lop/-/lop-0.4.2.tgz",
"integrity": "sha512-RefILVDQ4DKoRZsJ4Pj22TxE3omDO47yFpkIBoDKzkqPRISs5U1cnAdg/5583YPkWPaLIYHOKRMQSvjFsO26cw==",
"license": "BSD-2-Clause",
"dependencies": {
"duck": "^0.1.12",
"option": "~0.2.1",
"underscore": "^1.13.1"
}
},
"node_modules/lru-cache": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
@ -11570,6 +11484,39 @@
"node": ">= 0.6"
}
},
"node_modules/mammoth": {
"version": "1.12.0",
"resolved": "https://registry.npmjs.org/mammoth/-/mammoth-1.12.0.tgz",
"integrity": "sha512-cwnK1RIcRdDMi2HRx2EXGYlxqIEh0Oo3bLhorgnsVJi2UkbX1+jKxuBNR9PC5+JaX7EkmJxFPmo6mjLpqShI2w==",
"license": "BSD-2-Clause",
"dependencies": {
"@xmldom/xmldom": "^0.8.6",
"argparse": "~1.0.3",
"base64-js": "^1.5.1",
"bluebird": "~3.4.0",
"dingbat-to-unicode": "^1.0.1",
"jszip": "^3.7.1",
"lop": "^0.4.2",
"path-is-absolute": "^1.0.0",
"underscore": "^1.13.1",
"xmlbuilder": "^10.0.0"
},
"bin": {
"mammoth": "bin/mammoth"
},
"engines": {
"node": ">=12.0.0"
}
},
"node_modules/mammoth/node_modules/argparse": {
"version": "1.0.10",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
"integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
"license": "MIT",
"dependencies": {
"sprintf-js": "~1.0.2"
}
},
"node_modules/maplibre-gl": {
"version": "4.7.1",
"resolved": "https://registry.npmjs.org/maplibre-gl/-/maplibre-gl-4.7.1.tgz",
@ -13284,6 +13231,12 @@
"opencollective-postinstall": "index.js"
}
},
"node_modules/option": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/option/-/option-0.2.4.tgz",
"integrity": "sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==",
"license": "BSD-2-Clause"
},
"node_modules/optionator": {
"version": "0.9.4",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz",
@ -13628,6 +13581,15 @@
"node": ">=14.0.0"
}
},
"node_modules/path-is-absolute": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
"integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/path-key": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
@ -15641,6 +15603,12 @@
"node": ">= 10.x"
}
},
"node_modules/sprintf-js": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
"integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==",
"license": "BSD-3-Clause"
},
"node_modules/sql-escaper": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/sql-escaper/-/sql-escaper-1.3.3.tgz",
@ -16627,6 +16595,12 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/underscore": {
"version": "1.13.8",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.8.tgz",
"integrity": "sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==",
"license": "MIT"
},
"node_modules/undici": {
"version": "6.26.0",
"resolved": "https://registry.npmjs.org/undici/-/undici-6.26.0.tgz",
@ -17258,6 +17232,15 @@
"node": ">=16.0.0"
}
},
"node_modules/xmlbuilder": {
"version": "10.1.1",
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-10.1.1.tgz",
"integrity": "sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==",
"license": "MIT",
"engines": {
"node": ">=4.0"
}
},
"node_modules/y18n": {
"version": "5.0.8",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",

1
admin/package.json
View File

@ -105,6 +105,7 @@
"ipaddr.js": "2.4.0",
"jszip": "3.10.1",
"luxon": "3.7.2",
"mammoth": "^1.12.0",
"maplibre-gl": "4.7.1",
"mysql2": "3.22.5",
"ollama": "0.6.3",