diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 29b65cf..45f660c 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -10,6 +10,7 @@ import { createWorker } from 'tesseract.js' import { fromBuffer } from 'pdf2pic' import JSZip from 'jszip' import * as cheerio from 'cheerio' +import mammoth from 'mammoth' import { OllamaService } from './ollama_service.js' import { SERVICE_NAMES } from '../../constants/service_names.js' import { removeStopwords } from 'stopword' @@ -611,6 +612,16 @@ export class RagService { return await this.extractTXTText(fileBuffer) } + /** + * Extract text content from a DOCX file using mammoth. DOCX is a ZIP-based + * XML format, so raw-text extraction (extractTXTText) would return garbage — + * this parses the document XML properly and returns clean plain text. + */ + private async processDocxFile(fileBuffer: Buffer): Promise { + const { value: text } = await mammoth.extractRawText({ buffer: fileBuffer }) + return text + } + /** * Extract text content from an EPUB file. * EPUBs are ZIP archives containing XHTML content files. @@ -765,6 +776,9 @@ export class RagService { case 'pdf': extractedText = await this.processPDFFile(fileBuffer!) break + case 'docx': + extractedText = await this.processDocxFile(fileBuffer!) + break case 'epub': extractedText = await this.processEPUBFile(fileBuffer!) break diff --git a/admin/app/utils/fs.ts b/admin/app/utils/fs.ts index 69b5463..6711ebc 100644 --- a/admin/app/utils/fs.ts +++ b/admin/app/utils/fs.ts @@ -190,13 +190,15 @@ export function matchesDevice(fsPath: string, deviceName: string): boolean { return false } -export function determineFileType(filename: string): 'image' | 'pdf' | 'text' | 'epub' | 'zim' | 'unknown' { +export function determineFileType(filename: string): 'image' | 'pdf' | 'text' | 'docx' | 'epub' | 'zim' | 'unknown' { const ext = path.extname(filename).toLowerCase() if (['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp'].includes(ext)) { return 'image' } else if (ext === '.pdf') { return 'pdf' - } else if (['.txt', '.md', '.docx', '.rtf'].includes(ext)) { + } else if (ext === '.docx') { + return 'docx' + } else if (['.txt', '.md', '.rtf'].includes(ext)) { return 'text' } else if (ext === '.epub') { return 'epub' diff --git a/admin/package-lock.json b/admin/package-lock.json index 0a18db6..ccaa795 100644 --- a/admin/package-lock.json +++ b/admin/package-lock.json @@ -52,6 +52,7 @@ "ipaddr.js": "2.4.0", "jszip": "3.10.1", "luxon": "3.7.2", + "mammoth": "^1.12.0", "maplibre-gl": "4.7.1", "mysql2": "3.22.5", "ollama": "0.6.3", @@ -2230,9 +2231,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2249,9 +2247,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2268,9 +2263,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2287,9 +2279,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2306,9 +2295,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2325,9 +2311,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2344,9 +2327,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2363,9 +2343,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2382,9 +2359,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2407,9 +2381,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2432,9 +2403,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2457,9 +2425,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2482,9 +2447,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2507,9 +2469,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2532,9 +2491,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2557,9 +2513,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3435,9 +3388,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3454,9 +3404,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3473,9 +3420,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3492,9 +3436,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3511,9 +3452,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4199,9 +4137,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4215,9 +4150,6 @@ "cpu": [ "arm" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4231,9 +4163,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4247,9 +4176,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4263,9 +4189,6 @@ "cpu": [ "loong64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4279,9 +4202,6 @@ "cpu": [ "loong64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4295,9 +4215,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4311,9 +4228,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4327,9 +4241,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4343,9 +4254,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4359,9 +4267,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4375,9 +4280,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4391,9 +4293,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4590,6 +4489,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -4606,6 +4506,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -4622,6 +4523,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -4638,9 +4540,7 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -4657,9 +4557,7 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -4676,9 +4574,7 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -4695,9 +4591,7 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -4714,6 +4608,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -4730,6 +4625,7 @@ "cpu": [ "ia32" ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -4746,6 +4642,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -4938,9 +4835,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4957,9 +4851,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4976,9 +4867,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4995,9 +4883,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -6347,6 +6232,15 @@ "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" } }, + "node_modules/@xmldom/xmldom": { + "version": "0.8.13", + "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.13.tgz", + "integrity": "sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/abbrev": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-3.0.1.tgz", @@ -6806,6 +6700,12 @@ "readable-stream": "^3.4.0" } }, + "node_modules/bluebird": { + "version": "3.4.7", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", + "integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==", + "license": "MIT" + }, "node_modules/bmp-js": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", @@ -8069,6 +7969,12 @@ "node": ">=0.3.1" } }, + "node_modules/dingbat-to-unicode": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dingbat-to-unicode/-/dingbat-to-unicode-1.0.1.tgz", + "integrity": "sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w==", + "license": "BSD-2-Clause" + }, "node_modules/dlv": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", @@ -8175,6 +8081,15 @@ "url": "https://dotenvx.com" } }, + "node_modules/duck": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/duck/-/duck-0.1.12.tgz", + "integrity": "sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg==", + "license": "BSD", + "dependencies": { + "underscore": "^1.13.1" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -11240,9 +11155,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -11263,9 +11175,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -11286,9 +11195,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -11309,9 +11215,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -11491,6 +11394,17 @@ "loose-envify": "cli.js" } }, + "node_modules/lop": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/lop/-/lop-0.4.2.tgz", + "integrity": "sha512-RefILVDQ4DKoRZsJ4Pj22TxE3omDO47yFpkIBoDKzkqPRISs5U1cnAdg/5583YPkWPaLIYHOKRMQSvjFsO26cw==", + "license": "BSD-2-Clause", + "dependencies": { + "duck": "^0.1.12", + "option": "~0.2.1", + "underscore": "^1.13.1" + } + }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -11570,6 +11484,39 @@ "node": ">= 0.6" } }, + "node_modules/mammoth": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/mammoth/-/mammoth-1.12.0.tgz", + "integrity": "sha512-cwnK1RIcRdDMi2HRx2EXGYlxqIEh0Oo3bLhorgnsVJi2UkbX1+jKxuBNR9PC5+JaX7EkmJxFPmo6mjLpqShI2w==", + "license": "BSD-2-Clause", + "dependencies": { + "@xmldom/xmldom": "^0.8.6", + "argparse": "~1.0.3", + "base64-js": "^1.5.1", + "bluebird": "~3.4.0", + "dingbat-to-unicode": "^1.0.1", + "jszip": "^3.7.1", + "lop": "^0.4.2", + "path-is-absolute": "^1.0.0", + "underscore": "^1.13.1", + "xmlbuilder": "^10.0.0" + }, + "bin": { + "mammoth": "bin/mammoth" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/mammoth/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, "node_modules/maplibre-gl": { "version": "4.7.1", "resolved": "https://registry.npmjs.org/maplibre-gl/-/maplibre-gl-4.7.1.tgz", @@ -13284,6 +13231,12 @@ "opencollective-postinstall": "index.js" } }, + "node_modules/option": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/option/-/option-0.2.4.tgz", + "integrity": "sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==", + "license": "BSD-2-Clause" + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -13628,6 +13581,15 @@ "node": ">=14.0.0" } }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -15641,6 +15603,12 @@ "node": ">= 10.x" } }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "license": "BSD-3-Clause" + }, "node_modules/sql-escaper": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/sql-escaper/-/sql-escaper-1.3.3.tgz", @@ -16627,6 +16595,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/underscore": { + "version": "1.13.8", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.8.tgz", + "integrity": "sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==", + "license": "MIT" + }, "node_modules/undici": { "version": "6.26.0", "resolved": "https://registry.npmjs.org/undici/-/undici-6.26.0.tgz", @@ -17258,6 +17232,15 @@ "node": ">=16.0.0" } }, + "node_modules/xmlbuilder": { + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-10.1.1.tgz", + "integrity": "sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==", + "license": "MIT", + "engines": { + "node": ">=4.0" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/admin/package.json b/admin/package.json index 18254d8..fb960a6 100644 --- a/admin/package.json +++ b/admin/package.json @@ -105,6 +105,7 @@ "ipaddr.js": "2.4.0", "jszip": "3.10.1", "luxon": "3.7.2", + "mammoth": "^1.12.0", "maplibre-gl": "4.7.1", "mysql2": "3.22.5", "ollama": "0.6.3",