diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 9c0fafb..3fc5493 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -10,7 +10,6 @@ import { createWorker } from 'tesseract.js' import { fromBuffer } from 'pdf2pic' import JSZip from 'jszip' import * as cheerio from 'cheerio' -import mammoth from 'mammoth' import { OllamaService } from './ollama_service.js' import { SERVICE_NAMES } from '../../constants/service_names.js' import { removeStopwords } from 'stopword' @@ -616,16 +615,6 @@ export class RagService { return await this.extractTXTText(fileBuffer) } - /** - * Extract text content from a DOCX file using mammoth. DOCX is a ZIP-based - * XML format, so raw-text extraction (extractTXTText) would return garbage — - * this parses the document XML properly and returns clean plain text. - */ - private async processDocxFile(fileBuffer: Buffer): Promise { - const { value: text } = await mammoth.extractRawText({ buffer: fileBuffer }) - return text - } - /** * Extract text content from an EPUB file. * EPUBs are ZIP archives containing XHTML content files. @@ -783,9 +772,6 @@ export class RagService { case 'pdf': extractedText = await this.processPDFFile(fileBuffer!) break - case 'docx': - extractedText = await this.processDocxFile(fileBuffer!) - break case 'epub': extractedText = await this.processEPUBFile(fileBuffer!) break diff --git a/admin/app/utils/fs.ts b/admin/app/utils/fs.ts index 6711ebc..69b5463 100644 --- a/admin/app/utils/fs.ts +++ b/admin/app/utils/fs.ts @@ -190,15 +190,13 @@ export function matchesDevice(fsPath: string, deviceName: string): boolean { return false } -export function determineFileType(filename: string): 'image' | 'pdf' | 'text' | 'docx' | 'epub' | 'zim' | 'unknown' { +export function determineFileType(filename: string): 'image' | 'pdf' | 'text' | 'epub' | 'zim' | 'unknown' { const ext = path.extname(filename).toLowerCase() if (['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp'].includes(ext)) { return 'image' } else if (ext === '.pdf') { return 'pdf' - } else if (ext === '.docx') { - return 'docx' - } else if (['.txt', '.md', '.rtf'].includes(ext)) { + } else if (['.txt', '.md', '.docx', '.rtf'].includes(ext)) { return 'text' } else if (ext === '.epub') { return 'epub' diff --git a/admin/package-lock.json b/admin/package-lock.json index ccaa795..2c08c3c 100644 --- a/admin/package-lock.json +++ b/admin/package-lock.json @@ -52,7 +52,6 @@ "ipaddr.js": "2.4.0", "jszip": "3.10.1", "luxon": "3.7.2", - "mammoth": "^1.12.0", "maplibre-gl": "4.7.1", "mysql2": "3.22.5", "ollama": "0.6.3", @@ -6232,15 +6231,6 @@ "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" } }, - "node_modules/@xmldom/xmldom": { - "version": "0.8.13", - "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.13.tgz", - "integrity": "sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==", - "license": "MIT", - "engines": { - "node": ">=10.0.0" - } - }, "node_modules/abbrev": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-3.0.1.tgz", @@ -6700,12 +6690,6 @@ "readable-stream": "^3.4.0" } }, - "node_modules/bluebird": { - "version": "3.4.7", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", - "integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==", - "license": "MIT" - }, "node_modules/bmp-js": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", @@ -7969,12 +7953,6 @@ "node": ">=0.3.1" } }, - "node_modules/dingbat-to-unicode": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dingbat-to-unicode/-/dingbat-to-unicode-1.0.1.tgz", - "integrity": "sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w==", - "license": "BSD-2-Clause" - }, "node_modules/dlv": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", @@ -8081,15 +8059,6 @@ "url": "https://dotenvx.com" } }, - "node_modules/duck": { - "version": "0.1.12", - "resolved": "https://registry.npmjs.org/duck/-/duck-0.1.12.tgz", - "integrity": "sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg==", - "license": "BSD", - "dependencies": { - "underscore": "^1.13.1" - } - }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -11394,17 +11363,6 @@ "loose-envify": "cli.js" } }, - "node_modules/lop": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/lop/-/lop-0.4.2.tgz", - "integrity": "sha512-RefILVDQ4DKoRZsJ4Pj22TxE3omDO47yFpkIBoDKzkqPRISs5U1cnAdg/5583YPkWPaLIYHOKRMQSvjFsO26cw==", - "license": "BSD-2-Clause", - "dependencies": { - "duck": "^0.1.12", - "option": "~0.2.1", - "underscore": "^1.13.1" - } - }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -11484,39 +11442,6 @@ "node": ">= 0.6" } }, - "node_modules/mammoth": { - "version": "1.12.0", - "resolved": "https://registry.npmjs.org/mammoth/-/mammoth-1.12.0.tgz", - "integrity": "sha512-cwnK1RIcRdDMi2HRx2EXGYlxqIEh0Oo3bLhorgnsVJi2UkbX1+jKxuBNR9PC5+JaX7EkmJxFPmo6mjLpqShI2w==", - "license": "BSD-2-Clause", - "dependencies": { - "@xmldom/xmldom": "^0.8.6", - "argparse": "~1.0.3", - "base64-js": "^1.5.1", - "bluebird": "~3.4.0", - "dingbat-to-unicode": "^1.0.1", - "jszip": "^3.7.1", - "lop": "^0.4.2", - "path-is-absolute": "^1.0.0", - "underscore": "^1.13.1", - "xmlbuilder": "^10.0.0" - }, - "bin": { - "mammoth": "bin/mammoth" - }, - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/mammoth/node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "license": "MIT", - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, "node_modules/maplibre-gl": { "version": "4.7.1", "resolved": "https://registry.npmjs.org/maplibre-gl/-/maplibre-gl-4.7.1.tgz", @@ -13231,12 +13156,6 @@ "opencollective-postinstall": "index.js" } }, - "node_modules/option": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/option/-/option-0.2.4.tgz", - "integrity": "sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==", - "license": "BSD-2-Clause" - }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -13581,15 +13500,6 @@ "node": ">=14.0.0" } }, - "node_modules/path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -15603,12 +15513,6 @@ "node": ">= 10.x" } }, - "node_modules/sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", - "license": "BSD-3-Clause" - }, "node_modules/sql-escaper": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/sql-escaper/-/sql-escaper-1.3.3.tgz", @@ -16595,12 +16499,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/underscore": { - "version": "1.13.8", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.8.tgz", - "integrity": "sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==", - "license": "MIT" - }, "node_modules/undici": { "version": "6.26.0", "resolved": "https://registry.npmjs.org/undici/-/undici-6.26.0.tgz", @@ -17232,15 +17130,6 @@ "node": ">=16.0.0" } }, - "node_modules/xmlbuilder": { - "version": "10.1.1", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-10.1.1.tgz", - "integrity": "sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==", - "license": "MIT", - "engines": { - "node": ">=4.0" - } - }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/admin/package.json b/admin/package.json index fb960a6..18254d8 100644 --- a/admin/package.json +++ b/admin/package.json @@ -105,7 +105,6 @@ "ipaddr.js": "2.4.0", "jszip": "3.10.1", "luxon": "3.7.2", - "mammoth": "^1.12.0", "maplibre-gl": "4.7.1", "mysql2": "3.22.5", "ollama": "0.6.3",