chore(rag): remove .docx fix from this branch, split into #1100
Per review feedback, the .docx extraction fix is unrelated to the collections feature and can merge independently. Moved to a standalone PR (Crosstalk-Solutions/project-nomad#1100) off dev.
This commit is contained in:
John Cortright
parent
b93ec297e4
commit
e88b3e2252
|
|
@ -10,7 +10,6 @@ import { createWorker } from 'tesseract.js'
|
|||
import { fromBuffer } from 'pdf2pic'
|
||||
import JSZip from 'jszip'
|
||||
import * as cheerio from 'cheerio'
|
||||
import mammoth from 'mammoth'
|
||||
import { OllamaService } from './ollama_service.js'
|
||||
import { SERVICE_NAMES } from '../../constants/service_names.js'
|
||||
import { removeStopwords } from 'stopword'
|
||||
|
|
@ -616,16 +615,6 @@ export class RagService {
|
|||
return await this.extractTXTText(fileBuffer)
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract text content from a DOCX file using mammoth. DOCX is a ZIP-based
|
||||
* XML format, so raw-text extraction (extractTXTText) would return garbage —
|
||||
* this parses the document XML properly and returns clean plain text.
|
||||
*/
|
||||
private async processDocxFile(fileBuffer: Buffer): Promise<string> {
|
||||
const { value: text } = await mammoth.extractRawText({ buffer: fileBuffer })
|
||||
return text
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract text content from an EPUB file.
|
||||
* EPUBs are ZIP archives containing XHTML content files.
|
||||
|
|
@ -783,9 +772,6 @@ export class RagService {
|
|||
case 'pdf':
|
||||
extractedText = await this.processPDFFile(fileBuffer!)
|
||||
break
|
||||
case 'docx':
|
||||
extractedText = await this.processDocxFile(fileBuffer!)
|
||||
break
|
||||
case 'epub':
|
||||
extractedText = await this.processEPUBFile(fileBuffer!)
|
||||
break
|
||||
|
|
|
|||
|
|
@ -190,15 +190,13 @@ export function matchesDevice(fsPath: string, deviceName: string): boolean {
|
|||
return false
|
||||
}
|
||||
|
||||
export function determineFileType(filename: string): 'image' | 'pdf' | 'text' | 'docx' | 'epub' | 'zim' | 'unknown' {
|
||||
export function determineFileType(filename: string): 'image' | 'pdf' | 'text' | 'epub' | 'zim' | 'unknown' {
|
||||
const ext = path.extname(filename).toLowerCase()
|
||||
if (['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp'].includes(ext)) {
|
||||
return 'image'
|
||||
} else if (ext === '.pdf') {
|
||||
return 'pdf'
|
||||
} else if (ext === '.docx') {
|
||||
return 'docx'
|
||||
} else if (['.txt', '.md', '.rtf'].includes(ext)) {
|
||||
} else if (['.txt', '.md', '.docx', '.rtf'].includes(ext)) {
|
||||
return 'text'
|
||||
} else if (ext === '.epub') {
|
||||
return 'epub'
|
||||
|
|
|
|||
|
|
@ -52,7 +52,6 @@
|
|||
"ipaddr.js": "2.4.0",
|
||||
"jszip": "3.10.1",
|
||||
"luxon": "3.7.2",
|
||||
"mammoth": "^1.12.0",
|
||||
"maplibre-gl": "4.7.1",
|
||||
"mysql2": "3.22.5",
|
||||
"ollama": "0.6.3",
|
||||
|
|
@ -6232,15 +6231,6 @@
|
|||
"vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@xmldom/xmldom": {
|
||||
"version": "0.8.13",
|
||||
"resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.13.tgz",
|
||||
"integrity": "sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/abbrev": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/abbrev/-/abbrev-3.0.1.tgz",
|
||||
|
|
@ -6700,12 +6690,6 @@
|
|||
"readable-stream": "^3.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/bluebird": {
|
||||
"version": "3.4.7",
|
||||
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz",
|
||||
"integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/bmp-js": {
|
||||
"version": "0.1.0",
|
||||
"resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz",
|
||||
|
|
@ -7969,12 +7953,6 @@
|
|||
"node": ">=0.3.1"
|
||||
}
|
||||
},
|
||||
"node_modules/dingbat-to-unicode": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/dingbat-to-unicode/-/dingbat-to-unicode-1.0.1.tgz",
|
||||
"integrity": "sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w==",
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/dlv": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz",
|
||||
|
|
@ -8081,15 +8059,6 @@
|
|||
"url": "https://dotenvx.com"
|
||||
}
|
||||
},
|
||||
"node_modules/duck": {
|
||||
"version": "0.1.12",
|
||||
"resolved": "https://registry.npmjs.org/duck/-/duck-0.1.12.tgz",
|
||||
"integrity": "sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg==",
|
||||
"license": "BSD",
|
||||
"dependencies": {
|
||||
"underscore": "^1.13.1"
|
||||
}
|
||||
},
|
||||
"node_modules/dunder-proto": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
|
||||
|
|
@ -11394,17 +11363,6 @@
|
|||
"loose-envify": "cli.js"
|
||||
}
|
||||
},
|
||||
"node_modules/lop": {
|
||||
"version": "0.4.2",
|
||||
"resolved": "https://registry.npmjs.org/lop/-/lop-0.4.2.tgz",
|
||||
"integrity": "sha512-RefILVDQ4DKoRZsJ4Pj22TxE3omDO47yFpkIBoDKzkqPRISs5U1cnAdg/5583YPkWPaLIYHOKRMQSvjFsO26cw==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"duck": "^0.1.12",
|
||||
"option": "~0.2.1",
|
||||
"underscore": "^1.13.1"
|
||||
}
|
||||
},
|
||||
"node_modules/lru-cache": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
|
||||
|
|
@ -11484,39 +11442,6 @@
|
|||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/mammoth": {
|
||||
"version": "1.12.0",
|
||||
"resolved": "https://registry.npmjs.org/mammoth/-/mammoth-1.12.0.tgz",
|
||||
"integrity": "sha512-cwnK1RIcRdDMi2HRx2EXGYlxqIEh0Oo3bLhorgnsVJi2UkbX1+jKxuBNR9PC5+JaX7EkmJxFPmo6mjLpqShI2w==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"@xmldom/xmldom": "^0.8.6",
|
||||
"argparse": "~1.0.3",
|
||||
"base64-js": "^1.5.1",
|
||||
"bluebird": "~3.4.0",
|
||||
"dingbat-to-unicode": "^1.0.1",
|
||||
"jszip": "^3.7.1",
|
||||
"lop": "^0.4.2",
|
||||
"path-is-absolute": "^1.0.0",
|
||||
"underscore": "^1.13.1",
|
||||
"xmlbuilder": "^10.0.0"
|
||||
},
|
||||
"bin": {
|
||||
"mammoth": "bin/mammoth"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/mammoth/node_modules/argparse": {
|
||||
"version": "1.0.10",
|
||||
"resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
|
||||
"integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"sprintf-js": "~1.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/maplibre-gl": {
|
||||
"version": "4.7.1",
|
||||
"resolved": "https://registry.npmjs.org/maplibre-gl/-/maplibre-gl-4.7.1.tgz",
|
||||
|
|
@ -13231,12 +13156,6 @@
|
|||
"opencollective-postinstall": "index.js"
|
||||
}
|
||||
},
|
||||
"node_modules/option": {
|
||||
"version": "0.2.4",
|
||||
"resolved": "https://registry.npmjs.org/option/-/option-0.2.4.tgz",
|
||||
"integrity": "sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==",
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/optionator": {
|
||||
"version": "0.9.4",
|
||||
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz",
|
||||
|
|
@ -13581,15 +13500,6 @@
|
|||
"node": ">=14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/path-is-absolute": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
|
||||
"integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/path-key": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
|
||||
|
|
@ -15603,12 +15513,6 @@
|
|||
"node": ">= 10.x"
|
||||
}
|
||||
},
|
||||
"node_modules/sprintf-js": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
|
||||
"integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==",
|
||||
"license": "BSD-3-Clause"
|
||||
},
|
||||
"node_modules/sql-escaper": {
|
||||
"version": "1.3.3",
|
||||
"resolved": "https://registry.npmjs.org/sql-escaper/-/sql-escaper-1.3.3.tgz",
|
||||
|
|
@ -16595,12 +16499,6 @@
|
|||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/underscore": {
|
||||
"version": "1.13.8",
|
||||
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.8.tgz",
|
||||
"integrity": "sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/undici": {
|
||||
"version": "6.26.0",
|
||||
"resolved": "https://registry.npmjs.org/undici/-/undici-6.26.0.tgz",
|
||||
|
|
@ -17232,15 +17130,6 @@
|
|||
"node": ">=16.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/xmlbuilder": {
|
||||
"version": "10.1.1",
|
||||
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-10.1.1.tgz",
|
||||
"integrity": "sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/y18n": {
|
||||
"version": "5.0.8",
|
||||
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
|
||||
|
|
|
|||
|
|
@ -105,7 +105,6 @@
|
|||
"ipaddr.js": "2.4.0",
|
||||
"jszip": "3.10.1",
|
||||
"luxon": "3.7.2",
|
||||
"mammoth": "^1.12.0",
|
||||
"maplibre-gl": "4.7.1",
|
||||
"mysql2": "3.22.5",
|
||||
"ollama": "0.6.3",
|
||||
|
|
|
|||
Loading…
Reference in New Issue