diff --git a/admin/app/services/zim_extraction_service.ts b/admin/app/services/zim_extraction_service.ts index 196add9..072098c 100644 --- a/admin/app/services/zim_extraction_service.ts +++ b/admin/app/services/zim_extraction_service.ts @@ -1,6 +1,7 @@ import { Archive, Entry } from '@openzim/libzim' import * as cheerio from 'cheerio' import { HTML_SELECTORS_TO_REMOVE, NON_CONTENT_HEADING_PATTERNS } from '../../constants/zim_extraction.js' +import { extractStructuredContent } from '../utils/zim_html.js' import logger from '@adonisjs/core/services/logger' import { ExtractZIMChunkingStrategy, ExtractZIMContentOptions, ZIMContentChunk, ZIMArchiveMetadata } from '../../types/zim.js' import { randomUUID } from 'node:crypto' @@ -115,7 +116,7 @@ export class ZIMExtractionService { let chunks: ZIMContentChunk[] if (strategy === 'structured') { - const structured = this.extractStructuredContent(html) + const structured = extractStructuredContent(html) chunks = structured.sections.map(s => ({ text: s.text, articleTitle, @@ -210,76 +211,6 @@ export class ZIMExtractionService { } } - private extractStructuredContent(html: string) { - const $ = cheerio.load(html); - - const title = $('h1').first().text().trim() || $('title').text().trim(); - - // Extract sections with their headings and heading levels - const sections: Array<{ heading: string; text: string; level: number }> = []; - let currentSection = { heading: 'Introduction', content: [] as string[], level: 2 }; - - // Walk the full DOM rather than only direct children of . Modern ZIMs (Devdocs, - // Wikipedia, FreeCodeCamp, etc.) wrap article content in a container div, which under - // .children() would be a single non-heading/non-paragraph element and yield zero sections. - $('body').find('h2, h3, h4, p, ul, ol, dl, table').each((_, element) => { - const $el = $(element); - const tagName = element.tagName?.toLowerCase(); - - if (['h2', 'h3', 'h4'].includes(tagName)) { - // Save current section if it has content - if (currentSection.content.length > 0) { - sections.push({ - heading: currentSection.heading, - text: currentSection.content.join(' ').replace(/\s+/g, ' ').trim(), - level: currentSection.level, - }); - } - // Start new section - const level = parseInt(tagName.substring(1)); // Extract number from h2, h3, h4 - currentSection = { - heading: $el.text().replace(/\[edit\]/gi, '').trim(), - content: [], - level, - }; - } else if (['p', 'ul', 'ol', 'dl', 'table'].includes(tagName)) { - const text = $el.text().trim(); - if (text.length > 0) { - currentSection.content.push(text); - } - } - }); - - // Push the last section if it has content - if (currentSection.content.length > 0) { - sections.push({ - heading: currentSection.heading, - text: currentSection.content.join(' ').replace(/\s+/g, ' ').trim(), - level: currentSection.level, - }); - } - - // Fallback: if the selector walk produced no sections but the body has meaningful - // text (unusual structure, minimal markup), emit one section with the full body text - // so the article still contributes to the knowledge base. - if (sections.length === 0) { - const bodyText = $('body').text().replace(/\s+/g, ' ').trim(); - if (bodyText.length > 0) { - sections.push({ - heading: title || 'Content', - text: bodyText, - level: 2, - }); - } - } - - return { - title, - sections, - fullText: sections.map(s => `${s.heading}\n${s.text}`).join('\n\n'), - }; - } - private hasStructuredHeadings(html: string): boolean { const $ = cheerio.load(html); diff --git a/admin/app/utils/zim_html.ts b/admin/app/utils/zim_html.ts new file mode 100644 index 0000000..2a4bcfd --- /dev/null +++ b/admin/app/utils/zim_html.ts @@ -0,0 +1,127 @@ +import * as cheerio from 'cheerio' +import { NON_CONTENT_HEADING_PATTERNS } from '../../constants/zim_extraction.js' + +export interface ZIMSection { + heading: string + text: string + level: number +} + +export interface StructuredContent { + title: string + sections: ZIMSection[] + fullText: string +} + +/** + * True when a section heading is one of the low-signal boilerplate headings + * (See also / References / External links / etc.). Sections under these + * headings are reference apparatus, not article content, and shouldn't reach + * the embedder. (#902) + */ +export function isNonContentHeading(heading: string): boolean { + return NON_CONTENT_HEADING_PATTERNS.some((pattern) => pattern.test(heading)) +} + +/** + * Render an HTML into delimited text. cheerio's `.text()` concatenates + * every cell with no separators ("AgeDoseAdult500mg" word salad), which is + * unsearchable and pollutes embeddings. Instead, join cells with " | " and rows + * with newlines so row/column structure survives into the chunk. (#902) + */ +export function tableToText($: cheerio.CheerioAPI, table: any): string { + const rows: string[] = [] + $(table) + .find('tr') + .each((_, tr) => { + const cells = $(tr) + .find('th, td') + .map((__, cell) => $(cell).text().replace(/\s+/g, ' ').trim()) + .get() + .filter((cell) => cell.length > 0) + if (cells.length > 0) { + rows.push(cells.join(' | ')) + } + }) + return rows.join('\n') +} + +/** + * Break a cleaned article's HTML into heading-delimited sections for chunking. + * Skips non-content sections (References, See also, ...) at emit time and + * renders tables as delimited text rather than concatenated cell soup. (#902) + */ +export function extractStructuredContent(html: string): StructuredContent { + const $ = cheerio.load(html) + + const title = $('h1').first().text().trim() || $('title').text().trim() + + const sections: ZIMSection[] = [] + let currentSection = { heading: 'Introduction', content: [] as string[], level: 2, skip: false } + + const flushSection = () => { + if (!currentSection.skip && currentSection.content.length > 0) { + sections.push({ + heading: currentSection.heading, + text: currentSection.content.join(' ').replace(/\s+/g, ' ').trim(), + level: currentSection.level, + }) + } + } + + // Walk the full DOM rather than only direct children of . Modern ZIMs (Devdocs, + // Wikipedia, FreeCodeCamp, etc.) wrap article content in a container div, which under + // .children() would be a single non-heading/non-paragraph element and yield zero sections. + $('body') + .find('h2, h3, h4, p, ul, ol, dl, table') + .each((_, element) => { + const $el = $(element) + const tagName = element.tagName?.toLowerCase() + + if (['h2', 'h3', 'h4'].includes(tagName)) { + // Save the section we just finished, then open the next one. + flushSection() + const heading = $el + .text() + .replace(/\[edit\]/gi, '') + .trim() + const level = Number.parseInt(tagName.substring(1)) // Extract number from h2, h3, h4 + currentSection = { + heading, + content: [], + level, + skip: isNonContentHeading(heading), + } + } else if (['p', 'ul', 'ol', 'dl', 'table'].includes(tagName)) { + // Don't bother collecting content for a section we're going to drop. + if (currentSection.skip) return + const text = tagName === 'table' ? tableToText($, element) : $el.text().trim() + if (text.length > 0) { + currentSection.content.push(text) + } + } + }) + + // Push the last section if it has content + flushSection() + + // Fallback: if the selector walk produced no sections but the body has meaningful + // text (unusual structure, minimal markup), emit one section with the full body text + // so the article still contributes to the knowledge base. + if (sections.length === 0) { + const bodyText = $('body').text().replace(/\s+/g, ' ').trim() + if (bodyText.length > 0) { + sections.push({ + heading: title || 'Content', + text: bodyText, + level: 2, + }) + } + } + + return { + title, + sections, + fullText: sections.map((s) => `${s.heading}\n${s.text}`).join('\n\n'), + } +} diff --git a/admin/tests/unit/zim_html.spec.ts b/admin/tests/unit/zim_html.spec.ts new file mode 100644 index 0000000..e46b6cf --- /dev/null +++ b/admin/tests/unit/zim_html.spec.ts @@ -0,0 +1,99 @@ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { + extractStructuredContent, + isNonContentHeading, + tableToText, +} from '../../app/utils/zim_html.js' +import * as cheerio from 'cheerio' + +test('isNonContentHeading flags boilerplate headings, case-insensitive', () => { + assert.equal(isNonContentHeading('References'), true) + assert.equal(isNonContentHeading('see also'), true) + assert.equal(isNonContentHeading('EXTERNAL LINKS'), true) + assert.equal(isNonContentHeading('Further reading'), true) +}) + +test('isNonContentHeading keeps real content headings', () => { + assert.equal(isNonContentHeading('Uses'), false) + assert.equal(isNonContentHeading('Side effects'), false) + assert.equal(isNonContentHeading('History'), false) +}) + +test('extractStructuredContent drops non-content sections at emit time (#902)', () => { + const html = ` +

Aspirin

+

Aspirin is a medication used to reduce pain and fever.

+

Uses

+

It is used to treat fever, pain, and inflammation.

+

References

+ +

See also

+

Ibuprofen, Paracetamol

+

External links

+

https://example.com/aspirin

+ ` + + const { sections } = extractStructuredContent(html) + const headings = sections.map((s) => s.heading) + + assert.deepEqual(headings, ['Introduction', 'Uses']) + assert.equal( + sections.some((s) => /Smith 2020|Ibuprofen|example\.com/.test(s.text)), + false, + 'boilerplate-section content must not reach any emitted chunk' + ) +}) + +test('extractStructuredContent keeps real sections including their content', () => { + const html = ` +

Aspirin

+

Intro text here.

+

Side effects

+

May cause stomach upset.

+ ` + + const { sections } = extractStructuredContent(html) + const sideEffects = sections.find((s) => s.heading === 'Side effects') + + assert.ok(sideEffects, 'real content section should be emitted') + assert.match(sideEffects.text, /stomach upset/) +}) + +test('tableToText renders rows as delimited text, not concatenated cell soup (#902)', () => { + const html = `
+ + + +
AgeDose
Adult500 mg
Child250 mg
` + const $ = cheerio.load(html) + const out = tableToText($, $('table').get(0)) + + assert.match(out, /Age \| Dose/) + assert.match(out, /Adult \| 500 mg/) + assert.match(out, /Child \| 250 mg/) + assert.ok(out.includes('\n'), 'rows should be newline-separated') + assert.equal( + /AgeDose|Adult500/.test(out), + false, + 'cells must not be concatenated without separators' + ) +}) + +test('extractStructuredContent keeps table structure inside a content section', () => { + const html = ` +

Dosage

+ + + +
AgeDose
Adult500 mg
+ ` + + const { sections } = extractStructuredContent(html) + const dosage = sections.find((s) => s.heading === 'Dosage') + + assert.ok(dosage, 'section with a table should be emitted') + assert.match(dosage.text, /Age \| Dose/) + assert.match(dosage.text, /Adult \| 500 mg/) +})