chore(KB): filter non-content sections + render tables in ZIM extraction (#1044)
Closes #902. Two gaps in the structured ZIM extraction path: 1. NON_CONTENT_HEADING_PATTERNS was only used by the structure heuristic to count meaningful sections, never at section-emit time. Sections under "See also" / "References" / "External links" / etc. were still chunked and embedded. They're now flagged when the heading opens and dropped. 2. <table> elements were run through cheerio's `.text()`, concatenating every cell with no separators ("AgeDoseAdult500mg") into unsearchable word salad. New tableToText() joins cells with " | " and rows with newlines so row/column structure survives into the chunk. Refactor: moved extractStructuredContent out of ZIMExtractionService into a pure, cheerio-only util (app/utils/zim_html.ts) so it can be unit-tested without the native @openzim/libzim binding. Service delegates to it; behavior is otherwise unchanged. Adds tests/unit/zim_html.spec.ts (6 tests). Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
chriscrosstalk
GitHub
parent
59c2fdbb32
commit
8f56d76fe7
|
|
@ -1,6 +1,7 @@
|
|||
import { Archive, Entry } from '@openzim/libzim'
|
||||
import * as cheerio from 'cheerio'
|
||||
import { HTML_SELECTORS_TO_REMOVE, NON_CONTENT_HEADING_PATTERNS } from '../../constants/zim_extraction.js'
|
||||
import { extractStructuredContent } from '../utils/zim_html.js'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { ExtractZIMChunkingStrategy, ExtractZIMContentOptions, ZIMContentChunk, ZIMArchiveMetadata } from '../../types/zim.js'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
|
|
@ -115,7 +116,7 @@ export class ZIMExtractionService {
|
|||
let chunks: ZIMContentChunk[]
|
||||
|
||||
if (strategy === 'structured') {
|
||||
const structured = this.extractStructuredContent(html)
|
||||
const structured = extractStructuredContent(html)
|
||||
chunks = structured.sections.map(s => ({
|
||||
text: s.text,
|
||||
articleTitle,
|
||||
|
|
@ -210,76 +211,6 @@ export class ZIMExtractionService {
|
|||
}
|
||||
}
|
||||
|
||||
private extractStructuredContent(html: string) {
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
const title = $('h1').first().text().trim() || $('title').text().trim();
|
||||
|
||||
// Extract sections with their headings and heading levels
|
||||
const sections: Array<{ heading: string; text: string; level: number }> = [];
|
||||
let currentSection = { heading: 'Introduction', content: [] as string[], level: 2 };
|
||||
|
||||
// Walk the full DOM rather than only direct children of <body>. Modern ZIMs (Devdocs,
|
||||
// Wikipedia, FreeCodeCamp, etc.) wrap article content in a container div, which under
|
||||
// .children() would be a single non-heading/non-paragraph element and yield zero sections.
|
||||
$('body').find('h2, h3, h4, p, ul, ol, dl, table').each((_, element) => {
|
||||
const $el = $(element);
|
||||
const tagName = element.tagName?.toLowerCase();
|
||||
|
||||
if (['h2', 'h3', 'h4'].includes(tagName)) {
|
||||
// Save current section if it has content
|
||||
if (currentSection.content.length > 0) {
|
||||
sections.push({
|
||||
heading: currentSection.heading,
|
||||
text: currentSection.content.join(' ').replace(/\s+/g, ' ').trim(),
|
||||
level: currentSection.level,
|
||||
});
|
||||
}
|
||||
// Start new section
|
||||
const level = parseInt(tagName.substring(1)); // Extract number from h2, h3, h4
|
||||
currentSection = {
|
||||
heading: $el.text().replace(/\[edit\]/gi, '').trim(),
|
||||
content: [],
|
||||
level,
|
||||
};
|
||||
} else if (['p', 'ul', 'ol', 'dl', 'table'].includes(tagName)) {
|
||||
const text = $el.text().trim();
|
||||
if (text.length > 0) {
|
||||
currentSection.content.push(text);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Push the last section if it has content
|
||||
if (currentSection.content.length > 0) {
|
||||
sections.push({
|
||||
heading: currentSection.heading,
|
||||
text: currentSection.content.join(' ').replace(/\s+/g, ' ').trim(),
|
||||
level: currentSection.level,
|
||||
});
|
||||
}
|
||||
|
||||
// Fallback: if the selector walk produced no sections but the body has meaningful
|
||||
// text (unusual structure, minimal markup), emit one section with the full body text
|
||||
// so the article still contributes to the knowledge base.
|
||||
if (sections.length === 0) {
|
||||
const bodyText = $('body').text().replace(/\s+/g, ' ').trim();
|
||||
if (bodyText.length > 0) {
|
||||
sections.push({
|
||||
heading: title || 'Content',
|
||||
text: bodyText,
|
||||
level: 2,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
title,
|
||||
sections,
|
||||
fullText: sections.map(s => `${s.heading}\n${s.text}`).join('\n\n'),
|
||||
};
|
||||
}
|
||||
|
||||
private hasStructuredHeadings(html: string): boolean {
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,127 @@
|
|||
import * as cheerio from 'cheerio'
|
||||
import { NON_CONTENT_HEADING_PATTERNS } from '../../constants/zim_extraction.js'
|
||||
|
||||
export interface ZIMSection {
|
||||
heading: string
|
||||
text: string
|
||||
level: number
|
||||
}
|
||||
|
||||
export interface StructuredContent {
|
||||
title: string
|
||||
sections: ZIMSection[]
|
||||
fullText: string
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a section heading is one of the low-signal boilerplate headings
|
||||
* (See also / References / External links / etc.). Sections under these
|
||||
* headings are reference apparatus, not article content, and shouldn't reach
|
||||
* the embedder. (#902)
|
||||
*/
|
||||
export function isNonContentHeading(heading: string): boolean {
|
||||
return NON_CONTENT_HEADING_PATTERNS.some((pattern) => pattern.test(heading))
|
||||
}
|
||||
|
||||
/**
|
||||
* Render an HTML <table> into delimited text. cheerio's `.text()` concatenates
|
||||
* every cell with no separators ("AgeDoseAdult500mg" word salad), which is
|
||||
* unsearchable and pollutes embeddings. Instead, join cells with " | " and rows
|
||||
* with newlines so row/column structure survives into the chunk. (#902)
|
||||
*/
|
||||
export function tableToText($: cheerio.CheerioAPI, table: any): string {
|
||||
const rows: string[] = []
|
||||
$(table)
|
||||
.find('tr')
|
||||
.each((_, tr) => {
|
||||
const cells = $(tr)
|
||||
.find('th, td')
|
||||
.map((__, cell) => $(cell).text().replace(/\s+/g, ' ').trim())
|
||||
.get()
|
||||
.filter((cell) => cell.length > 0)
|
||||
if (cells.length > 0) {
|
||||
rows.push(cells.join(' | '))
|
||||
}
|
||||
})
|
||||
return rows.join('\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* Break a cleaned article's HTML into heading-delimited sections for chunking.
|
||||
* Skips non-content sections (References, See also, ...) at emit time and
|
||||
* renders tables as delimited text rather than concatenated cell soup. (#902)
|
||||
*/
|
||||
export function extractStructuredContent(html: string): StructuredContent {
|
||||
const $ = cheerio.load(html)
|
||||
|
||||
const title = $('h1').first().text().trim() || $('title').text().trim()
|
||||
|
||||
const sections: ZIMSection[] = []
|
||||
let currentSection = { heading: 'Introduction', content: [] as string[], level: 2, skip: false }
|
||||
|
||||
const flushSection = () => {
|
||||
if (!currentSection.skip && currentSection.content.length > 0) {
|
||||
sections.push({
|
||||
heading: currentSection.heading,
|
||||
text: currentSection.content.join(' ').replace(/\s+/g, ' ').trim(),
|
||||
level: currentSection.level,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Walk the full DOM rather than only direct children of <body>. Modern ZIMs (Devdocs,
|
||||
// Wikipedia, FreeCodeCamp, etc.) wrap article content in a container div, which under
|
||||
// .children() would be a single non-heading/non-paragraph element and yield zero sections.
|
||||
$('body')
|
||||
.find('h2, h3, h4, p, ul, ol, dl, table')
|
||||
.each((_, element) => {
|
||||
const $el = $(element)
|
||||
const tagName = element.tagName?.toLowerCase()
|
||||
|
||||
if (['h2', 'h3', 'h4'].includes(tagName)) {
|
||||
// Save the section we just finished, then open the next one.
|
||||
flushSection()
|
||||
const heading = $el
|
||||
.text()
|
||||
.replace(/\[edit\]/gi, '')
|
||||
.trim()
|
||||
const level = Number.parseInt(tagName.substring(1)) // Extract number from h2, h3, h4
|
||||
currentSection = {
|
||||
heading,
|
||||
content: [],
|
||||
level,
|
||||
skip: isNonContentHeading(heading),
|
||||
}
|
||||
} else if (['p', 'ul', 'ol', 'dl', 'table'].includes(tagName)) {
|
||||
// Don't bother collecting content for a section we're going to drop.
|
||||
if (currentSection.skip) return
|
||||
const text = tagName === 'table' ? tableToText($, element) : $el.text().trim()
|
||||
if (text.length > 0) {
|
||||
currentSection.content.push(text)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// Push the last section if it has content
|
||||
flushSection()
|
||||
|
||||
// Fallback: if the selector walk produced no sections but the body has meaningful
|
||||
// text (unusual structure, minimal markup), emit one section with the full body text
|
||||
// so the article still contributes to the knowledge base.
|
||||
if (sections.length === 0) {
|
||||
const bodyText = $('body').text().replace(/\s+/g, ' ').trim()
|
||||
if (bodyText.length > 0) {
|
||||
sections.push({
|
||||
heading: title || 'Content',
|
||||
text: bodyText,
|
||||
level: 2,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
title,
|
||||
sections,
|
||||
fullText: sections.map((s) => `${s.heading}\n${s.text}`).join('\n\n'),
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
import * as assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
|
||||
import {
|
||||
extractStructuredContent,
|
||||
isNonContentHeading,
|
||||
tableToText,
|
||||
} from '../../app/utils/zim_html.js'
|
||||
import * as cheerio from 'cheerio'
|
||||
|
||||
test('isNonContentHeading flags boilerplate headings, case-insensitive', () => {
|
||||
assert.equal(isNonContentHeading('References'), true)
|
||||
assert.equal(isNonContentHeading('see also'), true)
|
||||
assert.equal(isNonContentHeading('EXTERNAL LINKS'), true)
|
||||
assert.equal(isNonContentHeading('Further reading'), true)
|
||||
})
|
||||
|
||||
test('isNonContentHeading keeps real content headings', () => {
|
||||
assert.equal(isNonContentHeading('Uses'), false)
|
||||
assert.equal(isNonContentHeading('Side effects'), false)
|
||||
assert.equal(isNonContentHeading('History'), false)
|
||||
})
|
||||
|
||||
test('extractStructuredContent drops non-content sections at emit time (#902)', () => {
|
||||
const html = `<body>
|
||||
<h1>Aspirin</h1>
|
||||
<p>Aspirin is a medication used to reduce pain and fever.</p>
|
||||
<h2>Uses</h2>
|
||||
<p>It is used to treat fever, pain, and inflammation.</p>
|
||||
<h2>References</h2>
|
||||
<ul><li>Smith 2020</li><li>Jones 2019</li></ul>
|
||||
<h2>See also</h2>
|
||||
<p>Ibuprofen, Paracetamol</p>
|
||||
<h2>External links</h2>
|
||||
<p>https://example.com/aspirin</p>
|
||||
</body>`
|
||||
|
||||
const { sections } = extractStructuredContent(html)
|
||||
const headings = sections.map((s) => s.heading)
|
||||
|
||||
assert.deepEqual(headings, ['Introduction', 'Uses'])
|
||||
assert.equal(
|
||||
sections.some((s) => /Smith 2020|Ibuprofen|example\.com/.test(s.text)),
|
||||
false,
|
||||
'boilerplate-section content must not reach any emitted chunk'
|
||||
)
|
||||
})
|
||||
|
||||
test('extractStructuredContent keeps real sections including their content', () => {
|
||||
const html = `<body>
|
||||
<h1>Aspirin</h1>
|
||||
<p>Intro text here.</p>
|
||||
<h2>Side effects</h2>
|
||||
<p>May cause stomach upset.</p>
|
||||
</body>`
|
||||
|
||||
const { sections } = extractStructuredContent(html)
|
||||
const sideEffects = sections.find((s) => s.heading === 'Side effects')
|
||||
|
||||
assert.ok(sideEffects, 'real content section should be emitted')
|
||||
assert.match(sideEffects.text, /stomach upset/)
|
||||
})
|
||||
|
||||
test('tableToText renders rows as delimited text, not concatenated cell soup (#902)', () => {
|
||||
const html = `<table>
|
||||
<tr><th>Age</th><th>Dose</th></tr>
|
||||
<tr><td>Adult</td><td>500 mg</td></tr>
|
||||
<tr><td>Child</td><td>250 mg</td></tr>
|
||||
</table>`
|
||||
const $ = cheerio.load(html)
|
||||
const out = tableToText($, $('table').get(0))
|
||||
|
||||
assert.match(out, /Age \| Dose/)
|
||||
assert.match(out, /Adult \| 500 mg/)
|
||||
assert.match(out, /Child \| 250 mg/)
|
||||
assert.ok(out.includes('\n'), 'rows should be newline-separated')
|
||||
assert.equal(
|
||||
/AgeDose|Adult500/.test(out),
|
||||
false,
|
||||
'cells must not be concatenated without separators'
|
||||
)
|
||||
})
|
||||
|
||||
test('extractStructuredContent keeps table structure inside a content section', () => {
|
||||
const html = `<body>
|
||||
<h2>Dosage</h2>
|
||||
<table>
|
||||
<tr><th>Age</th><th>Dose</th></tr>
|
||||
<tr><td>Adult</td><td>500 mg</td></tr>
|
||||
</table>
|
||||
</body>`
|
||||
|
||||
const { sections } = extractStructuredContent(html)
|
||||
const dosage = sections.find((s) => s.heading === 'Dosage')
|
||||
|
||||
assert.ok(dosage, 'section with a table should be emitted')
|
||||
assert.match(dosage.text, /Age \| Dose/)
|
||||
assert.match(dosage.text, /Adult \| 500 mg/)
|
||||
})
|
||||
Loading…
Reference in New Issue