mirror of https://github.com/scrapy/scrapy.git
Remove deprecated match common prefix feature from IBL code
This commit is contained in:
Martin Olveyra
parent
633ebc4c43
commit
d9a3df45c6
|
|
@ -19,8 +19,7 @@ Main departures from the original algorithm:
|
|||
from operator import itemgetter
|
||||
from .regionextract import build_extraction_tree
|
||||
from .pageparsing import parse_template, parse_extraction_page
|
||||
from .pageobjects import TokenDict, AnnotationText
|
||||
from .similarity import common_prefix
|
||||
from .pageobjects import TokenDict
|
||||
|
||||
class InstanceBasedLearningExtractor(object):
|
||||
"""Implementation of the instance based learning algorithm to
|
||||
|
|
@ -50,35 +49,6 @@ class InstanceBasedLearningExtractor(object):
|
|||
in parsed_plus_templates if _annotation_count(p)]
|
||||
parsed_templates = map(itemgetter(0), parsed_plus_epages)
|
||||
|
||||
# calculate common text prefixes for annotations of same field across all templates
|
||||
extracted_text = {}
|
||||
extraction_pages = map(itemgetter(1), parsed_plus_epages)
|
||||
for i, parsed in enumerate(parsed_templates):
|
||||
for annot in parsed.annotations:
|
||||
if annot.match_common_prefix:
|
||||
field = annot.surrounds_attribute
|
||||
if field is not None:
|
||||
descriptor = type_descriptor.attribute_map.get(field) if type_descriptor else None
|
||||
allow_markup = descriptor.allow_markup if descriptor else False
|
||||
start, end = annot.start_index, annot.end_index
|
||||
if allow_markup:
|
||||
text = extraction_pages[i].html_between_tokens(start, end)
|
||||
else:
|
||||
text = extraction_pages[i].text_between_tokens(start, end)
|
||||
extracted_text.setdefault(field, []).append(text)
|
||||
common_prefixes = {}
|
||||
for field, data in extracted_text.iteritems():
|
||||
if len(data) > 1:
|
||||
cprefix = common_prefix(*data)
|
||||
if cprefix:
|
||||
common_prefixes[field] = "".join(cprefix).strip()
|
||||
# now apply common prefixes to annotations
|
||||
for i, parsed in enumerate(parsed_templates):
|
||||
for annot in parsed.annotations:
|
||||
for field, prefix in common_prefixes.iteritems():
|
||||
if annot.surrounds_attribute == field and not annot.annotation_text:
|
||||
annot.annotation_text = AnnotationText(prefix)
|
||||
|
||||
# templates with more attributes are considered first
|
||||
sorted_templates = sorted(parsed_templates, key=_annotation_count, reverse=True)
|
||||
self.extraction_trees = [build_extraction_tree(t, type_descriptor,
|
||||
|
|
|
|||
|
|
@ -189,23 +189,20 @@ class AnnotationTag(object):
|
|||
tag_attributes - list of (tag attribute, extracted attribute) tuples
|
||||
for each item to be extracted from a tag attribute
|
||||
annotation_text - text prefix and suffix for the attribute to be extracted
|
||||
match_common_prefix - use this annotation for calculating across-template prefixes
|
||||
metadata - dict with annotation data not used by IBL extractor
|
||||
"""
|
||||
__slots__ = ('surrounds_attribute', 'start_index', 'end_index',
|
||||
'tag_attributes', 'annotation_text', 'variant_id',
|
||||
'match_common_prefix', 'metadata')
|
||||
'metadata')
|
||||
|
||||
def __init__(self, start_index, end_index, surrounds_attribute=None,
|
||||
annotation_text=None, tag_attributes=None, variant_id=None,
|
||||
match_common_prefix=False):
|
||||
annotation_text=None, tag_attributes=None, variant_id=None):
|
||||
self.start_index = start_index
|
||||
self.end_index = end_index
|
||||
self.surrounds_attribute = surrounds_attribute
|
||||
self.annotation_text = annotation_text
|
||||
self.tag_attributes = tag_attributes or []
|
||||
self.variant_id = variant_id
|
||||
self.match_common_prefix = match_common_prefix
|
||||
self.metadata = {}
|
||||
|
||||
def __str__(self):
|
||||
|
|
|
|||
|
|
@ -133,8 +133,6 @@ class TemplatePageParser(InstanceLearningParser):
|
|||
else:
|
||||
annotation.tag_attributes.append((extract_attribute, tag_value))
|
||||
self.annotations.append(annotation)
|
||||
if jannotation.pop('common_prefix', False):
|
||||
annotation.match_common_prefix = True
|
||||
|
||||
self.extra_required_attrs.extend(jannotation.pop('required', []))
|
||||
annotation.metadata = jannotation
|
||||
|
|
@ -196,9 +194,6 @@ class TemplatePageParser(InstanceLearningParser):
|
|||
ignored = self.ignored_regions.pop()
|
||||
self.ignored_regions.append((ignored[0]-1, ignored[1]))
|
||||
|
||||
if jannotation.pop('common_prefix', False):
|
||||
annotation.match_common_prefix = True
|
||||
|
||||
self.extra_required_attrs.extend(jannotation.pop('required', []))
|
||||
|
||||
attribute_annotations = jannotation.pop('annotations', {}).items()
|
||||
|
|
|
|||
|
|
@ -156,11 +156,8 @@ class BasicTypeExtractor(object):
|
|||
"""Create a basic type extractor for the annotation"""
|
||||
text_region = annotation.annotation_text
|
||||
if text_region is not None:
|
||||
if annotation.match_common_prefix:
|
||||
region_extract = TextPrefixRegionDataExtractor(text_region.start_text).extract
|
||||
else:
|
||||
region_extract = TextRegionDataExtractor(text_region.start_text,
|
||||
text_region.follow_text).extract
|
||||
region_extract = TextRegionDataExtractor(text_region.start_text,
|
||||
text_region.follow_text).extract
|
||||
# copy attribute_descriptors and add the text extractor
|
||||
descriptor_copy = dict(attribute_descriptors)
|
||||
attr_descr = descriptor_copy.get(annotation.surrounds_attribute,
|
||||
|
|
@ -627,25 +624,4 @@ class TextRegionDataExtractor(object):
|
|||
return None
|
||||
return text[pref_index:pref_index + sidx]
|
||||
|
||||
class TextPrefixRegionDataExtractor(object):
|
||||
"""
|
||||
Data extractor for extracting text fragment from within a
|
||||
larger body of text, based on a fixed prefix.
|
||||
>>> extractor = TextPrefixRegionDataExtractor("£s;")
|
||||
>>> extractor.extract("£s; 17.00")
|
||||
' 17.00'
|
||||
>>> extractor.extract("€ 17.00") is None
|
||||
True
|
||||
>>> extractor.extract("$ 17.00") is None
|
||||
True
|
||||
>>> extractor.extract(" £s; 17.00 ")
|
||||
' 17.00 '
|
||||
"""
|
||||
def __init__(self, prefix):
|
||||
self.prefix = prefix
|
||||
def extract(self, text):
|
||||
text = text.lstrip()
|
||||
# attempt to extract a substring from the text
|
||||
if text.startswith(self.prefix):
|
||||
return text.replace(self.prefix, '')
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -238,70 +238,6 @@ EXTRACT_PAGE9 = u"""
|
|||
</body></html>
|
||||
"""
|
||||
|
||||
ANNOTATED_PAGE10a = u"""
|
||||
<html><body>
|
||||
<table><tbody>
|
||||
<tr data-scrapy-annotate="{"variant": 0, "common_prefix": true, "annotations": {"content": "site_id"}}">
|
||||
<td>SKU</td><td>L345</td>
|
||||
</tr>
|
||||
<tr data-scrapy-annotate="{"variant": 0, "common_prefix": true, "annotations": {"content": "dimensions"}}">
|
||||
<td>Size</td><td>10cmx20cm</td>
|
||||
</tr>
|
||||
<tr data-scrapy-annotate="{"variant": 0, "common_prefix": true, "annotations": {"content": "price"}}">
|
||||
<td>Price</td><td>£s;99.00</td>
|
||||
</tr>
|
||||
</tbody></table>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
ANNOTATED_PAGE10b = u"""
|
||||
<html><body>
|
||||
<table><tbody>
|
||||
<tr data-scrapy-annotate="{"variant": 0, "common_prefix": true, "annotations": {"content": "site_id"}}">
|
||||
<td>SKU</td><td>S220</td>
|
||||
</tr>
|
||||
<tr data-scrapy-annotate="{"variant": 0, "common_prefix": true, "annotations": {"content": "dimensions"}}">
|
||||
<td>Size</td><td>20cmx20cm</td>
|
||||
</tr>
|
||||
<tr data-scrapy-annotate="{"variant": 0, "common_prefix": true, "annotations": {"content": "price"}}">
|
||||
<td>Price</td><td>£s;85.00</td>
|
||||
</tr>
|
||||
</tbody></table>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
EXTRACT_PAGE10a = u"""
|
||||
<html><body>
|
||||
<table><tbody>
|
||||
<tr>
|
||||
<td>Offer</td><td>From $2500.00</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Description</td><td>Electrorheological Cyborgs</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Series:</td><td>T2000</td>
|
||||
</tr>
|
||||
</tbody></table>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
EXTRACT_PAGE10b = u"""
|
||||
<html><body>
|
||||
<table><tbody>
|
||||
<tr>
|
||||
<td>SKU</td><td>K80</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Size</td><td>50cm</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Price</td><td>&euros;85.00</td>
|
||||
</tr>
|
||||
</tbody></table>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
ANNOTATED_PAGE11 = u"""
|
||||
<html><body>
|
||||
<p data-scrapy-annotate="{"variant": 0, "annotations": {"content": "description"}}">
|
||||
|
|
@ -868,35 +804,6 @@ TEST_DATA = [
|
|||
'price': [u'\n12.00\n(VAT exc.)'],
|
||||
}
|
||||
),
|
||||
# detection of common prefixes across templates, all templates marked
|
||||
(# wrong extraction
|
||||
'without_match_common_prefix', [ANNOTATED_PAGE10a], EXTRACT_PAGE10a, None,
|
||||
{
|
||||
'price': [u'\n Series: T2000 \n'],
|
||||
'dimensions': [u'\n Description Electrorheological Cyborgs \n'],
|
||||
'site_id': [u'\n Offer From $2500.00 \n'],
|
||||
}
|
||||
),
|
||||
(# right extraction
|
||||
'with_match_common_prefix', [ANNOTATED_PAGE10a, ANNOTATED_PAGE10b], EXTRACT_PAGE10a, None,
|
||||
{}
|
||||
),
|
||||
(# another example
|
||||
'match_common_prefix', [ANNOTATED_PAGE10a, ANNOTATED_PAGE10b], EXTRACT_PAGE10b, None,
|
||||
{
|
||||
'dimensions': [u'50cm'],
|
||||
'site_id': [u'K80'],
|
||||
}
|
||||
),
|
||||
(# common_prefix with allow_markup attribute
|
||||
'common_prefix_allow_markup', [ANNOTATED_PAGE10a, ANNOTATED_PAGE10b], EXTRACT_PAGE10b,
|
||||
ItemDescriptor('test', 'product test',
|
||||
[A('dimensions', "something about dimensions", allow_markup=True)]),
|
||||
{
|
||||
'dimensions': [u'50cm</td>'],
|
||||
'site_id': [u'K80'],
|
||||
}
|
||||
),
|
||||
(# special case with partial annotations
|
||||
'special_partial_annotation', [ANNOTATED_PAGE11], EXTRACT_PAGE11, None,
|
||||
{
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ LABELLED_PAGE1 = u"""
|
|||
This is such a nice item<br/>
|
||||
Everybody likes it.
|
||||
</p>
|
||||
<p data-scrapy-annotate="{"variant": 0, "common_prefix": true, "annotations": {"content": "price"}}"/>
|
||||
<p data-scrapy-annotate="{"variant": 0, "annotations": {"content": "price"}}"/>
|
||||
\xa310.00
|
||||
<br/>
|
||||
<p data-scrapy-annotate="{"variant": 0, "annotations": {"content": "short_description"}}">
|
||||
|
|
@ -220,19 +220,6 @@ class TestPageParsing(TestCase):
|
|||
self.assertEqual(len(lp.annotations), 5)
|
||||
self._validate_annotation(lp, lp.annotations[0],
|
||||
'name', '<h1>', '</h1>')
|
||||
self.assertEqual(lp.annotations[0].match_common_prefix, False)
|
||||
self._validate_annotation(lp, lp.annotations[1],
|
||||
'description', '<p>', '</p>')
|
||||
self.assertEqual(lp.annotations[1].match_common_prefix, False)
|
||||
self._validate_annotation(lp, lp.annotations[2],
|
||||
'price', '<p/>', '<p>')
|
||||
self.assertEqual(lp.annotations[2].match_common_prefix, True)
|
||||
self._validate_annotation(lp, lp.annotations[3],
|
||||
'short_description', '<p>', '<p>')
|
||||
self.assertEqual(lp.annotations[3].match_common_prefix, False)
|
||||
self._validate_annotation(lp, lp.annotations[4],
|
||||
'short_description', '<p>', '<p>')
|
||||
self.assertEqual(lp.annotations[4].match_common_prefix, False)
|
||||
|
||||
# all tags were closed
|
||||
self.assertEqual(len(lp.labelled_tag_stacks), 0)
|
||||
|
|
|
|||
Loading…
Reference in New Issue