From d9a3df45c600224f75bc235fd663faf684a8ed0b Mon Sep 17 00:00:00 2001 From: Martin Olveyra Date: Thu, 23 Dec 2010 14:40:22 -0200 Subject: [PATCH] Remove deprecated match common prefix feature from IBL code --- scrapy/contrib/ibl/extraction/__init__.py | 32 +------ scrapy/contrib/ibl/extraction/pageobjects.py | 7 +- scrapy/contrib/ibl/extraction/pageparsing.py | 5 - .../contrib/ibl/extraction/regionextract.py | 30 +----- .../tests/test_contrib_ibl/test_extraction.py | 93 ------------------- .../test_contrib_ibl/test_pageparsing.py | 15 +-- 6 files changed, 7 insertions(+), 175 deletions(-) diff --git a/scrapy/contrib/ibl/extraction/__init__.py b/scrapy/contrib/ibl/extraction/__init__.py index 1c8fa1802..299fa31ae 100644 --- a/scrapy/contrib/ibl/extraction/__init__.py +++ b/scrapy/contrib/ibl/extraction/__init__.py @@ -19,8 +19,7 @@ Main departures from the original algorithm: from operator import itemgetter from .regionextract import build_extraction_tree from .pageparsing import parse_template, parse_extraction_page -from .pageobjects import TokenDict, AnnotationText -from .similarity import common_prefix +from .pageobjects import TokenDict class InstanceBasedLearningExtractor(object): """Implementation of the instance based learning algorithm to @@ -50,35 +49,6 @@ class InstanceBasedLearningExtractor(object): in parsed_plus_templates if _annotation_count(p)] parsed_templates = map(itemgetter(0), parsed_plus_epages) - # calculate common text prefixes for annotations of same field across all templates - extracted_text = {} - extraction_pages = map(itemgetter(1), parsed_plus_epages) - for i, parsed in enumerate(parsed_templates): - for annot in parsed.annotations: - if annot.match_common_prefix: - field = annot.surrounds_attribute - if field is not None: - descriptor = type_descriptor.attribute_map.get(field) if type_descriptor else None - allow_markup = descriptor.allow_markup if descriptor else False - start, end = annot.start_index, annot.end_index - if allow_markup: - text = extraction_pages[i].html_between_tokens(start, end) - else: - text = extraction_pages[i].text_between_tokens(start, end) - extracted_text.setdefault(field, []).append(text) - common_prefixes = {} - for field, data in extracted_text.iteritems(): - if len(data) > 1: - cprefix = common_prefix(*data) - if cprefix: - common_prefixes[field] = "".join(cprefix).strip() - # now apply common prefixes to annotations - for i, parsed in enumerate(parsed_templates): - for annot in parsed.annotations: - for field, prefix in common_prefixes.iteritems(): - if annot.surrounds_attribute == field and not annot.annotation_text: - annot.annotation_text = AnnotationText(prefix) - # templates with more attributes are considered first sorted_templates = sorted(parsed_templates, key=_annotation_count, reverse=True) self.extraction_trees = [build_extraction_tree(t, type_descriptor, diff --git a/scrapy/contrib/ibl/extraction/pageobjects.py b/scrapy/contrib/ibl/extraction/pageobjects.py index e73b4e699..fa290d3c7 100644 --- a/scrapy/contrib/ibl/extraction/pageobjects.py +++ b/scrapy/contrib/ibl/extraction/pageobjects.py @@ -189,23 +189,20 @@ class AnnotationTag(object): tag_attributes - list of (tag attribute, extracted attribute) tuples for each item to be extracted from a tag attribute annotation_text - text prefix and suffix for the attribute to be extracted - match_common_prefix - use this annotation for calculating across-template prefixes metadata - dict with annotation data not used by IBL extractor """ __slots__ = ('surrounds_attribute', 'start_index', 'end_index', 'tag_attributes', 'annotation_text', 'variant_id', - 'match_common_prefix', 'metadata') + 'metadata') def __init__(self, start_index, end_index, surrounds_attribute=None, - annotation_text=None, tag_attributes=None, variant_id=None, - match_common_prefix=False): + annotation_text=None, tag_attributes=None, variant_id=None): self.start_index = start_index self.end_index = end_index self.surrounds_attribute = surrounds_attribute self.annotation_text = annotation_text self.tag_attributes = tag_attributes or [] self.variant_id = variant_id - self.match_common_prefix = match_common_prefix self.metadata = {} def __str__(self): diff --git a/scrapy/contrib/ibl/extraction/pageparsing.py b/scrapy/contrib/ibl/extraction/pageparsing.py index b672ef502..2df63267f 100644 --- a/scrapy/contrib/ibl/extraction/pageparsing.py +++ b/scrapy/contrib/ibl/extraction/pageparsing.py @@ -133,8 +133,6 @@ class TemplatePageParser(InstanceLearningParser): else: annotation.tag_attributes.append((extract_attribute, tag_value)) self.annotations.append(annotation) - if jannotation.pop('common_prefix', False): - annotation.match_common_prefix = True self.extra_required_attrs.extend(jannotation.pop('required', [])) annotation.metadata = jannotation @@ -196,9 +194,6 @@ class TemplatePageParser(InstanceLearningParser): ignored = self.ignored_regions.pop() self.ignored_regions.append((ignored[0]-1, ignored[1])) - if jannotation.pop('common_prefix', False): - annotation.match_common_prefix = True - self.extra_required_attrs.extend(jannotation.pop('required', [])) attribute_annotations = jannotation.pop('annotations', {}).items() diff --git a/scrapy/contrib/ibl/extraction/regionextract.py b/scrapy/contrib/ibl/extraction/regionextract.py index 5fb368f5e..db5eb2a38 100644 --- a/scrapy/contrib/ibl/extraction/regionextract.py +++ b/scrapy/contrib/ibl/extraction/regionextract.py @@ -156,11 +156,8 @@ class BasicTypeExtractor(object): """Create a basic type extractor for the annotation""" text_region = annotation.annotation_text if text_region is not None: - if annotation.match_common_prefix: - region_extract = TextPrefixRegionDataExtractor(text_region.start_text).extract - else: - region_extract = TextRegionDataExtractor(text_region.start_text, - text_region.follow_text).extract + region_extract = TextRegionDataExtractor(text_region.start_text, + text_region.follow_text).extract # copy attribute_descriptors and add the text extractor descriptor_copy = dict(attribute_descriptors) attr_descr = descriptor_copy.get(annotation.surrounds_attribute, @@ -627,25 +624,4 @@ class TextRegionDataExtractor(object): return None return text[pref_index:pref_index + sidx] -class TextPrefixRegionDataExtractor(object): - """ - Data extractor for extracting text fragment from within a - larger body of text, based on a fixed prefix. - >>> extractor = TextPrefixRegionDataExtractor("£s;") - >>> extractor.extract("£s; 17.00") - ' 17.00' - >>> extractor.extract("€ 17.00") is None - True - >>> extractor.extract("$ 17.00") is None - True - >>> extractor.extract(" £s; 17.00 ") - ' 17.00 ' - """ - def __init__(self, prefix): - self.prefix = prefix - def extract(self, text): - text = text.lstrip() - # attempt to extract a substring from the text - if text.startswith(self.prefix): - return text.replace(self.prefix, '') - + diff --git a/scrapy/tests/test_contrib_ibl/test_extraction.py b/scrapy/tests/test_contrib_ibl/test_extraction.py index 0dabccd5b..f79c9dc78 100644 --- a/scrapy/tests/test_contrib_ibl/test_extraction.py +++ b/scrapy/tests/test_contrib_ibl/test_extraction.py @@ -238,70 +238,6 @@ EXTRACT_PAGE9 = u""" """ -ANNOTATED_PAGE10a = u""" - - - - - - - - - - - -
SKUL345
Size10cmx20cm
Price£s;99.00
- -""" - -ANNOTATED_PAGE10b = u""" - - - - - - - - - - - -
SKUS220
Size20cmx20cm
Price£s;85.00
- -""" - -EXTRACT_PAGE10a = u""" - - - - - - - - - - - -
OfferFrom $2500.00
DescriptionElectrorheological Cyborgs
Series:T2000
- -""" - -EXTRACT_PAGE10b = u""" - - - - - - - - - - - -
SKUK80
Size50cm
Price&euros;85.00
- -""" - ANNOTATED_PAGE11 = u"""

@@ -868,35 +804,6 @@ TEST_DATA = [ 'price': [u'\n12.00\n(VAT exc.)'], } ), - # detection of common prefixes across templates, all templates marked - (# wrong extraction - 'without_match_common_prefix', [ANNOTATED_PAGE10a], EXTRACT_PAGE10a, None, - { - 'price': [u'\n Series: T2000 \n'], - 'dimensions': [u'\n Description Electrorheological Cyborgs \n'], - 'site_id': [u'\n Offer From $2500.00 \n'], - } - ), - (# right extraction - 'with_match_common_prefix', [ANNOTATED_PAGE10a, ANNOTATED_PAGE10b], EXTRACT_PAGE10a, None, - {} - ), - (# another example - 'match_common_prefix', [ANNOTATED_PAGE10a, ANNOTATED_PAGE10b], EXTRACT_PAGE10b, None, - { - 'dimensions': [u'50cm'], - 'site_id': [u'K80'], - } - ), - (# common_prefix with allow_markup attribute - 'common_prefix_allow_markup', [ANNOTATED_PAGE10a, ANNOTATED_PAGE10b], EXTRACT_PAGE10b, - ItemDescriptor('test', 'product test', - [A('dimensions', "something about dimensions", allow_markup=True)]), - { - 'dimensions': [u'50cm'], - 'site_id': [u'K80'], - } - ), (# special case with partial annotations 'special_partial_annotation', [ANNOTATED_PAGE11], EXTRACT_PAGE11, None, { diff --git a/scrapy/tests/test_contrib_ibl/test_pageparsing.py b/scrapy/tests/test_contrib_ibl/test_pageparsing.py index 8b88c81d8..1e3d610bb 100644 --- a/scrapy/tests/test_contrib_ibl/test_pageparsing.py +++ b/scrapy/tests/test_contrib_ibl/test_pageparsing.py @@ -34,7 +34,7 @@ LABELLED_PAGE1 = u""" This is such a nice item
Everybody likes it.

-

+

\xa310.00

@@ -220,19 +220,6 @@ class TestPageParsing(TestCase): self.assertEqual(len(lp.annotations), 5) self._validate_annotation(lp, lp.annotations[0], 'name', '

', '

') - self.assertEqual(lp.annotations[0].match_common_prefix, False) - self._validate_annotation(lp, lp.annotations[1], - 'description', '

', '

') - self.assertEqual(lp.annotations[1].match_common_prefix, False) - self._validate_annotation(lp, lp.annotations[2], - 'price', '

', '

') - self.assertEqual(lp.annotations[2].match_common_prefix, True) - self._validate_annotation(lp, lp.annotations[3], - 'short_description', '

', '

') - self.assertEqual(lp.annotations[3].match_common_prefix, False) - self._validate_annotation(lp, lp.annotations[4], - 'short_description', '

', '

') - self.assertEqual(lp.annotations[4].match_common_prefix, False) # all tags were closed self.assertEqual(len(lp.labelled_tag_stacks), 0)