diff --git a/scrapy/contrib/ibl/extraction/__init__.py b/scrapy/contrib/ibl/extraction/__init__.py index 4d5cf6559..1c8fa1802 100644 --- a/scrapy/contrib/ibl/extraction/__init__.py +++ b/scrapy/contrib/ibl/extraction/__init__.py @@ -111,7 +111,7 @@ class InstanceBasedLearningExtractor(object): correctly_extracted = [c for c in correctly_extracted if \ extra_required.intersection(c.keys()) == extra_required ] if len(correctly_extracted) > 0: - return correctly_extracted, extraction_tree.template.id + return correctly_extracted, extraction_tree.template return None, None def __str__(self): diff --git a/scrapy/contrib/ibl/extraction/pageobjects.py b/scrapy/contrib/ibl/extraction/pageobjects.py index c57cdfa75..404106674 100644 --- a/scrapy/contrib/ibl/extraction/pageobjects.py +++ b/scrapy/contrib/ibl/extraction/pageobjects.py @@ -190,10 +190,11 @@ class AnnotationTag(object): for each item to be extracted from a tag attribute annotation_text - text prefix and suffix for the attribute to be extracted match_common_prefix - use this annotation for calculating across-template prefixes + metadata - dict with annotation data not used by IBL extractor """ __slots__ = ('surrounds_attribute', 'start_index', 'end_index', 'tag_attributes', 'annotation_text', 'variant_id', - 'surrounds_variant','match_common_prefix') + 'surrounds_variant','match_common_prefix', 'metadata') def __init__(self, start_index, end_index, surrounds_attribute=None, annotation_text=None, tag_attributes=None, variant_id=None, @@ -206,6 +207,7 @@ class AnnotationTag(object): self.variant_id = variant_id self.surrounds_variant = surrounds_variant self.match_common_prefix = match_common_prefix + self.metadata = {} def __str__(self): return "AnnotationTag(%s)" % ", ".join( diff --git a/scrapy/contrib/ibl/extraction/pageparsing.py b/scrapy/contrib/ibl/extraction/pageparsing.py index 486862ae5..0e82159e8 100644 --- a/scrapy/contrib/ibl/extraction/pageparsing.py +++ b/scrapy/contrib/ibl/extraction/pageparsing.py @@ -125,7 +125,7 @@ class TemplatePageParser(InstanceLearningParser): self._close_unpaired_tag() annotation = AnnotationTag(self.next_tag_index, self.next_tag_index + 1) - attribute_annotations = jannotation.get('annotations', {}).items() + attribute_annotations = jannotation.pop('annotations', {}).items() for extract_attribute, tag_value in attribute_annotations: if extract_attribute == 'content': annotation.surrounds_attribute = tag_value @@ -133,10 +133,11 @@ class TemplatePageParser(InstanceLearningParser): else: annotation.tag_attributes.append((extract_attribute, tag_value)) self.annotations.append(annotation) - if jannotation.get('common_prefix', False): + if jannotation.pop('common_prefix', False): annotation.match_common_prefix = True - self.extra_required_attrs.extend(jannotation.get('required', [])) + self.extra_required_attrs.extend(jannotation.pop('required', [])) + annotation.metadata = jannotation self.next_tag_index += 1 @@ -183,7 +184,7 @@ class TemplatePageParser(InstanceLearningParser): return annotation = AnnotationTag(self.next_tag_index, None) - if jannotation.get('generated', False): + if jannotation.pop('generated', False): self.token_list.pop() annotation.start_index -= 1 if self.previous_element_class == HtmlTag: @@ -195,22 +196,24 @@ class TemplatePageParser(InstanceLearningParser): ignored = self.ignored_regions.pop() self.ignored_regions.append((ignored[0]-1, ignored[1])) - if jannotation.get('common_prefix', False): + if jannotation.pop('common_prefix', False): annotation.match_common_prefix = True - - self.extra_required_attrs.extend(jannotation.get('required', [])) + + self.extra_required_attrs.extend(jannotation.pop('required', [])) - variant_id = jannotation.get('variant', 0) + variant_id = jannotation.pop('variant', 0) if variant_id > 0: self.variant_stack.append(variant_id) annotation.surrounds_variant = variant_id - attribute_annotations = jannotation.get('annotations', {}).items() + attribute_annotations = jannotation.pop('annotations', {}).items() for extract_attribute, tag_value in attribute_annotations: if extract_attribute == 'content': annotation.surrounds_attribute = tag_value else: annotation.tag_attributes.append((extract_attribute, tag_value)) + annotation.metadata = jannotation + if annotation.annotation_text is None: self.next_tag_index += 1 if self.variant_stack: diff --git a/scrapy/tests/test_contrib_ibl/samples_pageparsing.json.gz b/scrapy/tests/test_contrib_ibl/samples_pageparsing.json.gz index 20a2eb09a..c3d950f73 100644 Binary files a/scrapy/tests/test_contrib_ibl/samples_pageparsing.json.gz and b/scrapy/tests/test_contrib_ibl/samples_pageparsing.json.gz differ