Populate annotation metadata with data not used by IBL extractor.

scrapy/contrib/ibl/extraction/__init__.py

@@ -111,7 +111,7 @@ class InstanceBasedLearningExtractor(object):
             correctly_extracted = [c for c in correctly_extracted if \
                 extra_required.intersection(c.keys()) == extra_required ]
             if len(correctly_extracted) > 0:
-                return correctly_extracted, extraction_tree.template.id
+                return correctly_extracted, extraction_tree.template
         return None, None
 
     def __str__(self):

scrapy/contrib/ibl/extraction/pageobjects.py

@@ -190,10 +190,11 @@ class AnnotationTag(object):
                          for each item to be extracted from a tag attribute
         annotation_text - text prefix and suffix for the attribute to be extracted
         match_common_prefix - use this annotation for calculating across-template prefixes
+        metadata - dict with annotation data not used by IBL extractor
     """
     __slots__ = ('surrounds_attribute', 'start_index', 'end_index',
             'tag_attributes', 'annotation_text', 'variant_id', 
-            'surrounds_variant','match_common_prefix')
+            'surrounds_variant','match_common_prefix', 'metadata')
     
     def __init__(self, start_index, end_index, surrounds_attribute=None, 
             annotation_text=None, tag_attributes=None, variant_id=None,
@@ -206,6 +207,7 @@ class AnnotationTag(object):
         self.variant_id = variant_id
         self.surrounds_variant = surrounds_variant
         self.match_common_prefix = match_common_prefix
+        self.metadata = {}
 
     def __str__(self):
         return "AnnotationTag(%s)" % ", ".join(

scrapy/contrib/ibl/extraction/pageparsing.py

@@ -125,7 +125,7 @@ class TemplatePageParser(InstanceLearningParser):
                 self._close_unpaired_tag()
                 
             annotation = AnnotationTag(self.next_tag_index, self.next_tag_index + 1)
-            attribute_annotations = jannotation.get('annotations', {}).items()
+            attribute_annotations = jannotation.pop('annotations', {}).items()
             for extract_attribute, tag_value in attribute_annotations:
                 if extract_attribute == 'content':
                     annotation.surrounds_attribute = tag_value
@@ -133,10 +133,11 @@ class TemplatePageParser(InstanceLearningParser):
                 else:
                     annotation.tag_attributes.append((extract_attribute, tag_value))
             self.annotations.append(annotation)
-            if jannotation.get('common_prefix', False):
+            if jannotation.pop('common_prefix', False):
                 annotation.match_common_prefix = True
 
-            self.extra_required_attrs.extend(jannotation.get('required', []))
+            self.extra_required_attrs.extend(jannotation.pop('required', []))
+            annotation.metadata = jannotation
 
         self.next_tag_index += 1
 
@@ -183,7 +184,7 @@ class TemplatePageParser(InstanceLearningParser):
             return
         
         annotation = AnnotationTag(self.next_tag_index, None)
-        if jannotation.get('generated', False):
+        if jannotation.pop('generated', False):
             self.token_list.pop()
             annotation.start_index -= 1
             if self.previous_element_class == HtmlTag:
@@ -195,22 +196,24 @@ class TemplatePageParser(InstanceLearningParser):
                 ignored = self.ignored_regions.pop()
                 self.ignored_regions.append((ignored[0]-1, ignored[1]))
                 
-        if jannotation.get('common_prefix', False):
+        if jannotation.pop('common_prefix', False):
             annotation.match_common_prefix = True
-            
-        self.extra_required_attrs.extend(jannotation.get('required', []))
+                
+        self.extra_required_attrs.extend(jannotation.pop('required', []))
         
-        variant_id = jannotation.get('variant', 0)
+        variant_id = jannotation.pop('variant', 0)
         if variant_id > 0:
             self.variant_stack.append(variant_id)
             annotation.surrounds_variant = variant_id
-        attribute_annotations = jannotation.get('annotations', {}).items()
+        attribute_annotations = jannotation.pop('annotations', {}).items()
         for extract_attribute, tag_value in attribute_annotations:
             if extract_attribute == 'content':
                 annotation.surrounds_attribute = tag_value
             else:
                 annotation.tag_attributes.append((extract_attribute, tag_value))
         
+        annotation.metadata = jannotation
+
         if annotation.annotation_text is None:
             self.next_tag_index += 1
         if self.variant_stack: