Added new instance based learning extraction library in scrapy.contrib.ibl. Documentation and tools will be added later.

scrapy/contrib/ibl/__init__.py

@@ -0,0 +1,10 @@
+"""
+This contrib implements an automatic extraction library based on an Instance
+Based Learning (IBL) algorithm, as described in the following papers:
+
+A hierarchical approach to wrapper induction
+    http://portal.acm.org/citation.cfm?id=301191 
+
+Extracting web data using instance based learning
+    http://portal.acm.org/citation.cfm?id=1265174 
+"""

scrapy/contrib/ibl/descriptor.py

@@ -0,0 +1,51 @@
+"""
+Extended types for IBL extraction
+"""
+from itertools import chain 
+
+from scrapy.contrib.ibl.extractors import text
+
+class FieldDescriptor(object):
+    """description of a scraped attribute"""
+    __slots__ = ('name', 'description', 'extractor', 'required', 'allow_markup')
+
+    def __init__(self, name, description, extractor=text, required=False,
+            allow_markup=False):
+        self.name = name
+        self.description = description
+        self.extractor = extractor
+        self.required = required
+        self.allow_markup = allow_markup
+    
+    def __str__(self):
+        return "FieldDescriptor(%s)" % self.name
+
+class ItemDescriptor(object):
+    """Simple auto scraping item descriptor. 
+
+    This used to describe type-specific operations and may be overridden where
+    necessary.
+    """
+
+    def __init__(self, name, description, attribute_descriptors):
+        self.name = name
+        self.attribute_map = dict((d.name, d) for d in attribute_descriptors)
+        self._required_attributes = [d.name for d in attribute_descriptors \
+                if d.required]
+
+    def validated(self, data):
+        """Only return the items in the data that are valid"""
+        return [d for d in data if self._item_validates(d)]
+
+    def _item_validates(self, item):
+        """simply checks that all mandatory attributes are present"""
+        variant_attrs = set(chain(*
+            [v.keys() for v in item.get('variants', [])]))
+        return all([(name in item or name in variant_attrs) \
+                for name in self._required_attributes])
+
+    def get_required_attributes(self):
+        return self._required_attributes
+    
+    def __str__(self):
+        return "ItemDescriptor(%s)" % self.name

scrapy/contrib/ibl/extraction/__init__.py

@@ -0,0 +1,126 @@
+"""
+IBL module
+
+This contains an extraction algorithm based on the paper Extracting Web Data
+Using Instance-Based Learning by Yanhong Zhai and Bing Liu.
+
+It defines the InstanceBasedLearningExtractor class, which implements this
+extraction algorithm.
+
+Main departures from the original algorithm:
+    * there is no limit in prefix or suffix size
+    * we have "attribute adaptors" that allow generic post processing and may
+      affect the extraction process. For example, a price field may require a 
+      numeric value to be present.
+    * tags can be inserted to extract regions not wrapped by html tags. These
+      regions are then identified using the longest unique character prefix and
+      suffix.
+"""
+from operator import itemgetter
+from .regionextract import build_extraction_tree
+from .pageparsing import parse_template, parse_extraction_page
+from .pageobjects import TokenDict, AnnotationText
+from .similarity import common_prefix
+
+class InstanceBasedLearningExtractor(object):
+    """Implementation of the instance based learning algorithm to 
+    extract data from web pages.
+    """
+
+    def __init__(self, templates, type_descriptor=None, trace=False):
+        """Initialise this extractor
+
+        templates should contain a sequence of strings, each containing 
+        annotated html that will be used as templates for extraction.
+
+        Tags surrounding areas to be extracted must contain a 
+        'data-scrapy-annotate' attribute and the value must be the name
+        of the attribute. If the tag was inserted and was not present in the 
+        original page, the data-scrapy-generated attribute must be present.
+        
+        type_descriptor may contain a type descriptor describing the item
+        to be extracted.
+        
+        if trace is true, the returned extracted data will have a 'trace'
+        property that contains a trace of the extraction execution.
+        """
+        self.token_dict = TokenDict()
+        parsed_plus_templates = [(parse_template(self.token_dict, t), t) for t in templates]
+        parsed_plus_epages = [(p, parse_extraction_page(self.token_dict, t)) for p, t \
+               in parsed_plus_templates if _annotation_count(p)]
+        parsed_templates = map(itemgetter(0), parsed_plus_epages)
+        
+        # calculate common text prefixes for annotations of same field across all templates
+        extracted_text = {}
+        extraction_pages = map(itemgetter(1), parsed_plus_epages)
+        for i, parsed in enumerate(parsed_templates):
+            for annot in parsed.annotations:
+                if annot.match_common_prefix:
+                    field = annot.surrounds_attribute
+                    if field is not None:
+                        descriptor = type_descriptor.attribute_map.get(field) if type_descriptor else None
+                        allow_markup = descriptor.allow_markup if descriptor else False
+                        start, end = annot.start_index, annot.end_index
+                        if allow_markup:
+                            text = extraction_pages[i].html_between_tokens(start, end)
+                        else:
+                            text = extraction_pages[i].text_between_tokens(start, end)
+                        extracted_text.setdefault(field, []).append(text)
+        common_prefixes = {}
+        for field, data in extracted_text.iteritems():
+            if len(data) > 1:
+                cprefix = common_prefix(*data)
+                if cprefix:
+                    common_prefixes[field] = "".join(cprefix).strip()
+        # now apply common prefixes to annotations
+        for i, parsed in enumerate(parsed_templates):
+            for annot in parsed.annotations:
+                for field, prefix in common_prefixes.iteritems():
+                    if annot.surrounds_attribute == field and not annot.annotation_text:
+                        annot.annotation_text = AnnotationText(prefix)
+        
+        # templates with more attributes are considered first
+        sorted_templates = sorted(parsed_templates, key=_annotation_count, reverse=True)
+        self.extraction_trees = [build_extraction_tree(t, type_descriptor, 
+            trace) for t in sorted_templates]
+        self.validated = type_descriptor.validated if type_descriptor else \
+                self._filter_not_none
+
+    def extract(self, html, pref_template_id=None, useone=False):
+        """extract data from an html page
+        
+        If pref_template_url is specified, the template with that url will be 
+        used first.
+        if useone is True and no data was extracted, no additional template will
+        be tried. If False and no data was extracted, try with rest of item templates
+        """
+        extraction_page = parse_extraction_page(self.token_dict, html)
+        if pref_template_id is not None:
+            if useone:
+                extraction_trees = [x for x in self.extraction_trees if x.template.id == pref_template_id]
+            else:
+                extraction_trees = sorted(self.extraction_trees, 
+                    key=lambda x: x.template.id != pref_template_id)
+        else:
+            extraction_trees = self.extraction_trees
+
+        for extraction_tree in extraction_trees:
+            extracted = extraction_tree.extract(extraction_page)
+            correctly_extracted = self.validated(extracted)
+            extra_required = extraction_tree.template.extra_required_attrs
+            correctly_extracted = [c for c in correctly_extracted if \
+                extra_required.intersection(c.keys()) == extra_required ]
+            if len(correctly_extracted) > 0:
+                return correctly_extracted, extraction_tree.template.id
+        return None, None
+
+    def __str__(self):
+        return "InstanceBasedLearningExtractor[\n%s\n]" % \
+                (',\n'.join(map(str, self.extraction_trees)))
+    
+    @staticmethod
+    def _filter_not_none(items):
+        return [d for d in items if d is not None]
+
+def _annotation_count(template):
+    return len(template.annotations)

scrapy/contrib/ibl/extraction/pageobjects.py

@@ -0,0 +1,230 @@
+"""
+Page objects
+
+This module contains objects representing pages and parts of pages (e.g. tokens
+and annotations) used in the instance based learning algorithm.
+"""
+from numpy import array, ndarray
+
+from scrapy.contrib.ibl.htmlpage import HtmlTagType
+
+class TokenType(object):
+    """constants for token types"""
+    WORD = 0
+    OPEN_TAG = HtmlTagType.OPEN_TAG
+    CLOSE_TAG = HtmlTagType.CLOSE_TAG
+    NON_PAIRED_TAG = HtmlTagType.UNPAIRED_TAG
+
+class TokenDict(object):
+    """Mapping from parse tokens to integers
+    
+    >>> d = TokenDict()
+    >>> d.tokenid('i')
+    0
+    >>> d.tokenid('b')
+    1
+    >>> d.tokenid('i')
+    0
+
+    Tokens can be searched for by id
+    >>> d.find_token(1)
+    'b'
+
+    The lower 24 bits store the token reference and the higher bits the type.
+    """
+    
+    def __init__(self):
+        self.token_ids = {}
+
+    def tokenid(self, token, token_type=TokenType.WORD):
+        """create an integer id from the token and token type passed"""
+        tid = self.token_ids.setdefault(token, len(self.token_ids))
+        return tid | (token_type << 24)
+    
+    @staticmethod
+    def token_type(token):
+        """extract the token type from the token id passed"""
+        return token >> 24
+
+    def find_token(self, tid):
+        """Search for a tag with the given ID
+
+        This is O(N) and is only intended for debugging
+        """
+        tid &= 0xFFFFFF
+        if tid >= len(self.token_ids) or tid < 0:
+            raise ValueError("tag id %s out of range" % tid)
+
+        for (token, token_id) in self.token_ids.items():
+            if token_id == tid:
+                return token
+        assert False, "token dictionary is corrupt"
+
+    def token_string(self, tid):
+        """create a string representation of a token
+
+        This is O(N).
+        """
+        templates = ["%s", "<%s>", "</%s>", "<%s/>"]
+        return templates[tid >> 24] % self.find_token(tid)
+
+class Page(object):
+    """Basic representation of a page. This consists of a reference to a
+    dictionary of tokens and an array of raw token ids
+    """
+
+    __slots__ = ('token_dict', 'page_tokens')
+
+    def __init__(self, token_dict, page_tokens):
+        self.token_dict = token_dict
+        # use a numpy array becuase we can index/slice easily and efficiently
+        if not isinstance(page_tokens, ndarray):
+            page_tokens = array(page_tokens)
+        self.page_tokens = page_tokens
+
+class TemplatePage(Page):
+    __slots__ = ('annotations', 'id', 'ignored_regions', 'extra_required_attrs')
+
+    def __init__(self, token_dict, page_tokens, annotations, template_id=None, \
+            ignored_regions=None, extra_required=None):
+        Page.__init__(self, token_dict, page_tokens)
+        # ensure order is the same as start tag order in the original page
+        annotations = sorted(annotations, key=lambda x: x.end_index, reverse=True)
+        self.annotations = sorted(annotations, key=lambda x: x.start_index)
+        self.id = template_id
+        self.ignored_regions = ignored_regions or []
+        self.extra_required_attrs = set(extra_required or [])
+
+    def __str__(self):
+        summary = []
+        for index, token in enumerate(self.page_tokens):
+            text = "%s: %s" % (index, self.token_dict.find_token(token))
+            summary.append(text)
+        return "TemplatePage\n============\nTokens: (index, token)\n%s\nAnnotations: %s\n" % \
+                ('\n'.join(summary), '\n'.join(map(str, self.annotations)))
+
+class ExtractionPage(Page):
+    """Parsed data belonging to a web page upon which we wish to perform
+    extraction.
+    """
+    __slots__ = ('text', 
+            'token_start_indexes', # index in text of the start of a token
+            'token_follow_indexes', # index in text of data following token
+            'tag_attributes'        # a map from token index to tag attributes
+        )
+
+    def __init__(self, text, token_dict, page_tokens, token_start_indexes, 
+            token_follow_indexes, tag_attributes):
+        Page.__init__(self, token_dict, page_tokens)
+        self.text = text
+        self.token_start_indexes = token_start_indexes
+        self.token_follow_indexes = token_follow_indexes
+        self.tag_attributes = tag_attributes
+
+    def token_html(self, token_index):
+        """The raw html for a page token at the given index in the page_tokens
+        list
+        """
+        text_start = self.token_start_indexes[token_index]
+        text_end = self.token_follow_indexes[token_index]
+        return self.text[text_start:text_end]
+
+    def html_between_tokens(self, start_token_index, end_token_index):
+        """The raw html between the tokens at the specified indexes in the 
+        page_tokens list
+
+        This assumes start_token_index <= end_token_index
+        """
+        text_start = self.token_follow_indexes[start_token_index]
+        text_end = self.token_start_indexes[end_token_index]
+        return self.text[text_start:text_end]
+    
+    def text_between_tokens(self, start_token_index, end_token_index, 
+            tag_replacement=u' '):
+        """The text between the  the tokens at the specified indexes in the 
+        page_tokens list. Tags are replaced by tag_replacement (default one space 
+        character)
+        """
+        return tag_replacement.join([self.text[
+            self.token_follow_indexes[i]:self.token_start_indexes[i+1]] \
+            for i in xrange(start_token_index, end_token_index)])
+
+    def tag_attribute(self, token_index, attribute):
+        """The value of a tag attribute. The tag is identified by its 
+        corresponding token index.
+
+        If the tag or attribute is not present, None is returned
+        """
+        return self.tag_attributes.get(token_index, {}).get(attribute) 
+    
+    def __str__(self):
+        summary = []
+        for (token, start, follow) in zip(self.page_tokens, self.token_start_indexes, 
+                self.token_follow_indexes):
+            text = "%s %s-%s (%s)" % (self.token_dict.find_token(token), start, follow, 
+                    self.text[start:follow])
+            summary.append(text)
+        return "ExtractionPage\n==============\nTokens: %s\n\nRaw text: %s\n\n" \
+                "Tag attributes: %s\n" % ('\n'.join(summary), self.text, 
+                        self.tag_attributes)
+
+class AnnotationText(object):
+    __slots__ = ('start_text', 'follow_text')
+
+    def __init__(self, start_text=None, follow_text=None):
+        self.start_text = start_text
+        self.follow_text = follow_text
+
+    def __str__(self):
+        return "AnnotationText(%s..%s)" % \
+                (repr(self.start_text), repr(self.follow_text))
+
+class AnnotationTag(object):
+    """A tag that annotates part of the document
+
+    It has the following properties:
+        start_index - index of the token for the opening tag
+        end_index - index of the token for the closing tag
+        surrounds_attribute - the attribute name surrounded by this tag
+        tag_attributes - list of (tag attribute, extracted attribute) tuples
+                         for each item to be extracted from a tag attribute
+        annotation_text - text prefix and suffix for the attribute to be extracted
+        match_common_prefix - use this annotation for calculating across-template prefixes
+    """
+    __slots__ = ('surrounds_attribute', 'start_index', 'end_index',
+            'tag_attributes', 'annotation_text', 'variant_id', 
+            'surrounds_variant','match_common_prefix')
+    
+    def __init__(self, start_index, end_index, surrounds_attribute=None, 
+            annotation_text=None, tag_attributes=None, variant_id=None,
+            surrounds_variant=None, match_common_prefix=False):
+        self.start_index = start_index
+        self.end_index = end_index
+        self.surrounds_attribute = surrounds_attribute
+        self.annotation_text = annotation_text
+        self.tag_attributes = tag_attributes or []
+        self.variant_id = variant_id
+        self.surrounds_variant = surrounds_variant
+        self.match_common_prefix = match_common_prefix
+
+    def __str__(self):
+        return "AnnotationTag(%s)" % ", ".join(
+                ["%s=%s" % (s, getattr(self, s)) \
+                for s in self.__slots__ if getattr(self, s)])
+
+    def __repr__(self):
+        return str(self)
+
+class LabelledRegion(object):
+    __slots__ = ('start_index', 'end_index')
+    
+    def __init__(self, start, end):
+        self.start_index = start
+        self.end_index = end
+        
+    def __str__(self):
+        return "LabelledRegion (%s, %s)" % (self.start_index, self.end_index)
+    
+    def __repr__(self):
+        return str(self)
+

scrapy/contrib/ibl/extraction/pageparsing.py

@@ -0,0 +1,318 @@
+"""
+Page parsing
+
+Parsing of web pages for extraction task.
+"""
+from collections import defaultdict
+from numpy import array
+
+from scrapy.utils.py26 import json
+
+from scrapy.contrib.ibl.htmlpage import HtmlTagType, HtmlTag, HtmlPage
+from scrapy.contrib.ibl.extraction.pageobjects import (AnnotationTag, 
+        TemplatePage, ExtractionPage, AnnotationText, TokenDict)
+
+def parse_strings(template_html, extraction_html):
+    """Create a template and extraction page from raw strings
+
+    this is useful for testing purposes
+    """
+    t = TokenDict()
+    template_page = HtmlPage(body=template_html)
+    extraction_page = HtmlPage(body=extraction_html)
+    return (parse_template(t, template_page), 
+            parse_extraction_page(t, extraction_page))
+
+def parse_template(token_dict, template_html):
+    """Create an TemplatePage object by parsing the annotated html"""
+    parser = TemplatePageParser(token_dict)
+    parser.feed(template_html)
+    return parser.to_template()
+
+def parse_extraction_page(token_dict, page_html):
+    """Create an ExtractionPage object by parsing the html"""
+    parser = ExtractionPageParser(token_dict)
+    parser.feed(page_html)
+    return parser.to_extraction_page()
+
+class InstanceLearningParser(object):
+    """Base parser for instance based learning algorithm
+    
+    This does not require correct HTML and the parsing method should not alter
+    the original tag order. It is important that parsing results do not vary.
+    """
+    def __init__(self, token_dict):
+        self.token_dict = token_dict
+        self.token_list = []
+    
+    def _add_token(self, token, token_type, start, end):
+        tid = self.token_dict.tokenid(token, token_type)
+        self.token_list.append(tid)
+
+    def feed(self, html_page):
+        self.html_page = html_page
+        self.previous_element_class = None
+        for data in html_page.parsed_body:
+            if isinstance(data, HtmlTag):
+                self._add_token(data.tag, data.tag_type, data.start, data.end)
+                self.handle_tag(data)
+            else:
+                self.handle_data(data)
+            self.previous_element_class = data.__class__
+
+    def handle_data(self, html_data_fragment):
+        pass
+
+    def handle_tag(self, html_tag):
+        pass
+
+_END_UNPAIREDTAG_TAGS = ["form", "div", "p", "table", "tr", "td"]
+
+class TemplatePageParser(InstanceLearningParser):
+    """Template parsing for instance based learning algorithm"""
+
+    def __init__(self, token_dict):
+        InstanceLearningParser.__init__(self, token_dict)
+        self.annotations = []
+        self.ignored_regions = []
+        self.extra_required_attrs = []
+        self.ignored_tag_stacks = defaultdict(list)
+        # tag names that have not been completed
+        self.labelled_tag_stacks = defaultdict(list)
+        self.replacement_stacks = defaultdict(list)
+        self.unpairedtag_stack = []
+        self.variant_stack = []
+        self.prev_data = None
+        self.last_text_region = None
+        self.next_tag_index = 0
+
+    def handle_tag(self, html_tag):
+        if self.last_text_region:
+            self._process_text('')
+        
+        if html_tag.tag_type == HtmlTagType.OPEN_TAG:
+            self._handle_open_tag(html_tag)
+        elif html_tag.tag_type == HtmlTagType.CLOSE_TAG:
+            self._handle_close_tag(html_tag)
+        else:
+            # the tag is not paired, it can contain only attribute annotations
+            self._handle_unpaired_tag(html_tag)
+    
+    @staticmethod
+    def _read_template_annotation(html_tag):
+        template_attr = html_tag.attributes.get('data-scrapy-annotate')
+        if template_attr is None:
+            return None
+        unescaped = template_attr.replace('"', '"')
+        return json.loads(unescaped)
+    
+    @staticmethod
+    def _read_bool_template_attribute(html_tag, attribute):
+        return html_tag.attributes.get("data-scrapy-" + attribute) == "true"
+    
+    def _close_unpaired_tag(self):
+        self.unpairedtag_stack[0].end_index = self.next_tag_index
+        self.unpairedtag_stack = []
+
+    def _handle_unpaired_tag(self, html_tag):
+        if self._read_bool_template_attribute(html_tag, "ignore") and html_tag.tag == "img":
+            self.ignored_regions.append((self.next_tag_index, self.next_tag_index + 1))
+        elif self._read_bool_template_attribute(html_tag, "ignore-beneath") and html_tag.tag == "img":
+            self.ignored_regions.append((self.next_tag_index, None))
+        jannotation = self._read_template_annotation(html_tag)
+        if jannotation:
+            if self.unpairedtag_stack:
+                self._close_unpaired_tag()
+                
+            annotation = AnnotationTag(self.next_tag_index, self.next_tag_index + 1)
+            attribute_annotations = jannotation.get('annotations', {}).items()
+            for extract_attribute, tag_value in attribute_annotations:
+                if extract_attribute == 'content':
+                    annotation.surrounds_attribute = tag_value
+                    self.unpairedtag_stack.append(annotation)
+                else:
+                    annotation.tag_attributes.append((extract_attribute, tag_value))
+            self.annotations.append(annotation)
+            if jannotation.get('common_prefix', False):
+                annotation.match_common_prefix = True
+
+            self.extra_required_attrs.extend(jannotation.get('required', []))
+
+        self.next_tag_index += 1
+
+    def _handle_open_tag(self, html_tag):
+        if self._read_bool_template_attribute(html_tag, "ignore"):
+            if html_tag.tag == "img":
+                self.ignored_regions.append((self.next_tag_index, self.next_tag_index + 1))
+            else:
+                self.ignored_regions.append((self.next_tag_index, None))
+                self.ignored_tag_stacks[html_tag.tag].append(html_tag)
+                
+        elif self.ignored_tag_stacks.get(html_tag.tag):
+            self.ignored_tag_stacks[html_tag.tag].append(None)
+        if self._read_bool_template_attribute(html_tag, "ignore-beneath"):
+            self.ignored_regions.append((self.next_tag_index, None))
+        
+        replacement = html_tag.attributes.pop("data-scrapy-replacement", None)
+        if replacement:
+            self.token_list.pop()
+            self._add_token(replacement, html_tag.tag_type, html_tag.start, html_tag.end)
+            self.replacement_stacks[html_tag.tag].append(replacement)
+        elif html_tag.tag in self.replacement_stacks:
+            self.replacement_stacks[html_tag.tag].append(None)
+
+        if self.unpairedtag_stack:
+            if html_tag.tag in _END_UNPAIREDTAG_TAGS:
+                self._close_unpaired_tag()
+            else:
+                self.unpairedtag_stack.append(html_tag.tag)
+            
+        # can't be a p inside another p. Also, an open p element closes
+        # a previous open p element.
+        if html_tag.tag == "p" and html_tag.tag in self.labelled_tag_stacks:
+            annotation = self.labelled_tag_stacks.pop(html_tag.tag)[0]
+            annotation.end_index = self.next_tag_index
+            self.annotations.append(annotation)
+                
+        jannotation = self._read_template_annotation(html_tag)
+        if not jannotation:
+            if html_tag.tag in self.labelled_tag_stacks:
+                # add this tag to the stack to match correct end tag
+                self.labelled_tag_stacks[html_tag.tag].append(None)
+            self.next_tag_index += 1
+            return
+        
+        annotation = AnnotationTag(self.next_tag_index, None)
+        if jannotation.get('generated', False):
+            self.token_list.pop()
+            annotation.start_index -= 1
+            if self.previous_element_class == HtmlTag:
+                annotation.annotation_text = AnnotationText('')
+            else:
+                annotation.annotation_text = AnnotationText(self.prev_data)
+            if self._read_bool_template_attribute(html_tag, "ignore") \
+                    or self._read_bool_template_attribute(html_tag, "ignore-beneath"):
+                ignored = self.ignored_regions.pop()
+                self.ignored_regions.append((ignored[0]-1, ignored[1]))
+                
+        if jannotation.get('common_prefix', False):
+            annotation.match_common_prefix = True
+            
+        self.extra_required_attrs.extend(jannotation.get('required', []))
+        
+        variant_id = jannotation.get('variant', 0)
+        if variant_id > 0:
+            self.variant_stack.append(variant_id)
+            annotation.surrounds_variant = variant_id
+        attribute_annotations = jannotation.get('annotations', {}).items()
+        for extract_attribute, tag_value in attribute_annotations:
+            if extract_attribute == 'content':
+                annotation.surrounds_attribute = tag_value
+            else:
+                annotation.tag_attributes.append((extract_attribute, tag_value))
+        
+        if annotation.annotation_text is None:
+            self.next_tag_index += 1
+        if self.variant_stack:
+            variant_id = self.variant_stack[-1]
+            if variant_id == '0':
+                variant_id = None
+            annotation.variant_id = variant_id
+        
+        # look for a closing tag if the content is important
+        if annotation.surrounds_attribute or annotation.surrounds_variant:
+            self.labelled_tag_stacks[html_tag.tag].append(annotation)
+        else:
+            annotation.end_index = annotation.start_index + 1
+            self.annotations.append(annotation)
+
+    def _handle_close_tag(self, html_tag):
+        
+        if self.unpairedtag_stack:
+            if html_tag.tag == self.unpairedtag_stack[-1]:
+                self.unpairedtag_stack.pop()
+            else:
+                self._close_unpaired_tag()
+        ignored_tags = self.ignored_tag_stacks.get(html_tag.tag)
+        if ignored_tags is not None:
+            tag = ignored_tags.pop()
+            if isinstance(tag, HtmlTag):
+                for i in range(-1, -len(self.ignored_regions) - 1, -1):
+                    if self.ignored_regions[i][1] is None:
+                        self.ignored_regions[i] = (self.ignored_regions[i][0], self.next_tag_index)
+                        break
+            if len(ignored_tags) == 0:
+                del self.ignored_tag_stacks[html_tag.tag]
+
+        if html_tag.tag in self.replacement_stacks:
+            replacement = self.replacement_stacks[html_tag.tag].pop()
+            if replacement:
+                self.token_list.pop()
+                self._add_token(replacement, html_tag.tag_type, html_tag.start, html_tag.end)
+            if len(self.replacement_stacks[html_tag.tag]) == 0:
+                del self.replacement_stacks[html_tag.tag]
+
+        labelled_tags = self.labelled_tag_stacks.get(html_tag.tag)
+        if labelled_tags is None:
+            self.next_tag_index += 1
+            return
+        annotation = labelled_tags.pop()
+        if annotation is None:
+            self.next_tag_index += 1
+        else:
+            annotation.end_index = self.next_tag_index
+            self.annotations.append(annotation)
+            if annotation.annotation_text is not None:
+                self.token_list.pop()
+                self.last_text_region = annotation
+            else:
+                self.next_tag_index += 1
+            if len(labelled_tags) == 0:
+                del self.labelled_tag_stacks[html_tag.tag]
+            if annotation.surrounds_variant and self.variant_stack:
+                prev = self.variant_stack.pop()
+                if prev != annotation.surrounds_variant:
+                    raise ValueError("unbalanced variant annotation tags")
+                    
+    def handle_data(self, html_data_fragment):
+        fragment_text = self.html_page.fragment_data(html_data_fragment)
+        self._process_text(fragment_text)
+
+    def _process_text(self, text):
+        if self.last_text_region is not None:
+            self.last_text_region.annotation_text.follow_text = text
+            self.last_text_region = None
+        self.prev_data = text
+
+    def to_template(self):
+        """create a TemplatePage from the data fed to this parser"""
+        return TemplatePage(self.token_dict, self.token_list, self.annotations,
+                self.html_page.page_id, self.ignored_regions, self.extra_required_attrs)
+
+class ExtractionPageParser(InstanceLearningParser):
+    """Parse an HTML page for extraction using the instance based learning
+    algorithm
+
+    This needs to extract the tokens in a similar way to LabelledPageParser,
+    it needs to also maintain a mapping from token index to the original content
+    so that once regions are identified, the original content can be extracted.
+    """
+    def __init__(self, token_dict):
+        InstanceLearningParser.__init__(self, token_dict)
+        self.page_data = []
+        self.token_start_index = []
+        self.token_follow_index = []
+        self.tag_attrs = {}
+
+    def _add_token(self, token, token_type, start, end):
+        InstanceLearningParser._add_token(self, token, token_type, start, end)
+        self.token_start_index.append(start)
+        self.token_follow_index.append(end)
+
+    def handle_tag(self, html_tag):
+        if html_tag.attributes:
+            self.tag_attrs[len(self.token_list) - 1] = html_tag.attributes
+    
+    def to_extraction_page(self):
+        return ExtractionPage(self.html_page.body, self.token_dict, array(self.token_list), 
+                self.token_start_index, self.token_follow_index, self.tag_attrs)

scrapy/contrib/ibl/extraction/regionextract.py

@@ -0,0 +1,651 @@
+"""
+Region Extract
+
+Custom extraction for regions in a document
+"""
+import operator
+import copy
+import pprint
+import cStringIO
+from itertools import groupby
+
+import nltk
+from numpy import array
+
+from scrapy.contrib.ibl.descriptor import FieldDescriptor
+from scrapy.contrib.ibl.extraction.similarity import (similar_region,
+    longest_unique_subsequence, common_prefix)
+from scrapy.contrib.ibl.extraction.pageobjects import AnnotationTag, LabelledRegion
+
+def build_extraction_tree(template, type_descriptor, trace=True):
+    """Build a tree of region extractors corresponding to the 
+    template
+    """
+    attribute_map = type_descriptor.attribute_map if type_descriptor else None
+    extractors = BasicTypeExtractor.create(template.annotations, attribute_map)
+    if trace:
+        extractors = TraceExtractor.apply(template, extractors)
+    for cls in (RepeatedDataExtractor, AdjacentVariantExtractor, RepeatedDataExtractor,
+            RecordExtractor):
+        extractors = cls.apply(template, extractors)
+        if trace:
+            extractors = TraceExtractor.apply(template, extractors)
+
+    return TemplatePageExtractor(template, extractors)
+
+_ID = lambda x: x
+_DEFAULT_DESCRIPTOR = FieldDescriptor('none', None)
+
+def _labelled(obj):
+    """
+    Returns labelled element of the object (extractor or labelled region)
+    """
+    if hasattr(obj, "annotation"):
+        return obj.annotation
+    return obj
+
+def _compose(f, g):
+    """given unary functions f and g, return a function that computes f(g(x))
+    """
+    def _exec(x):
+        ret = g(x)
+        return f(ret) if ret is not None else None
+    return _exec
+
+class BasicTypeExtractor(object):
+    """The BasicTypeExtractor extracts single attributes corresponding to 
+    annotations.
+    
+    For example:
+    >>> from scrapy.contrib.ibl.extraction.pageparsing import parse_strings
+    >>> template, page = parse_strings( \
+        u'<h1 data-scrapy-annotate="{&quot;annotations&quot;: {&quot;content&quot;: &quot;name&quot;}}">x</h1>', u'<h1> a name</h1>')
+    >>> ex = BasicTypeExtractor(template.annotations[0])
+    >>> ex.extract(page, 0, 1, None)
+    [(u'name', u' a name')]
+
+    It supports attribute descriptors
+    >>> descriptor = FieldDescriptor('name', None, lambda x: x.strip())
+    >>> ex = BasicTypeExtractor(template.annotations[0], {'name': descriptor})
+    >>> ex.extract(page, 0, 1, None)
+    [(u'name', u'a name')]
+    
+    It supports ignoring regions
+    >>> template, page = parse_strings(\
+        u'<div data-scrapy-annotate="{&quot;annotations&quot;: {&quot;content&quot;: &quot;name&quot;}}">x<b> xx</b></div>',\
+        u'<div>a name<b> id-9</b></div>')
+    >>> ex = BasicTypeExtractor(template.annotations[0])
+    >>> ex.extract(page, 0, 3, [LabelledRegion(*(1,2))])
+    [(u'name', u'a name')]
+    """
+
+    def __init__(self, annotation, attribute_descriptors=None):
+        self.annotation = annotation
+        if attribute_descriptors is None:
+            attribute_descriptors = {}
+
+        if annotation.surrounds_attribute:
+            descriptor = attribute_descriptors.get(annotation.surrounds_attribute)
+            if descriptor:
+                self.content_validate = descriptor.extractor
+                self.allow_markup = descriptor.allow_markup
+            else:
+                self.content_validate = _ID
+                self.allow_markup = False
+            self.extract = self._extract_content
+
+        if annotation.tag_attributes:
+            self.tag_data = []
+            for (tag_attr, extraction_attr) in annotation.tag_attributes:
+                descriptor = attribute_descriptors.get(extraction_attr)
+                extractf = descriptor.extractor if descriptor else _ID
+                self.tag_data.append((extractf, tag_attr, extraction_attr))
+
+            self.extract = self._extract_both if \
+                    annotation.surrounds_attribute else self._extract_attribute
+        
+    def _extract_both(self, page, start_index, end_index, ignored_regions=None):
+        return self._extract_content(page, start_index, end_index, ignored_regions) + \
+            self._extract_attribute(page, start_index, end_index, ignored_regions)
+
+    def _extract_content(self, extraction_page, start_index, end_index, ignored_regions=None):
+        # we might want to add opening/closing ul/ol/table if we have the
+        # middle of a region. This would require support in the scrapy
+        # cleansing.
+        complete_data = ""
+        start = start_index
+        end = ignored_regions[0].start_index if ignored_regions else end_index
+        while start is not None:
+            if self.allow_markup:
+                data = extraction_page.html_between_tokens(start, end)
+            else:
+                data = extraction_page.text_between_tokens(start, end)
+            complete_data += data
+            if ignored_regions:
+                start = ignored_regions[0].end_index
+                ignored_regions.pop(0)
+                end = ignored_regions[0].start_index if ignored_regions else end_index
+            else:
+                start = None
+        complete_data = self.content_validate(complete_data)
+        return [(self.annotation.surrounds_attribute, complete_data)] if complete_data else []
+    
+    def _extract_attribute(self, extraction_page, start_index, end_index, ignored_regions=None):
+        data = []
+        for (f, ta, ea) in self.tag_data:
+            tag_value = extraction_page.tag_attribute(start_index, ta)
+            if tag_value:
+                extracted = f(tag_value)
+                if extracted is not None:
+                    data.append((ea, extracted))
+        return data
+
+    @classmethod
+    def create(cls, annotations, attribute_descriptors=None):
+        """Create a list of basic extractors from the given annotations
+        and attribute descriptors
+        """
+        if attribute_descriptors is None:
+            attribute_descriptors = {}
+        return [cls._create_basic_extractor(annotation, attribute_descriptors) \
+            for annotation in annotations \
+            if annotation.surrounds_attribute or annotation.tag_attributes]
+    
+    @staticmethod
+    def _create_basic_extractor(annotation, attribute_descriptors):
+        """Create a basic type extractor for the annotation"""
+        text_region = annotation.annotation_text
+        if text_region is not None:
+            if annotation.match_common_prefix:
+                region_extract = TextPrefixRegionDataExtractor(text_region.start_text).extract
+            else:
+                region_extract = TextRegionDataExtractor(text_region.start_text, 
+                    text_region.follow_text).extract
+            # copy attribute_descriptors and add the text extractor
+            descriptor_copy = dict(attribute_descriptors)
+            attr_descr = descriptor_copy.get(annotation.surrounds_attribute, 
+                    _DEFAULT_DESCRIPTOR)
+            attr_descr = copy.copy(attr_descr)
+            attr_descr.extractor = _compose(attr_descr.extractor, region_extract)
+            descriptor_copy[annotation.surrounds_attribute] = attr_descr
+            attribute_descriptors = descriptor_copy
+        return BasicTypeExtractor(annotation, attribute_descriptors)
+    
+    def extracted_item(self):
+        """key used to identify the item extracted"""
+        return (self.annotation.surrounds_attribute, self.annotation.tag_attributes)
+    
+    def __repr__(self):
+        return str(self)
+
+    def __str__(self):
+        messages = ['BasicTypeExtractor(']
+        if self.annotation.surrounds_attribute:
+            messages += [self.annotation.surrounds_attribute, ': ', 
+                    'html content' if self.allow_markup else 'text content',
+            ]
+            if self.content_validate != _ID:
+                messages += [', extracted with \'', 
+                        self.content_validate.__name__, '\'']
+        
+        if self.annotation.tag_attributes:
+            if self.annotation.surrounds_attribute:
+                messages.append(';')
+            for (f, ta, ea) in self.tag_data:
+                messages += [ea, ': tag attribute "', ta, '"']
+                if f != _ID:
+                    messages += [', validated by ', str(f)]
+        messages.append(", template[%s:%s])" % \
+                (self.annotation.start_index, self.annotation.end_index))
+        return ''.join(messages)
+
+class RepeatedDataExtractor(object):
+    """Data extractor for handling repeated data"""
+
+    def __init__(self, prefix, suffix, extractors):
+        self.prefix = array(prefix)
+        self.suffix = array(suffix)
+        self.extractor = copy.copy(extractors[0])
+        self.annotation = copy.copy(self.extractor.annotation)
+        self.annotation.end_index = extractors[-1].annotation.end_index
+
+    def extract(self, page, start_index, end_index, ignored_regions):
+        """repeatedly find regions bounded by the repeated 
+        prefix and suffix and extract them
+        """
+        prefixlen = len(self.prefix)
+        suffixlen = len(self.suffix)
+        index = max(0, start_index - prefixlen)
+        max_index = min(len(page.page_tokens) - suffixlen, end_index + len(self.suffix))
+        max_start_index = max_index - prefixlen
+        extracted = []
+        while index <= max_start_index:
+            prefix_end = index + prefixlen
+            if (page.page_tokens[index:prefix_end] == self.prefix).all():
+                for peek in xrange(prefix_end, max_index):
+                    if (page.page_tokens[peek:peek + suffixlen] \
+                            == self.suffix).all():
+                        extracted += self.extractor.extract(page, 
+                                prefix_end - 1, peek, ignored_regions)
+                        index = max(peek, index + 1)
+                        break
+                else:
+                    break
+            else:
+                index += 1
+        return extracted
+
+    @staticmethod
+    def apply(template, extractors):
+        tokens = template.page_tokens
+        output_extractors = []
+        group_key = lambda x: x.extracted_item()
+        for extr_key, extraction_group in groupby(extractors, group_key):
+            extraction_group = list(extraction_group)
+            if extr_key is None or len(extraction_group) == 1:
+                output_extractors += extraction_group
+                continue
+
+            separating_tokens = [ \
+                tokens[x.annotation.end_index:y.annotation.start_index+1] \
+                for (x, y) in zip(extraction_group[:-1], extraction_group[1:])]
+            
+            # calculate the common prefix
+            group_start = extraction_group[0].annotation.start_index
+            prefix_start = max(0, group_start - len(separating_tokens[0]))
+            first_prefix = tokens[prefix_start:group_start+1]
+            prefixes = [first_prefix] + separating_tokens
+            prefix_pattern = list(reversed(
+                common_prefix(*map(reversed, prefixes))))
+            
+            # calculate the common suffix
+            group_end = extraction_group[-1].annotation.end_index
+            last_suffix = tokens[group_end:group_end + \
+                    len(separating_tokens[-1])]
+            suffixes = separating_tokens + [last_suffix]
+            suffix_pattern = common_prefix(*suffixes)
+            
+            # create a repeated data extractor, if there is a suitable 
+            # prefix and suffix. (TODO: tune this heuristic)
+            matchlen = len(prefix_pattern) + len(suffix_pattern)
+            if matchlen >= len(separating_tokens):
+                group_extractor = RepeatedDataExtractor(prefix_pattern, 
+                    suffix_pattern, extraction_group)
+                output_extractors.append(group_extractor)
+            else:
+                output_extractors += extraction_group
+        return output_extractors
+    
+    def extracted_item(self):
+        """key used to identify the item extracted"""
+        return self.extractor.extracted_item()
+    
+    def __repr__(self):
+        return "Repeat(%r)" % self.extractor
+
+    def __str__(self):
+        return "Repeat(%s)" % self.extractor
+
+class TransposedDataExtractor(object):
+    """ """
+    pass
+
+_namef = operator.itemgetter(0)
+_valuef = operator.itemgetter(1)
+def _attrs2dict(attributes):
+    """convert a list of attributes (name, value) tuples
+    into a dict of lists. 
+
+    For example:
+    >>> l = [('name', 'sofa'), ('colour', 'red'), ('colour', 'green')]
+    >>> _attrs2dict(l) == {'name': ['sofa'], 'colour': ['red', 'green']}
+    True
+    """
+    grouped_data = groupby(sorted(attributes, key=_namef), _namef)
+    return dict((name, map(_valuef, data)) for (name, data)  in grouped_data)
+
+class RecordExtractor(object):
+    """The RecordExtractor will extract records given annotations.
+    
+    It looks for a similar region in the target document, using the ibl
+    similarity algorithm. The annotations are partitioned by the first similar
+    region found and searched recursively.
+
+    Records are represented as dicts mapping attribute names to lists
+    containing their values.
+    
+    For example:
+    >>> from scrapy.contrib.ibl.extraction.pageparsing import parse_strings
+    >>> template, page = parse_strings( \
+            u'<h1 data-scrapy-annotate="{&quot;annotations&quot;: {&quot;content&quot;: &quot;name&quot;}}">x</h1>' + \
+            u'<p data-scrapy-annotate="{&quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">y</p>', \
+            u'<h1>name</h1> <p>description</p>')
+    >>> basic_extractors = map(BasicTypeExtractor, template.annotations)
+    >>> ex = RecordExtractor.apply(template, basic_extractors)[0]
+    >>> ex.extract(page)
+    [{u'description': [u'description'], u'name': [u'name']}]
+    """
+    
+    def __init__(self, extractors, template_tokens):
+        """Construct a RecordExtractor for the given annotations and their
+        corresponding region extractors
+        """
+        self.extractors = extractors
+        self.template_tokens = template_tokens
+        self.template_ignored_regions = []
+        start_index = min(e.annotation.start_index for e in extractors)
+        end_index = max(e.annotation.end_index for e in extractors)
+        self.annotation = AnnotationTag(start_index, end_index)
+    
+    def extract(self, page, start_index=0, end_index=None, ignored_regions=None):
+        """extract data from an extraction page
+        
+        The region in the page to be extracted from may be specified using
+        start_index and end_index
+        """
+        ignored_regions = [LabelledRegion(*i) for i in (ignored_regions or [])]
+        region_elements = sorted(self.extractors + ignored_regions, key=lambda x: _labelled(x).start_index)
+        _, _, attributes = self._doextract(page, region_elements, start_index, 
+                end_index)
+        # collect variant data, maintaining the order of variants
+        variant_ids = []; variants = {}; items = []
+        for k, v in attributes:
+            if isinstance(k, int):
+                if k in variants:
+                    variants[k] += v
+                else:
+                    variant_ids.append(k)
+                    variants[k] = v
+            else:
+                items.append((k, v))
+        
+        variant_records = [('variants', _attrs2dict(variants[vid])) \
+                for vid in variant_ids]
+        items += variant_records
+        return [_attrs2dict(items)]
+    
+    def _doextract(self, page, region_elements, start_index, end_index, nested_regions=None, ignored_regions=None):
+        """Carry out extraction of records using the given annotations
+        in the page tokens bounded by start_index and end_index
+        """
+        # reorder extractors leaving nested ones for the end and separating
+        # ignore regions
+        nested_regions = nested_regions or []
+        ignored_regions = ignored_regions or []
+        first_region, following_regions = region_elements[0], region_elements[1:]
+        while following_regions and _labelled(following_regions[0]).start_index \
+                < _labelled(first_region).end_index:
+            region = following_regions.pop(0)
+            labelled = _labelled(region)
+            if isinstance(labelled, AnnotationTag) or (nested_regions and \
+                    _labelled(nested_regions[-1]).start_index < labelled.start_index \
+                    < _labelled(nested_regions[-1]).end_index):
+                nested_regions.append(region)
+            else:
+                ignored_regions.append(region)
+        extracted_data = []
+        # end_index is inclusive, but similar_region treats it as exclusive
+        end_region = None if end_index is None else end_index + 1
+        labelled = _labelled(first_region)
+        score, pindex, sindex = \
+            similar_region(page.page_tokens, self.template_tokens,
+                labelled, start_index, end_region)
+        if score > 0:
+            if isinstance(labelled, AnnotationTag):
+                similar_ignored_regions = []
+                start = pindex
+                for i in ignored_regions:
+                    s, p, e = similar_region(page.page_tokens, self.template_tokens, \
+                              i, start, sindex)
+                    if s > 0:
+                        similar_ignored_regions.append(LabelledRegion(*(p, e)))
+                        start = e or start
+                extracted_data = first_region.extract(page, pindex, sindex, similar_ignored_regions)
+                if extracted_data:
+                    if first_region.annotation.variant_id:
+                        extracted_data = [(first_region.annotation.variant_id, extracted_data)]
+            
+            if nested_regions:
+                _, _, nested_data = self._doextract(page, nested_regions, pindex, sindex)
+                extracted_data += nested_data
+            if following_regions:
+                _, _, following_data = self._doextract(page, following_regions, sindex or start_index, end_region)
+                extracted_data += following_data
+        
+        elif following_regions:
+            end_index, _, following_data = self._doextract(page, following_regions, start_index, end_region)
+            if end_index is not None:
+                pindex, sindex, extracted_data = self._doextract(page, [first_region], start_index, end_index - 1, nested_regions, ignored_regions)
+                extracted_data += following_data
+        elif nested_regions:
+            _, _, nested_data = self._doextract(page, nested_regions, start_index, end_region)
+            extracted_data += nested_data
+        return pindex, sindex, extracted_data
+                
+    @classmethod
+    def apply(cls, template, extractors):
+        return [cls(extractors, template.page_tokens)]
+    
+    def extracted_item(self):
+        return [self.__class__.__name__] + \
+                sorted(e.extracted_item() for e in self.extractors)
+    
+    def __repr__(self):
+        return str(self)
+
+    def __str__(self):
+        stream = cStringIO.StringIO()
+        pprint.pprint(self.extractors, stream)
+        stream.seek(0)
+        template_data = stream.read()
+        if template_data:
+            return "%s[\n%s\n]" % (self.__class__.__name__, template_data)
+        return "%s[none]" % (self.__class__.__name__)
+
+class AdjacentVariantExtractor(RecordExtractor):
+    """Extractor for variants
+
+    This simply extends the RecordExtractor to output data in a "variants"
+    attribute.
+
+    The "apply" method will only apply to variants whose items are all adjacent and 
+    it will appear as one record so that it can be handled by the RepeatedDataExtractor. 
+    """
+
+    def extract(self, page, start_index=0, end_index=None, ignored_regions=None):
+        records = RecordExtractor.extract(self, page, start_index, end_index, ignored_regions)
+        return [('variants', r['variants'][0]) for r in records if r]
+    
+    @classmethod
+    def apply(cls, template, extractors):
+        adjacent_variants = set([])
+        variantf = lambda x: x.annotation.variant_id
+        for vid, egroup in groupby(extractors, variantf):
+            if not vid:
+                continue
+            if vid in adjacent_variants:
+                adjacent_variants.remove(vid)
+            elif len(list(egroup)) > 1:
+                adjacent_variants.add(vid)
+        
+        new_extractors = []
+        for variant, group_seq in groupby(extractors, variantf):
+            group_seq = list(group_seq)
+            if variant in adjacent_variants:
+                record_extractor = AdjacentVariantExtractor(group_seq, template.page_tokens)
+                new_extractors.append(record_extractor)
+            else:
+                new_extractors += group_seq
+        return new_extractors
+
+    def __repr__(self):
+        return str(self)
+
+class TraceExtractor(object):
+    """Extractor that wraps other extractors and prints an execution
+    trace of the extraction process to aid debugging
+    """
+
+    def __init__(self, traced, template):
+        self.traced = traced
+        self.annotation = traced.annotation
+        tstart = traced.annotation.start_index
+        tend = traced.annotation.end_index
+        self.tprefix = " ".join([template.token_dict.token_string(t)
+            for t in template.page_tokens[tstart-4:tstart+1]])
+        self.tsuffix = " ".join([template.token_dict.token_string(t)
+            for t in template.page_tokens[tend:tend+5]])
+    
+    def summarize_trace(self, page, start, end, ret):
+        text_start = page.token_follow_indexes[start]
+        text_end = page.token_start_indexes[end or -1]
+        page_snippet = "(...%s)%s(%s...)" % (
+                page.text[text_start-50:text_start].replace('\n', ' '), 
+                page.text[text_start:text_end], 
+                page.text[text_end:text_end+50].replace('\n', ' '))
+        pre_summary = "\nstart %s page[%s:%s]\n" % (self.traced.__class__.__name__, start, end)
+        post_summary = """
+%s page[%s:%s] 
+
+html
+%s
+
+annotation
+...%s
+%s
+%s...
+
+extracted
+%s
+        """ % (self.traced.__class__.__name__, start, end, page_snippet, 
+                self.tprefix, self.annotation, self.tsuffix, [r for r in ret if 'trace' not in r])
+        return pre_summary, post_summary
+
+    def extract(self, page, start, end, ignored_regions):
+        ret = self.traced.extract(page, start, end, ignored_regions)
+        if not ret:
+            return []
+
+        # handle records by inserting a trace and combining with variant traces
+        if len(ret) == 1 and isinstance(ret[0], dict):
+            item = ret[0]
+            trace = item.pop('trace', [])
+            variants = item.get('variants', ())
+            for variant in variants:
+                trace += variant.pop('trace', [])
+            pre_summary, post_summary = self.summarize_trace(page, start, end, ret)
+            item['trace'] = [pre_summary] + trace + [post_summary]
+            return ret
+        
+        pre_summary, post_summary = self.summarize_trace(page, start, end, ret)
+        return [('trace', pre_summary)] + ret + [('trace', post_summary)]
+    
+    @staticmethod
+    def apply(template, extractors):
+        output = []
+        for extractor in extractors:
+            if not isinstance(extractor, TraceExtractor):
+                extractor = TraceExtractor(extractor, template)
+            output.append(extractor)
+        return output
+    
+    def extracted_item(self):
+        return self.traced.extracted_item()
+
+    def __repr__(self):
+        return "Trace(%s)" % repr(self.traced)
+
+class TemplatePageExtractor(object):
+    """Top level extractor for a template page"""
+
+    def __init__(self, template, extractors):
+        # fixme: handle multiple items per page
+        self.extractor = extractors[0]
+        self.template = template
+
+    def extract(self, page, start_index=0, end_index=None):
+        return self.extractor.extract(page, start_index, end_index, self.template.ignored_regions)
+    
+    def __repr__(self):
+        return repr(self.extractor)
+
+    def __str__(self):
+        return str(self.extractor)
+
+_tokenize = nltk.tokenize.WordPunctTokenizer().tokenize
+
+class TextRegionDataExtractor(object):
+    """Data Extractor for extracting text fragments from within a larger
+    body of text. It extracts based on the longest unique prefix and suffix.
+
+    for example:
+    >>> extractor = TextRegionDataExtractor('designed by ', '.')
+    >>> extractor.extract("by Marc Newson.")
+    'Marc Newson'
+
+    Both prefix and suffix are optional:
+    >>> extractor = TextRegionDataExtractor('designed by ')
+    >>> extractor.extract("by Marc Newson.")
+    'Marc Newson.'
+    >>> extractor = TextRegionDataExtractor(suffix='.')
+    >>> extractor.extract("by Marc Newson.")
+    'by Marc Newson'
+
+    It requires a minimum match of at least one word or punctuation character:
+    >>> extractor = TextRegionDataExtractor('designed by')
+    >>> extractor.extract("y Marc Newson.") is None
+    True
+    """
+    def __init__(self, prefix=None, suffix=None):
+        self.prefix = (prefix or '')[::-1]
+        self.suffix = suffix or ''
+        self.minprefix = self.minmatch(self.prefix)
+        self.minsuffix = self.minmatch(self.suffix)
+ 
+    @staticmethod
+    def minmatch(matchstring):
+        """the minimum number of characters that should match in order
+        to consider it a match for that string.
+
+        This uses the last word of punctuation character
+        """
+        tokens = _tokenize(matchstring or '')
+        return len(tokens[0]) if tokens else 0
+
+    def extract(self, text):
+        """attempt to extract a substring from the text"""
+        pref_index = 0
+        if self.minprefix > 0:
+            rev_idx, plen = longest_unique_subsequence(text[::-1], self.prefix)
+            if plen < self.minprefix:
+                return None
+            pref_index = -rev_idx
+        if self.minsuffix == 0:
+            return text[pref_index:]
+        sidx, slen = longest_unique_subsequence(text[pref_index:], self.suffix)
+        if slen < self.minsuffix:
+            return None
+        return text[pref_index:pref_index + sidx]
+
+class TextPrefixRegionDataExtractor(object):
+    """
+    Data extractor for extracting text fragment from within a
+    larger body of text, based on a fixed prefix.
+    >>> extractor = TextPrefixRegionDataExtractor("&pounds;")
+    >>> extractor.extract("&pounds; 17.00")
+    ' 17.00'
+    >>> extractor.extract("&euro; 17.00") is None
+    True
+    >>> extractor.extract("$ 17.00") is None
+    True
+    >>> extractor.extract(" &pounds; 17.00 ")
+    ' 17.00 '
+    """
+    def __init__(self, prefix):
+        self.prefix = prefix
+    def extract(self, text):
+        text = text.lstrip()
+        # attempt to extract a substring from the text
+        if text.startswith(self.prefix):
+            return text.replace(self.prefix, '')
+    

scrapy/contrib/ibl/extraction/similarity.py

@@ -0,0 +1,136 @@
+"""
+Similarity calculation for Instance based extraction algorithm.
+"""
+from itertools import izip, count
+from operator import itemgetter
+from heapq import nlargest
+
+def common_prefix_length(a, b):
+    """Calculate the length of the common prefix in both sequences passed.
+    
+    For example, the common prefix in this example is [1, 3]
+    >>> common_prefix_length([1, 3, 4], [1, 3, 5, 1])
+    2
+    
+    If there is no common prefix, 0 is returned
+    >>> common_prefix_length([1], [])
+    0
+    """
+    i = -1
+    for i, x, y in izip(count(), a, b):
+        if x != y:
+            return i
+    return i + 1
+
+def common_prefix(*sequences):
+    """determine the common prefix of all sequences passed
+    
+    For example:
+    >>> common_prefix('abcdef', 'abc', 'abac')
+    ['a', 'b']
+    """
+    prefix = []
+    for sample in izip(*sequences):
+        first = sample[0]
+        if all(x == first for x in sample[1:]):
+            prefix.append(first)
+        else:
+            break
+    return prefix
+
+def longest_unique_subsequence(to_search, subsequence, range_start=0, 
+        range_end=None):
+    """Find the longest unique subsequence of items in a list or array.  This
+    searches the to_search list or array looking for the longest overlapping
+    match with subsequence. If the largest match is unique (there is no other
+    match of equivalent length), the index and length of match is returned.  If
+    there is no match, (None, None) is returned.
+
+    Please see section 3.2 of Extracting Web Data Using Instance-Based
+    Learning by Yanhong Zhai and Bing Liu
+
+    For example, the longest match occurs at index 2 and has length 3
+    >>> to_search = [6, 3, 2, 4, 3, 2, 5]
+    >>> longest_unique_subsequence(to_search, [2, 4, 3])
+    (2, 3)
+    
+    When there are two equally long subsequences, it does not generate a match
+    >>> longest_unique_subsequence(to_search, [3, 2])
+    (None, None)
+
+    range_start and range_end specify a range in which the match must begin
+    >>> longest_unique_subsequence(to_search, [3, 2], 3)
+    (4, 2)
+    >>> longest_unique_subsequence(to_search, [3, 2], 0, 2)
+    (1, 2)
+    """
+    startval = subsequence[0]
+    if range_end is None:
+        range_end = len(to_search)
+    
+    # the comparison to startval ensures only matches of length >= 1 and 
+    # reduces the number of calls to the common_length function
+    matches = ((i, common_prefix_length(to_search[i:], subsequence)) \
+        for i in xrange(range_start, range_end) if startval == to_search[i])
+    best2 = nlargest(2, matches, key=itemgetter(1))
+    # if there is a single unique best match, return that
+    if len(best2) == 1 or len(best2) == 2 and best2[0][1] != best2[1][1]:
+        return best2[0]
+    return None, None
+
+def similar_region(extracted_tokens, template_tokens, labelled_region, 
+        range_start=0, range_end=None):
+    """Given a labelled section in a template, identify a similar region
+    in the extracted tokens.
+
+    The start and end index of the similar region in the extracted tokens
+    is returned.
+
+    This will return a tuple containing:
+    (match score, start index, end index)
+    where match score is the sum of the length of the matching prefix and 
+    suffix. If there is no unique match, (0, None, None) will be returned.
+
+    start_index and end_index specify a range in which the match must begin
+    """
+    data_length = len(extracted_tokens)
+    if range_end is None:
+        range_end = data_length
+    # calculate the prefix score by finding a longest subsequence in 
+    # reverse order
+    reverse_prefix = template_tokens[labelled_region.start_index::-1]
+    reverse_tokens = extracted_tokens[::-1]
+    (rpi, pscore) = longest_unique_subsequence(reverse_tokens, reverse_prefix,
+            data_length - range_end, data_length - range_start)
+
+    # None means nothing exracted. Index 0 means there cannot be a suffix.
+    if not rpi:
+        return 0, None, None
+    
+    # convert to an index from the start instead of in reverse
+    prefix_index = len(extracted_tokens) - rpi - 1
+
+    if labelled_region.end_index is None:
+        return pscore, prefix_index, None
+
+    suffix = template_tokens[labelled_region.end_index:]
+
+    # if it's not a paired tag, use the best match between prefix & suffix
+    if labelled_region.start_index == labelled_region.end_index:
+        (match_index, sscore) = longest_unique_subsequence(extracted_tokens,
+            suffix, prefix_index, range_end)
+        if match_index == prefix_index:
+            return (pscore + sscore, prefix_index, match_index)
+        elif pscore > sscore:
+            return pscore, prefix_index, prefix_index
+        elif sscore > pscore:
+            return sscore, match_index, match_index
+        return 0, None, None
+
+    # calculate the suffix match on the tokens following the prefix. We could
+    # consider the whole page and require a good match.
+    (match_index, sscore) = longest_unique_subsequence(extracted_tokens,
+            suffix, prefix_index + 1, range_end)
+    if match_index is None:
+        return 0, None, None
+    return (pscore + sscore, prefix_index, match_index)

scrapy/contrib/ibl/extractors.py

@@ -0,0 +1,156 @@
+"""
+Extractors for attributes
+"""
+import re
+import urlparse
+
+from scrapy.utils.markup import remove_entities
+from scrapy.utils.url import safe_url_string
+
+#FIXME: the use of "." needs to be localized
+_NUMERIC_ENTITIES = re.compile("&#([0-9]+)(?:;|\s)", re.U)
+_PRICE_NUMBER_RE = re.compile('(?:^|[^a-zA-Z0-9])(\d+(?:\.\d+)?)(?:$|[^a-zA-Z0-9])')
+_NUMBER_RE = re.compile('(\d+(?:\.\d+)?)')
+
+_IMAGES = (
+    'mng', 'pct', 'bmp', 'gif', 'jpg', 'jpeg', 'png', 'pst', 'psp', 'tif',
+    'tiff', 'ai', 'drw', 'dxf', 'eps', 'ps', 'svg',
+)
+
+_IMAGES_TYPES = '|'.join(_IMAGES)
+_CSS_IMAGERE = re.compile("background(?:-image)?\s*:\s*url\((.*?)\)", re.I)
+_BASE_PATH_RE = "/?(?:[^/]+/)*(?:.+%s)"
+_IMAGE_PATH_RE = re.compile(_BASE_PATH_RE % '\.(?:%s)' % _IMAGES_TYPES, re.I)
+_GENERIC_PATH_RE = re.compile(_BASE_PATH_RE % '', re.I)
+
+def text(txt):
+    stripped = txt.strip() if txt else None
+    if stripped:
+        return stripped
+
+def contains_any_numbers(txt):
+    """text that must contain at least one number
+    >>> contains_any_numbers('foo')
+    >>> contains_any_numbers('$67 at 15% discount')
+    '$67 at 15% discount'
+    """
+    if _NUMBER_RE.search(txt) is not None:
+        return txt
+
+def contains_prices(txt):
+    """text must contain a number that is not joined to text"""
+    if _PRICE_NUMBER_RE.findall(txt) is not None:
+        return txt
+
+def contains_numbers(txt, count=1):
+    """Must contain a certain amount of numbers
+    
+    >>> contains_numbers('foo', 2)
+    >>> contains_numbers('this 1 has 2 numbers', 2)
+    'this 1 has 2 numbers'
+    """
+    numbers = _NUMBER_RE.findall(txt)
+    if len(numbers) == count:
+        return txt
+
+def extract_number(txt):
+    """Extract a numeric value.
+    
+    This will fail if more than one numeric value is present.
+
+    >>> extract_number('  45.3')
+    '45.3'
+    >>> extract_number('  45.3, 7')
+
+    It will handle unescaped entities:
+    >>> extract_number(u'£129.99')
+    u'129.99'
+    """
+    txt = _NUMERIC_ENTITIES.sub(lambda m: unichr(int(m.groups()[0])), txt)
+    numbers = _NUMBER_RE.findall(txt)
+    if len(numbers) == 1:
+        return numbers[0]
+    
+def url(txt):
+    """convert text to a url
+    
+    this is quite conservative, since relative urls are supported
+    """
+    txt = txt.strip("\t\r\n '\"")
+    if txt:
+        return txt
+
+def image_url(txt):
+    """convert text to a url
+    
+    this is quite conservative, since relative urls are supported
+    Example:
+
+        >>> image_url('')
+
+        >>> image_url('   ')
+
+        >>> image_url(' \\n\\n  ')
+
+        >>> image_url('foo-bar.jpg')
+        ['foo-bar.jpg']
+        >>> image_url('/images/main_logo12.gif')
+        ['/images/main_logo12.gif']
+        >>> image_url("http://www.image.com/image.jpg")
+        ['http://www.image.com/image.jpg']
+        >>> image_url("http://www.domain.com/path1/path2/path3/image.jpg")
+        ['http://www.domain.com/path1/path2/path3/image.jpg']
+        >>> image_url("/path1/path2/path3/image.jpg")
+        ['/path1/path2/path3/image.jpg']
+        >>> image_url("path1/path2/image.jpg")
+        ['path1/path2/image.jpg']
+        >>> image_url("background-image : url(http://www.site.com/path1/path2/image.jpg)")
+        ['http://www.site.com/path1/path2/image.jpg']
+        >>> image_url("background-image : url('http://www.site.com/path1/path2/image.jpg')")
+        ['http://www.site.com/path1/path2/image.jpg']
+        >>> image_url('background-image : url("http://www.site.com/path1/path2/image.jpg")')
+        ['http://www.site.com/path1/path2/image.jpg']
+        >>> image_url("background : url(http://www.site.com/path1/path2/image.jpg)")
+        ['http://www.site.com/path1/path2/image.jpg']
+        >>> image_url("background : url('http://www.site.com/path1/path2/image.jpg')")
+        ['http://www.site.com/path1/path2/image.jpg']
+        >>> image_url('background : url("http://www.site.com/path1/path2/image.jpg")')
+        ['http://www.site.com/path1/path2/image.jpg']
+        >>> image_url('/getimage.php?image=totalgardens/outbbq2_400.jpg&type=prod&resizeto=350')
+        ['/getimage.php?image=totalgardens/outbbq2_400.jpg&type=prod&resizeto=350']
+        >>> image_url('http://www.site.com/getimage.php?image=totalgardens/outbbq2_400.jpg&type=prod&resizeto=350')
+        ['http://www.site.com/getimage.php?image=totalgardens/outbbq2_400.jpg&type=prod&resizeto=350']
+        >>> image_url('http://s7d4.scene7.com/is/image/Kohler/jaa03267?hei=425&wid=457&op_usm=2,1,2,1&qlt=80')
+        ['http://s7d4.scene7.com/is/image/Kohler/jaa03267?hei=425&wid=457&op_usm=2,1,2,1&qlt=80']
+        >>> image_url('../image.aspx?thumb=true&boxSize=175&img=Unknoportrait[1].jpg')
+        ['../image.aspx?thumb=true&boxSize=175&img=Unknoportrait%5B1%5D.jpg']
+        >>> image_url('http://www.sundancecatalog.com/mgen/catalog/test.ms?args=%2245932|MERIDIAN+PENDANT|.jpg%22&is=336,336,0xffffff')
+        ['http://www.sundancecatalog.com/mgen/catalog/test.ms?args=%2245932|MERIDIAN+PENDANT|.jpg%22&is=336,336,0xffffff']
+        >>> image_url('http://www.site.com/image.php')
+        ['http://www.site.com/image.php']
+        >>> image_url('background-image:URL(http://s7d5.scene7.com/is/image/wasserstrom/165133?wid=227&hei=227&defaultImage=noimage_wasserstrom)')
+        ['http://s7d5.scene7.com/is/image/wasserstrom/165133?wid=227&hei=227&defaultImage=noimage_wasserstrom']
+
+    """
+    txt = url(txt)
+    imgurl = None
+    if txt:
+        # check if the text is style content
+        m = _CSS_IMAGERE.search(txt)
+        txt = m.groups()[0] if m else txt
+        parsed = urlparse.urlparse(txt)
+        path = None
+        m = _IMAGE_PATH_RE.search(parsed.path)
+        if m:
+            path = m.group()
+        elif parsed.query:
+            m = _GENERIC_PATH_RE.search(parsed.path)
+            if m:
+                path = m.group()
+        if path is not None:
+            parsed = list(parsed)
+            parsed[2] = path
+            imgurl = urlparse.urlunparse(parsed)
+        if not imgurl:
+            imgurl = txt
+    return [safe_url_string(remove_entities(url(imgurl)))] if imgurl else None

scrapy/contrib/ibl/htmlpage.py

@@ -0,0 +1,212 @@
+"""
+htmlpage
+
+Container object for representing html pages in the IBL system. This
+encapsulates page related information and prevents parsing multiple times.
+"""
+import re
+import hashlib
+
+from scrapy.utils.python import str_to_unicode
+
+def create_page_from_jsonpage(jsonpage, body_key):
+    """Create an HtmlPage object from a dict object conforming to the schema
+    for a page
+
+    `body_key` is the key where the body is stored and can be either 'body'
+    (original page with annotations - if any) or 'original_body' (original
+    page, always). Classification typically uses 'original_body' to avoid
+    confusing the classifier with annotated pages, while extraction uses 'body'
+    to pass the annotated pages.
+    """
+    url = jsonpage['url']
+    headers = jsonpage.get('headers')
+    body = str_to_unicode(jsonpage[body_key])
+    page_id = jsonpage.get('page_id')
+    return HtmlPage(url, headers, body, page_id)
+
+class HtmlPage(object):
+    def __init__(self, url=None, headers=None, body=None, page_id=None):
+        assert isinstance(body, unicode), "unicode expected, got: %s" % type(body).__name__
+        self.headers = headers or {}
+        self.body = body
+        self.url = url or u''
+        if page_id is None and url:
+            self.page_id = hashlib.sha1(url).hexdigest()
+        else:
+            self.page_id = page_id 
+    
+
+    def _set_body(self, body):
+        self._body = body
+        self.parsed_body = list(parse_html(body))
+        
+    body = property(lambda x: x._body, _set_body)
+    
+    def fragment_data(self, data_fragment):
+        return self.body[data_fragment.start:data_fragment.end]
+    
+class HtmlTagType(object):
+    OPEN_TAG = 1
+    CLOSE_TAG = 2 
+    UNPAIRED_TAG = 3
+
+class HtmlDataFragment(object):
+    __slots__ = ('start', 'end')
+    
+    def __init__(self, start, end):
+        self.start = start
+        self.end = end
+        
+    def __str__(self):
+        return "<HtmlDataFragment [%s:%s]>" % (self.start, self.end)
+
+    def __repr__(self):
+        return str(self)
+    
+class HtmlTag(HtmlDataFragment):
+    __slots__ = ('tag_type', 'tag', 'attributes')
+
+    def __init__(self, tag_type, tag, attributes, start, end):
+        HtmlDataFragment.__init__(self, start, end)
+        self.tag_type = tag_type
+        self.tag = tag
+        self.attributes = attributes
+
+    def __str__(self):
+        return "<HtmlTag tag='%s' attributes={%s} [%s:%s]>" % (self.tag, ', '.join(sorted\
+                (["%s: %s" % (k, repr(v)) for k, v in self.attributes.items()])), self.start, self.end)
+    
+    def __repr__(self):
+        return str(self)
+
+_ATTR = "((?:[^=/>\s]|/(?!>))+)(?:\s*=(?:\s*\"(.*?)\"|\s*'(.*?)'|([^>\s]+))?)?"
+_TAG = "<(\/?)(\w+(?::\w+)?)((?:\s+" + _ATTR + ")+\s*|\s*)(\/?)>"
+_DOCTYPE = r"<!DOCTYPE.*?>"
+
+_ATTR_REGEXP = re.compile(_ATTR, re.I | re.DOTALL)
+_HTML_REGEXP = re.compile(_TAG, re.I | re.DOTALL)
+_DOCTYPE_REGEXP = re.compile("(?:%s)" % _DOCTYPE)
+_COMMENT_RE = re.compile("(<!--.*?-->)", re.DOTALL)
+_SCRIPT_RE = re.compile("(<script.*?>).*?(</script.*?>)", re.DOTALL | re.I)
+
+def parse_html(text):
+    """Higher level html parser. Calls lower level parsers and joins sucesive
+    HtmlDataFragment elements in a single one.
+    """
+    script_layer = lambda x: _parse_clean_html(x, _SCRIPT_RE, HtmlTag, _simple_parse_html)
+    comment_layer = lambda x: _parse_clean_html(x, _COMMENT_RE, HtmlDataFragment, script_layer)
+    delayed_element = None
+    for element in comment_layer(text):
+        if isinstance(element, HtmlTag):
+            if delayed_element is not None:
+                yield delayed_element
+                delayed_element = None
+            yield element
+        else:# element is HtmlDataFragment
+            if delayed_element is not None:
+                delayed_element.start = min(element.start, delayed_element.start)
+                delayed_element.end = max(element.end, delayed_element.end)
+            else:
+                delayed_element = element
+    if delayed_element is not None:
+        yield delayed_element
+
+def _parse_clean_html(text, regex, htype, func):
+    """
+    Removes regions from text, passes the cleaned text to the lower parse layer,
+    and reinserts removed regions.
+    regex - regular expression that defines regions to be removed/re inserted
+    htype - the html parser type of the removed elements
+    func - function that performs the lower parse layer
+    """
+    removed = [[m.start(), m.end(), m.groups()] for m in regex.finditer(text)]
+    
+    cleaned = regex.sub("", text)
+    shift = 0
+    for element in func(cleaned):
+        element.start += shift
+        element.end += shift
+        while removed:
+            if element.end <= removed[0][0]:
+                yield element
+                break
+            else:
+                start, end, groups = removed.pop(0)
+                add = end - start
+                element.end += add
+                shift += add
+                if element.start >= start:
+                    element.start += add
+                elif isinstance(element, HtmlTag):
+                    yield element
+                    break
+                
+                if element.start < start:
+                    yield HtmlDataFragment(element.start, start)
+                    element.start = end
+                
+                if htype == HtmlTag:
+                    begintag = _parse_tag(_HTML_REGEXP.match(groups[0]))
+                    endtag = _parse_tag(_HTML_REGEXP.match(groups[1]))
+                    begintag.start = start
+                    begintag.end += start
+                    
+                    endtag.start = end - endtag.end
+                    endtag.end = end
+                    content = None
+                    if begintag.end < endtag.start:
+                        content = HtmlDataFragment(begintag.end, endtag.start)
+                    yield begintag
+                    if content is not None:
+                        yield content
+                    yield endtag
+                else:
+                    yield htype(start, end)
+        else:
+            yield element
+
+def _simple_parse_html(text):
+    """Simple html parse. It returns a sequence of HtmlTag and HtmlDataFragment
+    objects. Does not ignore any region.
+    """
+    # If have doctype remove it.
+    start_pos = 0
+    match = _DOCTYPE_REGEXP.match(text)
+    if match:
+        start_pos = match.end()
+    prev_end = start_pos
+    for match in _HTML_REGEXP.finditer(text, start_pos):
+        start = match.start()
+        end = match.end()
+            
+        if start > prev_end:
+            yield HtmlDataFragment(prev_end, start)
+        
+        yield _parse_tag(match)
+        prev_end = end
+    textlen = len(text)
+    if prev_end < textlen:
+        yield HtmlDataFragment(prev_end, textlen)
+
+def _parse_tag(match):
+    """
+    parse a tag matched by _HTML_REGEXP
+    """
+    data = match.groups()
+    closing, tag, attr_text = data[:3]
+    # if tag is None then the match is a comment
+    if tag is not None:
+        unpaired = data[-1]
+        if closing:
+            tag_type = HtmlTagType.CLOSE_TAG
+        elif unpaired:
+            tag_type = HtmlTagType.UNPAIRED_TAG
+        else:
+            tag_type = HtmlTagType.OPEN_TAG
+        attributes = []
+        for attr_match in _ATTR_REGEXP.findall(attr_text):
+            name = attr_match[0].lower()
+            values = [v for v in attr_match[1:] if v]
+            attributes.append((name, values[0] if values else None))
+        return HtmlTag(tag_type, tag.lower(), dict(attributes), match.start(), match.end())

scrapy/tests/test_contrib_ibl/__init__.py

@@ -0,0 +1,2 @@
+import sys
+path = sys.modules[__name__].__path__[0]

scrapy/tests/test_contrib_ibl/test_extraction.py

@@ -0,0 +1,819 @@
+"""
+tests for page parsing
+
+Page parsing effectiveness is measured through the evaluation system. These
+tests should focus on specific bits of functionality work correctly.
+"""
+from unittest import TestCase
+from scrapy.contrib.ibl.htmlpage import HtmlPage
+from scrapy.contrib.ibl.extraction import InstanceBasedLearningExtractor
+from scrapy.contrib.ibl.descriptor import (FieldDescriptor as A, 
+        ItemDescriptor)
+from scrapy.contrib.ibl.extractors import (contains_any_numbers,
+        image_url)
+        
+# simple page with all features
+
+ANNOTATED_PAGE1 = u"""
+<html>
+<h1>COMPANY - <ins 
+    data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;generated&quot;: true, 
+    &quot;annotations&quot;: {&quot;content&quot;: &quot;title&quot;}}" 
+>Item Title</ins></h1>
+<p>introduction</p>
+<div>
+<img data-scrapy-annotate="{&quot;variant&quot;: 0, 
+    &quot;annotations&quot;: {&quot;src&quot;: &quot;image_url&quot;}}"
+    src="img.jpg"/>
+<p data-scrapy-annotate="{&quot;variant&quot;: 0, 
+    &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">
+This is such a nice item<br/> Everybody likes it.
+</p>
+<br/>
+</div>
+<p>click here for other items</p>
+</html>
+"""
+
+EXTRACT_PAGE1 = u"""
+<html>
+<h1>Scrapy - Nice Product</h1>
+<p>introduction</p>
+<div>
+<img src="nice_product.jpg" alt="a nice product image"/>
+<p>wonderful product</p>
+<br/>
+</div>
+</html>
+"""
+
+# single tag with multiple items extracted
+ANNOTATED_PAGE2 = u"""
+<a href="http://example.com/xxx" title="xxx"
+    data-scrapy-annotate="{&quot;variant&quot;: 0, 
+    &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;, 
+        &quot;href&quot;: &quot;image_url&quot;, &quot;title&quot;: &quot;name&quot;}}"
+>xx</a>
+xxx
+</a>
+"""
+EXTRACT_PAGE2 = u"""<a href='http://example.com/product1.jpg' 
+    title="product 1">product 1 is great</a>"""
+
+# matching must match the second attribute in order to find the first
+ANNOTATED_PAGE3 = u"""
+<p data-scrapy-annotate="{&quot;variant&quot;: 0, 
+    &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">xx</p>
+<div data-scrapy-annotate="{&quot;variant&quot;: 0, 
+    &quot;annotations&quot;: {&quot;content&quot;: &quot;delivery&quot;}}">xx</div>
+"""
+EXTRACT_PAGE3 = u"""
+<p>description</p>
+<div>delivery</div>
+<p>this is not the description</p>
+"""
+
+# test inferring repeated elements
+ANNOTATED_PAGE4 = u"""
+<ul>
+<li data-scrapy-annotate="{&quot;variant&quot;: 0, 
+    &quot;annotations&quot;: {&quot;content&quot;: &quot;features&quot;}}">feature1</li>
+<li data-scrapy-annotate="{&quot;variant&quot;: 0, 
+    &quot;annotations&quot;: {&quot;content&quot;: &quot;features&quot;}}">feature2</li>
+</ul>
+"""
+
+EXTRACT_PAGE4 = u"""
+<ul>
+<li>feature1</li> ignore this
+<li>feature2</li>
+<li>feature3</li>
+</ul>
+"""
+
+# test variant handling with identical repeated variant
+ANNOTATED_PAGE5 =  u"""
+<p data-scrapy-annotate="{&quot;annotations&quot;:
+    {&quot;content&quot;: &quot;description&quot;}}">description</p>
+<table>
+<tr>
+<td data-scrapy-annotate="{&quot;variant&quot;: 1, &quot;annotations&quot;: 
+    {&quot;content&quot;: &quot;colour&quot;}}" >colour 1</td>
+<td data-scrapy-annotate="{&quot;variant&quot;: 1, &quot;annotations&quot;: 
+    {&quot;content&quot;: &quot;price&quot;}}" >price 1</td>
+</tr>
+<tr>
+<td data-scrapy-annotate="{&quot;variant&quot;: 2, &quot;annotations&quot;: 
+    {&quot;content&quot;: &quot;colour&quot;}}" >colour 2</td>
+<td data-scrapy-annotate="{&quot;variant&quot;: 2, &quot;annotations&quot;: 
+    {&quot;content&quot;: &quot;price&quot;}}" >price 2</td>
+</tr>
+</table>
+"""
+
+EXTRACT_PAGE5 = u"""
+<p>description</p>
+<table>
+<tr>
+<td>colour 1</td>
+<td>price 1</td>
+</tr>
+<tr>
+<td>colour 2</td>
+<td>price 2</td>
+</tr>
+<tr>
+<td>colour 3</td>
+<td>price 3</td>
+</tr>
+</table>
+"""
+
+# test variant handling with irregular structure and some non-variant
+# attributes
+ANNOTATED_PAGE6 =  u"""
+<p data-scrapy-annotate="{&quot;annotations&quot;:
+    {&quot;content&quot;: &quot;description&quot;}}">description</p>
+<p data-scrapy-annotate="{&quot;variant&quot;: 1, &quot;annotations&quot;: 
+    {&quot;content&quot;: &quot;name&quot;}}">name 1</p>
+<div data-scrapy-annotate="{&quot;variant&quot;: 3, &quot;annotations&quot;: 
+    {&quot;content&quot;: &quot;name&quot;}}" >name 3</div>
+<p data-scrapy-annotate="{&quot;variant&quot;: 2, &quot;annotations&quot;: 
+    {&quot;content&quot;: &quot;name&quot;}}" >name 2</p>
+"""
+EXTRACT_PAGE6 =  u"""
+<p>description</p>
+<p>name 1</p>
+<div>name 3</div>
+<p>name 2</p>
+"""
+
+# test repeating variants at the table column level
+ANNOTATED_PAGE7 =  u"""
+<table>
+<tr>
+<td data-scrapy-annotate="{&quot;variant&quot;: 1, &quot;annotations&quot;: 
+    {&quot;content&quot;: &quot;colour&quot;}}" >colour 1</td>
+<td data-scrapy-annotate="{&quot;variant&quot;: 2, &quot;annotations&quot;: 
+    {&quot;content&quot;: &quot;colour&quot;}}" >colour 2</td>
+</tr>
+<tr>
+<td data-scrapy-annotate="{&quot;variant&quot;: 2, &quot;annotations&quot;: 
+    {&quot;content&quot;: &quot;price&quot;}}" >price 1</td>
+<td data-scrapy-annotate="{&quot;variant&quot;: 2, &quot;annotations&quot;: 
+    {&quot;content&quot;: &quot;price&quot;}}" >price 2</td>
+</tr>
+</table>
+"""
+
+EXTRACT_PAGE7 = u"""
+<table>
+<tr>
+<td>colour 1</td>
+<td>colour 2</td>
+<td>colour 3</td>
+</tr>
+<tr>
+<td>price 1</td>
+<td>price 2</td>
+<td>price 3</td>
+</tr>
+</table>
+"""
+
+ANNOTATED_PAGE8 = u"""
+<html><body>
+<h1>A product</h1>
+<div data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">
+<p>XXXX XXXX xxxxx</p>
+<div data-scrapy-ignore="true">
+<img scr="image.jpg" /><br/><a link="back.html">Click here to go back</a>
+</div>
+</div>
+<div data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}">
+10.00<p data-scrapy-ignore="true"> 13</p>
+</div>
+</body></html>
+"""
+
+EXTRACT_PAGE8 = u"""
+<html><body>
+<h1>A product</h1>
+<div>
+<p>A very nice product for all intelligent people</p>
+<div>
+<img scr="image.jpg" /><br/><a link="back.html">Click here to go back</a>
+</div>
+</div>
+<div>
+12.00<p> ID 15</p>
+(VAT exc.)</div>
+</body></html>
+"""
+
+ANNOTATED_PAGE9 = ANNOTATED_PAGE8
+
+EXTRACT_PAGE9 = u"""
+<html><body>
+<img src="logo.jpg" />
+<h1>A product</h1>
+<div>
+<p>A very nice product for all intelligent people</p>
+<div>
+<img scr="image.jpg" /><br/><a link="back.html">Click here to go back</a>
+</div>
+</div>
+<div>
+12.00<p> ID 16</p>
+(VAT exc.)</div>
+</body></html>
+"""
+
+ANNOTATED_PAGE10a = u"""
+<html><body>
+<table><tbody>
+<tr data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;common_prefix&quot;: true, &quot;annotations&quot;: {&quot;content&quot;: &quot;site_id&quot;}}">
+<td>SKU</td><td>L345</td>
+</tr>
+<tr data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;common_prefix&quot;: true, &quot;annotations&quot;: {&quot;content&quot;: &quot;dimensions&quot;}}">
+<td>Size</td><td>10cmx20cm</td>
+</tr>
+<tr data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;common_prefix&quot;: true, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}">
+<td>Price</td><td>&pounds;99.00</td>
+</tr>
+</tbody></table>
+</body></html>
+"""
+
+ANNOTATED_PAGE10b = u"""
+<html><body>
+<table><tbody>
+<tr data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;common_prefix&quot;: true, &quot;annotations&quot;: {&quot;content&quot;: &quot;site_id&quot;}}">
+<td>SKU</td><td>S220</td>
+</tr>
+<tr data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;common_prefix&quot;: true, &quot;annotations&quot;: {&quot;content&quot;: &quot;dimensions&quot;}}">
+<td>Size</td><td>20cmx20cm</td>
+</tr>
+<tr data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;common_prefix&quot;: true, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}">
+<td>Price</td><td>&pounds;85.00</td>
+</tr>
+</tbody></table>
+</body></html>
+"""
+
+EXTRACT_PAGE10a = u"""
+<html><body>
+<table><tbody>
+<tr>
+<td>Offer</td><td>From $2500.00</td>
+</tr>
+<tr>
+<td>Description</td><td>Electrorheological Cyborgs</td>
+</tr>
+<tr>
+<td>Series:</td><td>T2000</td>
+</tr>
+</tbody></table>
+</body></html>
+"""
+
+EXTRACT_PAGE10b = u"""
+<html><body>
+<table><tbody>
+<tr>
+<td>SKU</td><td>K80</td>
+</tr>
+<tr>
+<td>Size</td><td>50cm</td>
+</tr>
+<tr>
+<td>Price</td><td>&euros;85.00</td>
+</tr>
+</tbody></table>
+</body></html>
+"""
+
+ANNOTATED_PAGE11 = u"""
+<html><body>
+<p data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">
+<ins data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;generated&quot;: true,
+    &quot;annotations&quot;: {&quot;content&quot;: &quot;name&quot;}}">
+SL342
+</ins>
+<br/>
+Nice product for ladies
+<br/><ins data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;generated&quot;: true,
+     &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}">
+&pounds;85.00
+</ins>
+</p>
+<ins data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;generated&quot;: true,
+     &quot;annotations&quot;: {&quot;content&quot;: &quot;price_before_discount&quot;}}">
+&pounds;100.00
+</ins>
+</body></html>
+"""
+
+EXTRACT_PAGE11 = u"""
+<html><body>
+<p>
+SL342
+<br/>
+Nice product for ladies
+<br/>
+&pounds;85.00
+</p>
+&pounds;100.00
+</body></html>
+"""
+
+ANNOTATED_PAGE12 = u"""
+<html><body>
+<h1 data-scrapy-ignore-beneath="true">A product</h1>
+<div data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">
+<p>XXXX XXXX xxxxx</p>
+<div data-scrapy-ignore-beneath="true">
+<img scr="image.jpg" /><br/><a link="back.html">Click here to go back</a>
+</div>
+<div>
+10.00<p> 13</p>
+</div>
+</div>
+</body></html>
+"""
+
+EXTRACT_PAGE12a = u"""
+<html><body>
+<h1>A product</h1>
+<div>
+<p>A very nice product for all intelligent people</p>
+<div>
+<img scr="image.jpg" /><br/><a link="back.html">Click here to go back</a>
+</div>
+<div>
+12.00<p> ID 15</p>
+(VAT exc.)
+</div></div>
+</body></html>
+"""
+
+EXTRACT_PAGE12b = u"""
+<html><body>
+<h1>A product</h1>
+<div>
+<p>A very nice product for all intelligent people</p>
+<div>
+<img scr="image.jpg" /><br/><a link="back.html">Click here to go back</a>
+</div>
+<div>
+12.00<p> ID 15</p>
+(VAT exc.)
+</div>
+<ul>
+Features
+<li>Feature A</li>
+<li>Feature B</li>
+</ul>
+</div>
+</body></html>
+"""
+
+# Ex1: nested annotation with token sequence replica outside exterior annotation
+# and a possible sequence pattern can be extracted only with
+# correct handling of nested annotations
+ANNOTATED_PAGE13a = u"""
+<html><body>
+<span>
+<p data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">
+<hr/>
+<h3 data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;name&quot;}}">A product</h3>
+<b data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}">$50.00</b>
+This product is excelent. Buy it!
+</p>
+</span>
+<span>
+<p>
+<h3>See other products:</h3>
+<b>Product b</b>
+</p>
+</span>
+<hr/>
+</body></html>
+"""
+
+EXTRACT_PAGE13a = u"""
+<html><body>
+<span>
+<p>
+<h3>A product</h3>
+<b>$50.00</b>
+This product is excelent. Buy it!
+<hr/>
+</p>
+</span>
+<span>
+<p>
+<h3>See other products:</h3>
+<b>Product B</b>
+</p>
+</span>
+</body></html>
+"""
+
+# Ex2: annotation with token sequence replica inside a previous nested annotation
+# and a possible sequence pattern can be extracted only with
+# correct handling of nested annotations
+ANNOTATED_PAGE13b = u"""
+<html><body>
+<span>
+<p data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">
+<h3 data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;name&quot;}}">A product</h3>
+<b>Previous price: $50.00</b>
+This product is excelent. Buy it!
+</p>
+</span>
+<span>
+<p>
+<h3>Save 10%!!</h3>
+<b data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}">$45.00</b>
+</p>
+</span>
+</body></html>
+"""
+
+EXTRACT_PAGE13b = u"""
+<html><body>
+<span>
+<p>
+<h3>A product</h3>
+<b>$50.00</b>
+This product is excelent. Buy it!
+</p>
+</span>
+<span>
+<hr/>
+<p>
+<h3>Save 10%!!</h3>
+<b>$45.00</b>
+</p>
+</span>
+<hr/>
+</body></html>
+"""
+
+ANNOTATED_PAGE14 = u"""
+<html><body>
+<b data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}"></b>
+<p data-scrapy-ignore="true"></p>
+</body></html>
+"""
+
+EXTRACT_PAGE14 = u"""
+<html><body>
+</body></html>
+"""
+
+ANNOTATED_PAGE15 = u"""
+<html><body>
+<div data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;short_description&quot;}}">Short
+<div data-scrapy-ignore="true" data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;site_id&quot;}}">892342</div>
+</div>
+<hr/>
+<p data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">Description
+<b data-scrapy-ignore="true" data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}">90.00</b>
+</p>
+</body></html>
+"""
+
+EXTRACT_PAGE15 = u"""
+<html><body>
+<hr/>
+<p>Description
+<b>80.00</b>
+</p>
+</body></html>
+"""
+
+ANNOTATED_PAGE16 = u"""
+<html><body>
+<div data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">
+Description
+<p data-scrapy-ignore="true" data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;name&quot;}}">
+name</p>
+<p data-scrapy-ignore="true" data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}">
+80.00</p>
+</div>
+</body></html>
+"""
+
+EXTRACT_PAGE16 = u"""
+<html><body>
+<p>product name</p>
+<p>90.00</p>
+</body></html>
+"""
+
+ANNOTATED_PAGE17 = u"""
+<html><body>
+<span>
+<p data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">
+This product is excelent. Buy it!
+</p>
+</span>
+<table></table>
+<img src="line.jpg" data-scrapy-ignore-beneath="true"/>
+<span>
+<h3>See other products:</h3>
+<p>Product b
+</p>
+</span>
+</body></html>
+"""
+
+EXTRACT_PAGE17 = u"""
+<html><body>
+<span>
+<p>
+This product is excelent. Buy it!
+</p>
+</span>
+<img src="line.jpg"/>
+<span>
+<h3>See other products:</h3>
+<p>Product B
+</p>
+</span>
+</body></html>
+"""
+
+ANNOTATED_PAGE18 = u"""
+<html><body>
+<div data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">
+<ins data-scrapy-ignore="true" data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;generated&quot;: true, &quot;annotations&quot;: {&quot;content&quot;: &quot;site_id&quot;}}">Item Id</ins>
+<br>
+Description
+</div>
+</body></html>
+"""
+
+EXTRACT_PAGE18 = u"""
+<html><body>
+<div>
+Item Id
+<br>
+Description
+</div>
+</body></html>
+"""
+
+ANNOTATED_PAGE19 = u"""
+<html><body>
+<div>
+<p data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;name&quot;}}">Product name</p>
+<p data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}">60.00</p>
+<img data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;src&quot;: &quot;image_urls&quot;}}"src="image.jpg" />
+<p data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;required&quot;: [&quot;description&quot;], &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">description</p>
+</div>
+</body></html>
+"""
+
+EXTRACT_PAGE19a = u"""
+<html><body>
+<div>
+<p>Product name</p>
+<p>60.00</p>
+<img src="http://example.com/image.jpg" />
+<p>description</p>
+</div>
+</body></html>
+"""
+
+EXTRACT_PAGE19b = u"""
+<html><body>
+<div>
+<p>Range</p>
+<p>from 20.00</p>
+<img src="http://example.com/image1.jpg" />
+<p>
+<br/>
+</div>
+</body></html>
+"""
+
+SAMPLE_DESCRIPTOR1 = ItemDescriptor('test', 'product test', [
+    A('name', "Product name", required=True),
+    A('price', "Product price, including any discounts and tax or vat", 
+        contains_any_numbers, True),    
+    A('image_urls', "URLs for one or more images", image_url, True),
+    A('description', "The full description of the product", allow_markup=True),
+    ]
+)
+
+# A list of (test name, [templates], page, extractors, expected_result)
+TEST_DATA = [
+    # extract from a similar page
+    ('similar page extraction', [ANNOTATED_PAGE1], EXTRACT_PAGE1, None,
+        {u'title': [u'Nice Product'], u'description': [u'wonderful product'], 
+            u'image_url': [u'nice_product.jpg']}
+    ),
+    # strip the first 5 characters from the title
+    ('extractor test', [ANNOTATED_PAGE1], EXTRACT_PAGE1,
+        ItemDescriptor('test', 'product test', 
+            [A('title', "something about a title", lambda x: x[5:])]),
+        {u'title': [u'Product'], u'description': [u'wonderful product'], 
+            u'image_url': [u'nice_product.jpg']}
+    ),
+    # compilicated tag (multiple attributes and annotation)
+    ('multiple attributes and annotation', [ANNOTATED_PAGE2], EXTRACT_PAGE2, None,
+        {'name': [u'product 1'], 'image_url': [u'http://example.com/product1.jpg'],
+            'description': [u'product 1 is great']}
+    ),
+    # can only work out correct placement by matching the second attribute first
+    ('ambiguous description', [ANNOTATED_PAGE3], EXTRACT_PAGE3, None,
+        {'description': [u'description'], 'delivery': [u'delivery']}
+    ),
+    # infer a repeated structure
+    ('repeated elements', [ANNOTATED_PAGE4], EXTRACT_PAGE4, None,
+        {'features': [u'feature1', u'feature2', u'feature3']}
+    ),
+    # identical variants with a repeated structure
+    ('repeated identical variants', [ANNOTATED_PAGE5], EXTRACT_PAGE5, None,
+         {
+             'description': [u'description'],
+             'variants': [
+                 {u'colour': [u'colour 1'], u'price': [u'price 1']}, 
+                 {u'colour': [u'colour 2'], u'price': [u'price 2']}, 
+                 {u'colour': [u'colour 3'], u'price': [u'price 3']} 
+             ]
+         }
+    ),
+    # variants with an irregular structure
+    ('irregular variants', [ANNOTATED_PAGE6], EXTRACT_PAGE6, None,
+         {
+             'description': [u'description'],
+             'variants': [
+                 {u'name': [u'name 1']}, 
+                 {u'name': [u'name 3']}, 
+                 {u'name': [u'name 2']}
+             ]
+         }
+    ),
+
+    # discovering repeated variants in table columns
+#    ('variants in table columns', [ANNOTATED_PAGE7], EXTRACT_PAGE7, None,
+#         {'variants': [
+#             {u'colour': [u'colour 1'], u'price': [u'price 1']}, 
+#             {u'colour': [u'colour 2'], u'price': [u'price 2']}, 
+#             {u'colour': [u'colour 3'], u'price': [u'price 3']}
+#         ]}
+#    ),
+    
+    
+    # ignored regions
+    (
+    'ignored_regions', [ANNOTATED_PAGE8], EXTRACT_PAGE8, None,
+          {
+             'description': [u'\n A very nice product for all intelligent people \n\n'],
+             'price': [u'\n12.00\n(VAT exc.)'],
+          }
+    ),
+    # shifted ignored regions (detected by region similarity)
+    (
+    'shifted_ignored_regions', [ANNOTATED_PAGE9], EXTRACT_PAGE9, None,
+          {
+             'description': [u'\n A very nice product for all intelligent people \n\n'],
+             'price': [u'\n12.00\n(VAT exc.)'],
+          }
+    ),
+    # detection of common prefixes across templates, all templates marked
+    (# wrong extraction
+    'without_match_common_prefix', [ANNOTATED_PAGE10a], EXTRACT_PAGE10a, None,
+          {
+             'price': [u'\n Series:  T2000 \n'],
+             'dimensions': [u'\n Description  Electrorheological Cyborgs \n'],
+             'site_id': [u'\n Offer  From $2500.00 \n'],
+          }
+    ),
+    (# right extraction
+    'with_match_common_prefix', [ANNOTATED_PAGE10a, ANNOTATED_PAGE10b], EXTRACT_PAGE10a, None,
+          {}
+    ),
+    (# another example
+    'match_common_prefix', [ANNOTATED_PAGE10a, ANNOTATED_PAGE10b], EXTRACT_PAGE10b, None,
+          {
+             'dimensions': [u'50cm'],
+             'site_id': [u'K80'],
+          }
+    ),
+    (# common_prefix with allow_markup attribute
+    'common_prefix_allow_markup', [ANNOTATED_PAGE10a, ANNOTATED_PAGE10b], EXTRACT_PAGE10b,
+        ItemDescriptor('test', 'product test',
+            [A('dimensions', "something about dimensions", allow_markup=True)]),
+          {
+             'dimensions': [u'50cm</td>'],
+             'site_id': [u'K80'],
+          }
+    ),
+    (# special case with partial annotations
+    'special_partial_annotation', [ANNOTATED_PAGE11], EXTRACT_PAGE11, None,
+          {
+            'name': [u'SL342'],
+            'description': [u'\nSL342\n \nNice product for ladies\n \n&pounds;85.00\n'],
+            'price': [u'&pounds;85.00'],
+            'price_before_discount': [u'&pounds;100.00'],
+          }
+    ),
+    (# with ignore-beneath feature
+    'ignore-beneath', [ANNOTATED_PAGE12], EXTRACT_PAGE12a, None,
+          {
+            'description': [u'\n A very nice product for all intelligent people \n'],
+          }
+    ),
+    (# ignore-beneath with extra tags
+    'ignore-beneath with extra tags', [ANNOTATED_PAGE12], EXTRACT_PAGE12b, None,
+          {
+            'description': [u'\n A very nice product for all intelligent people \n'],
+          }
+    ),
+    ('nested annotation with replica outside', [ANNOTATED_PAGE13a], EXTRACT_PAGE13a, None,
+          {'description': [u'\n A product \n $50.00 \nThis product is excelent. Buy it!\n \n'],
+           'price': ["$50.00"],
+           'name': [u'A product']}
+    ),
+    ('outside annotation with nested replica', [ANNOTATED_PAGE13b], EXTRACT_PAGE13b, None,
+          {'description': [u'\n A product \n $50.00 \nThis product is excelent. Buy it!\n'],
+           'price': ["$45.00"],
+           'name': [u'A product']}
+    ),
+    ('consistency check', [ANNOTATED_PAGE14], EXTRACT_PAGE14, None,
+          {},
+    ),
+    ('consecutive nesting', [ANNOTATED_PAGE15], EXTRACT_PAGE15, None,
+          {'description': [u'Description\n\n'],
+           'price': [u'80.00']},
+    ),
+    ('nested inside not found', [ANNOTATED_PAGE16], EXTRACT_PAGE16, None,
+          {'price': [u'90.00'],
+           'name': [u'product name']},
+    ),
+    ('ignored region helps to find attributes', [ANNOTATED_PAGE17], EXTRACT_PAGE17, None,
+          {'description': [u'\nThis product is excelent. Buy it!\n']},
+    ),
+    ('ignored region in partial annotation', [ANNOTATED_PAGE18], EXTRACT_PAGE18, None,
+          {u'site_id': [u'Item Id'],
+           u'description': [u'\nDescription\n']},
+    ),
+    ('extra required attribute product', [ANNOTATED_PAGE19], EXTRACT_PAGE19a,
+         SAMPLE_DESCRIPTOR1,
+         {u'price': [u'60.00'],
+          u'description': [u'description'],
+          u'image_urls': [['http://example.com/image.jpg']],
+          u'name': [u'Product name']},
+    ),
+    ('extra required attribute no product', [ANNOTATED_PAGE19], EXTRACT_PAGE19b,
+         SAMPLE_DESCRIPTOR1,
+         None,
+    ),
+]
+
+class TestExtraction(TestCase):
+
+    def _run(self, name, templates, page, extractors, expected_output):
+        self.trace = None
+        template_pages = [HtmlPage(None, {}, t) for t in templates]
+        extractor = InstanceBasedLearningExtractor(template_pages, extractors, True)
+        actual_output, _ = extractor.extract(HtmlPage(None, {}, page))
+        if not actual_output:
+            if expected_output is None:
+                return
+            assert False, "failed to extract data for test '%s'" % name
+        actual_output = actual_output[0]
+        self.trace = ["Extractor:\n%s" % extractor] + actual_output.pop('trace', [])
+        expected_names = set(expected_output.keys())
+        actual_names = set(actual_output.keys())
+        
+        missing_in_output = filter(None, expected_names - actual_names)
+        error = "attributes '%s' were expected but were not present in test '%s'" % \
+                ("', '".join(missing_in_output), name)
+        assert len(missing_in_output) == 0, error
+
+        unexpected = actual_names - expected_names
+        error = "unexpected attributes %s in test '%s'" % \
+                (', '.join(unexpected), name)
+        assert len(unexpected) == 0, error
+
+        for k, v in expected_output.items():
+            extracted = actual_output[k]
+            assert v == extracted, "in test '%s' for attribute '%s', " \
+                "expected value '%s' but got '%s'" % (name, k, v, extracted)
+
+    def test_expected_outputs(self):
+        try:
+            for data in TEST_DATA:
+                self._run(*data)
+        except AssertionError:
+            if self.trace:
+                print "Trace:"
+                for line in self.trace:
+                    print "\n---\n%s" % line
+            raise

scrapy/tests/test_contrib_ibl/test_htmlpage.py

@@ -0,0 +1,134 @@
+"""
+htmlpage.py tests
+"""
+import os
+from gzip import GzipFile
+from unittest import TestCase
+
+from scrapy.utils.py26 import json
+from scrapy.tests.test_contrib_ibl import path
+from scrapy.contrib.ibl.htmlpage import parse_html, HtmlTag, HtmlDataFragment
+from scrapy.tests.test_contrib_ibl.test_htmlpage_data import *
+
+SAMPLES_FILE = "samples_htmlpage.json.gz"
+
+def _encode_element(el):
+    """
+    jsonize parse element
+    """
+    if isinstance(el, HtmlTag):
+        return {"tag": el.tag, "attributes": el.attributes,
+            "start": el.start, "end": el.end, "tag_type": el.tag_type}
+    if isinstance(el, HtmlDataFragment):
+        return {"start": el.start, "end": el.end}
+    raise TypeError
+
+def _decode_element(dct):
+    """
+    dejsonize parse element
+    """
+    if "tag" in dct:
+        return HtmlTag(dct["tag_type"], dct["tag"], \
+            dct["attributes"], dct["start"], dct["end"])
+    if "start" in dct:
+        return HtmlDataFragment(dct["start"], dct["end"])
+    return dct
+
+def add_sample(source):
+    """
+    Method for adding samples to test samples file
+    (use from console)
+    """
+    samples = []
+    if os.path.exists(SAMPLES_FILE):
+        for line in GzipFile(os.path.join(path, SAMPLES_FILE), "r").readlines():
+            samples.append(json.loads(line))
+    
+    new_sample = {"source": source}
+    new_sample["parsed"] = list(parse_html(source))
+    samples.append(new_sample)
+    samples_file = GzipFile(os.path.join(path, SAMPLES_FILE), "wb")
+    for sample in samples:
+        samples_file.write(json.dumps(sample, default=_encode_element) + "\n")
+    samples_file.close()
+
+class TestParseHtml(TestCase):
+    """Test for parse_html"""
+    def _test_sample(self, sample):
+        source = sample["source"]
+        expected_parsed = sample["parsed"]
+        parsed = parse_html(source)
+        count_element = 0
+        count_expected = 0
+        for element in parsed:
+            if type(element) == HtmlTag:
+                count_element += 1
+            expected = expected_parsed.pop(0)
+            if type(expected) == HtmlTag:
+                count_expected += 1
+            element_text = source[element.start:element.end]
+            expected_text = source[expected.start:expected.end]
+            if element.start != expected.start or element.end != expected.end:
+                assert False, "[%s,%s] %s != [%s,%s] %s" % (element.start, \
+                    element.end, element_text, expected.start, \
+                    expected.end, expected_text)
+            if type(element) != type(expected):
+                assert False, "(%s) %s != (%s) %s for text\n%s" % (count_element, \
+                    repr(type(element)), count_expected, repr(type(expected)), element_text)
+            if type(element) == HtmlTag:
+                self.assertEqual(element.tag, expected.tag)
+                self.assertEqual(element.attributes, expected.attributes)
+
+    def test_parse(self):
+        """simple parse_html test"""
+        parsed = [_decode_element(d) for d in PARSED]
+        sample = {"source": PAGE, "parsed": parsed}
+        self._test_sample(sample)
+        
+    def test_site_samples(self):
+        """test parse_html from real cases"""
+        samples = []
+        for line in GzipFile(os.path.join(path, SAMPLES_FILE), "r").readlines():
+            samples.append(json.loads(line, object_hook=_decode_element))
+        for sample in samples:
+            self._test_sample(sample)
+    
+    def test_bad(self):
+        """test parsing of bad html layout"""
+        parsed = [_decode_element(d) for d in PARSED2]
+        sample = {"source": PAGE2, "parsed": parsed}
+        self._test_sample(sample)
+
+    def test_comments(self):
+        """test parsing of tags inside comments"""
+        parsed = [_decode_element(d) for d in PARSED3]
+        sample = {"source": PAGE3, "parsed": parsed}
+        self._test_sample(sample)
+
+    def test_script_text(self):
+        """test parsing of tags inside scripts"""
+        parsed = [_decode_element(d) for d in PARSED4]
+        sample = {"source": PAGE4, "parsed": parsed}
+        self._test_sample(sample)
+        
+    def test_sucessive(self):
+        """test parsing of sucesive cleaned elements"""
+        parsed = [_decode_element(d) for d in PARSED5]
+        sample = {"source": PAGE5, "parsed": parsed}
+        self._test_sample(sample)
+        
+    def test_sucessive2(self):
+        """test parsing of sucesive cleaned elements (variant 2)"""
+        parsed = [_decode_element(d) for d in PARSED6]
+        sample = {"source": PAGE6, "parsed": parsed}
+        self._test_sample(sample)
+    
+    def test_special_cases(self):
+        """some special cases tests"""
+        parsed = list(parse_html("<meta http-equiv='Pragma' content='no-cache' />"))
+        self.assertEqual(parsed[0].attributes, {'content': 'no-cache', 'http-equiv': 'Pragma'})
+        parsed = list(parse_html("<html xmlns='http://www.w3.org/1999/xhtml' xml:lang='en' lang='en'>"))
+        self.assertEqual(parsed[0].attributes, {'xmlns': 'http://www.w3.org/1999/xhtml', 'xml:lang': 'en', 'lang': 'en'})
+        parsed = list(parse_html("<IMG SRC='http://images.play.com/banners/SAM550a.jpg' align='left' / hspace=5>"))
+        self.assertEqual(parsed[0].attributes, {'src': 'http://images.play.com/banners/SAM550a.jpg', \
+                                                'align': 'left', 'hspace': '5', '/': None})

scrapy/tests/test_contrib_ibl/test_htmlpage_data.py

@@ -0,0 +1,227 @@
+PAGE = u"""
+<style id="scrapy-style" type="text/css">@import url(http://localhost:8000/as/site_media/clean.css);                           
+</style>
+<body>
+<div class="scrapy-selected" id="header">
+<img src="company_logo.jpg" style="margin-left: 68px; padding-top:5px;" alt="Logo" width="530" height="105">
+<div id="vertrule">
+<h1>COMPANY - <ins data-scrapy-annotate="{&quot;variant&quot;: &quot;0&quot;, &quot;generated&quot;: true, &quot;annotations&quot;: {&quot;content&quot;: &quot;title&quot;}}">Item Title</ins></h1>
+<p>introduction</p>
+<div>
+<img src="/upload/img.jpg" classid=""
+    data-scrapy-annotate="{&quot;variant&quot;: &quot;0&quot;, &quot;annotations&quot;: {&quot;image_url&quot;: &quot;src&quot;}}"
+>
+<p classid="" data-scrapy-annotate="{&quot;variant&quot;: &quot;0&quot;, &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}"
+>
+This is such a nice item<br/> Everybody likes it.
+</p>
+<br></br>
+</div>
+<p data-scrapy-annotate="{&quot;variant&quot;: &quot;0&quot;, &quot;annotations&quot;: {&quot;content&quot;: &quot;features&quot;}}"
+class="" >Power: 50W</p>
+<!-- A comment --!>
+<ul data-scrapy-replacement='select' class='product'>
+<li data-scrapy-replacement='option'>Small</li>
+<li data-scrapy-replacement='option'>Big</li>
+</ul>
+<p>click here for other items</p>
+<h3>Louis Chair</h3>
+<table class="rulet" width="420" cellpadding="0" cellspacing="0"><tbody>
+<tr><td>Height</td>
+<td><ins data-scrapy-annotate="{&quot;variant&quot;: &quot;0&quot;, &quot;generated&quot;: true, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}">32.00</ins></td>
+</tr><tbody></table>
+<p onmouseover='xxx' class= style="my style">
+"""
+
+PARSED = [
+{'start': 0, 'end': 1},
+{'attributes': {'type': 'text/css', 'id': 'scrapy-style'}, 'tag': 'style', 'end': 42, 'start': 1, 'tag_type': 1},
+{'start': 42, 'end': 129},
+{'attributes': {}, 'tag': 'style', 'end': 137, 'start': 129, 'tag_type': 2},
+{'start': 137, 'end': 138},
+{'attributes': {}, 'tag': 'body', 'end': 144, 'start': 138, 'tag_type': 1},
+{'start': 144, 'end': 145},
+{'attributes': {'class': 'scrapy-selected', 'id': 'header'}, 'tag': 'div', 'end': 186, 'start': 145, 'tag_type': 1},
+{'start': 186, 'end': 187},
+{'attributes': {'src': 'company_logo.jpg', 'style': 'margin-left: 68px; padding-top:5px;', 'width': '530', 'alt': 'Logo', 'height': '105'}, 'tag': 'img', 'end': 295, 'start': 187, 'tag_type': 1},
+{'start': 295, 'end': 296},
+{'attributes': {'id': 'vertrule'}, 'tag': 'div', 'end': 315, 'start': 296, 'tag_type': 1},
+{'start': 315, 'end': 316},
+{'attributes': {}, 'tag': 'h1', 'end': 320, 'start': 316, 'tag_type': 1},
+{'start': 320, 'end': 330},
+{'attributes': {'data-scrapy-annotate': '{&quot;variant&quot;: &quot;0&quot;, &quot;generated&quot;: true, &quot;annotations&quot;: {&quot;content&quot;: &quot;title&quot;}}'}, 'tag': 'ins', 'end': 491, 'start': 330, 'tag_type': 1},
+{'start': 491, 'end': 501},
+{'attributes': {}, 'tag': 'ins', 'end': 507, 'start': 501, 'tag_type': 2},
+{'attributes': {}, 'tag': 'h1', 'end': 512, 'start': 507, 'tag_type': 2},
+{'start': 512, 'end': 513},
+{'attributes': {}, 'tag': 'p', 'end': 516, 'start': 513, 'tag_type': 1},
+{'start': 516, 'end': 528},
+{'attributes': {}, 'tag': 'p', 'end': 532, 'start': 528, 'tag_type': 2},
+{'start': 532, 'end': 533},
+{'attributes': {}, 'tag': 'div', 'end': 538, 'start': 533, 'tag_type': 1},
+{'start': 538, 'end': 539},
+{'attributes': {'classid': None, 'src': '/upload/img.jpg', 'data-scrapy-annotate': '{&quot;variant&quot;: &quot;0&quot;, &quot;annotations&quot;: {&quot;image_url&quot;: &quot;src&quot;}}'}, 'tag': 'img', 'end': 709, 'start': 539, 'tag_type': 1},
+{'start': 709, 'end': 710},
+{'attributes': {'classid': None, 'data-scrapy-annotate': '{&quot;variant&quot;: &quot;0&quot;, &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}'}, 'tag': 'p', 'end': 858, 'start': 710, 'tag_type': 1},
+{'start': 858, 'end': 883},
+{'attributes': {}, 'tag': 'br', 'end': 888, 'start': 883, 'tag_type': 3},
+{'start': 888, 'end': 909},
+{'attributes': {}, 'tag': 'p', 'end': 913, 'start': 909, 'tag_type': 2},
+{'start': 913, 'end': 914},
+{'attributes': {}, 'tag': 'br', 'end': 918, 'start': 914, 'tag_type': 1},
+{'attributes': {}, 'tag': 'br', 'end': 923, 'start': 918, 'tag_type': 2},
+{'start': 923, 'end': 924},
+{'attributes': {}, 'tag': 'div', 'end': 930, 'start': 924, 'tag_type': 2},
+{'start': 930, 'end': 931},
+{'attributes': {'data-scrapy-annotate': '{&quot;variant&quot;: &quot;0&quot;, &quot;annotations&quot;: {&quot;content&quot;: &quot;features&quot;}}', 'class': None}, 'tag': 'p', 'end': 1074, 'start': 931, 'tag_type': 1},
+{'start': 1074, 'end': 1084},
+{'attributes': {}, 'tag': 'p', 'end': 1088, 'start': 1084, 'tag_type': 2},
+{'start': 1088, 'end': 1109},
+{'attributes': {'data-scrapy-replacement': 'select', 'class': 'product'}, 'tag': 'ul', 'end': 1162, 'start': 1109, 'tag_type': 1},
+{'start': 1162, 'end': 1163},
+{'attributes': {'data-scrapy-replacement': 'option'}, 'tag': 'li', 'end': 1200, 'start': 1163, 'tag_type': 1},
+{'start': 1200, 'end': 1205},
+{'attributes': {}, 'tag': 'li', 'end': 1210, 'start': 1205, 'tag_type': 2},
+{'start': 1210, 'end': 1211},
+{'attributes': {'data-scrapy-replacement': 'option'}, 'tag': 'li', 'end': 1248, 'start': 1211, 'tag_type': 1},
+{'start': 1248, 'end': 1251},
+{'attributes': {}, 'tag': 'li', 'end': 1256, 'start': 1251, 'tag_type': 2},
+{'start': 1256, 'end': 1257},
+{'attributes': {}, 'tag': 'ul', 'end': 1262, 'start': 1257, 'tag_type': 2},
+{'start': 1262, 'end': 1263},
+{'attributes': {}, 'tag': 'p', 'end': 1266, 'start': 1263, 'tag_type': 1},
+{'start': 1266, 'end': 1292},
+{'attributes': {}, 'tag': 'p', 'end': 1296, 'start': 1292, 'tag_type': 2},
+{'start': 1296, 'end': 1297},
+{'attributes': {}, 'tag': 'h3', 'end': 1301, 'start': 1297, 'tag_type': 1},
+{'start': 1301, 'end': 1312},
+{'attributes': {}, 'tag': 'h3', 'end': 1317, 'start': 1312, 'tag_type': 2},
+{'start': 1317, 'end': 1318},
+{'attributes': {'cellpadding': '0', 'width': '420', 'cellspacing': '0', 'class': 'rulet'}, 'tag': 'table', 'end': 1383, 'start': 1318, 'tag_type': 1},
+{'attributes': {}, 'tag': 'tbody', 'end': 1390, 'start': 1383, 'tag_type': 1},
+{'start': 1390, 'end': 1391},
+{'attributes': {}, 'tag': 'tr', 'end': 1395, 'start': 1391, 'tag_type': 1},
+{'attributes': {}, 'tag': 'td', 'end': 1399, 'start': 1395, 'tag_type': 1},
+{'start': 1399, 'end': 1405},
+{'attributes': {}, 'tag': 'td', 'end': 1410, 'start': 1405, 'tag_type': 2},
+{'start': 1410, 'end': 1411},
+{'attributes': {}, 'tag': 'td', 'end': 1415, 'start': 1411, 'tag_type': 1},
+{'attributes': {'data-scrapy-annotate': '{&quot;variant&quot;: &quot;0&quot;, &quot;generated&quot;: true, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}'}, 'tag': 'ins', 'end': 1576, 'start': 1415, 'tag_type': 1},
+{'start': 1576, 'end': 1581},
+{'attributes': {}, 'tag': 'ins', 'end': 1587, 'start': 1581, 'tag_type': 2},
+{'attributes': {}, 'tag': 'td', 'end': 1592, 'start': 1587, 'tag_type': 2},
+{'start': 1592, 'end': 1593},
+{'attributes': {}, 'tag': 'tr', 'end': 1598, 'start': 1593, 'tag_type': 2},
+{'attributes': {}, 'tag': 'tbody', 'end': 1605, 'start': 1598, 'tag_type': 1},
+{'attributes': {}, 'tag': 'table', 'end': 1613, 'start': 1605, 'tag_type': 2},
+{'start': 1613, 'end': 1614},
+{'attributes': {'style': 'my style', 'onmouseover': 'xxx', 'class': None}, 'tag': 'p', 'end': 1659, 'start': 1614, 'tag_type': 1},
+{'start': 1659, 'end': 1660},
+]
+
+# for testing parsing of some invalid html code (but still managed by browsers)
+PAGE2 = u"""
+<html>
+<body>
+<p class=&#34;MsoNormal&#34; style=&#34;margin: 0cm 0cm 0pt&#34;><span lang=&#34;EN-GB&#34;>
+Hello world!
+</span>
+</p>
+</body>
+</html>
+"""
+
+PARSED2 = [
+ {'end': 1, 'start': 0},
+ {'attributes': {}, 'end': 7, 'start': 1, 'tag': u'html', 'tag_type': 1},
+ {'end': 8, 'start': 7},
+ {'attributes': {}, 'end': 14, 'start': 8, 'tag': u'body', 'tag_type': 1},
+ {'end': 15, 'start': 14},
+ {'attributes': {u'style': u'&#34;margin:', u'0pt&#34;': None, u'class': u'&#34;MsoNormal&#34;', u'0cm': None}, 'end': 80, 'start': 15, 'tag': u'p', 'tag_type': 2},
+ {'attributes': {u'lang': u'&#34;EN-GB&#34;'}, 'end': 107, 'start': 80, 'tag': u'span', 'tag_type': 1},
+ {'end': 121, 'start': 107},
+ {'attributes': {}, 'end': 128, 'start': 121, 'tag': u'span', 'tag_type': 2},
+ {'end': 129, 'start': 128},
+ {'attributes': {}, 'end': 133, 'start': 129, 'tag': u'p', 'tag_type': 2},
+ {'end': 134, 'start': 133},
+ {'attributes': {}, 'end': 141, 'start': 134, 'tag': u'body', 'tag_type': 2},
+ {'end': 142, 'start': 141},
+ {'attributes': {}, 'end': 149, 'start': 142, 'tag': u'html', 'tag_type': 2},
+ {'end': 150, 'start': 149},
+]
+
+# for testing tags inside comments
+PAGE3 = u"""<html><body><h1>Helloooo!!</h1><p>Did i say hello??</p><!--<p>
+</p>--><script type="text/javascript">bla<!--comment-->blabla</script></body></html>"""
+
+PARSED3 = [
+ {'attributes': {}, 'end': 6, 'start': 0, 'tag': u'html', 'tag_type': 1},
+ {'attributes': {}, 'end': 12, 'start': 6, 'tag': u'body', 'tag_type': 1},
+ {'attributes': {}, 'end': 16, 'start': 12, 'tag': u'h1', 'tag_type': 1},
+ {'end': 26, 'start': 16},
+ {'attributes': {}, 'end': 31, 'start': 26, 'tag': u'h1', 'tag_type': 2},
+ {'attributes': {}, 'end': 34, 'start': 31, 'tag': u'p', 'tag_type': 1},
+ {'end': 51, 'start': 34},
+ {'attributes': {}, 'end': 55, 'start': 51, 'tag': u'p', 'tag_type': 2},
+ {'end': 70, 'start': 55},
+ {'attributes': {u'type': u'text/javascript'}, 'end': 101, 'start': 70, 'tag': u'script', 'tag_type': 1},
+ {'end': 124, 'start': 101},
+ {'attributes': {}, 'end': 133, 'start': 124, 'tag': u'script', 'tag_type': 2},
+ {'attributes': {}, 'end': 140, 'start': 133, 'tag': u'body', 'tag_type': 2},
+ {'attributes': {}, 'end': 147, 'start': 140, 'tag': u'html', 'tag_type': 2}
+]
+
+# for testing tags inside scripts
+PAGE4 = u"""<html><body><h1>Konnichiwa!!</h1>hello<script type="text/javascript">\
+doc.write("<img src=" + base + "product/" + productid + ">");\
+</script>hello again</body></html>"""
+
+PARSED4 = [
+ {'attributes': {}, 'end': 6, 'start': 0, 'tag': u'html', 'tag_type': 1},
+ {'attributes': {}, 'end': 12, 'start': 6, 'tag': u'body', 'tag_type': 1},
+ {'attributes': {}, 'end': 16, 'start': 12, 'tag': u'h1', 'tag_type': 1},
+ {'end': 28,'start': 16},
+ {'attributes': {}, 'end': 33, 'start': 28, 'tag': u'h1', 'tag_type': 2},
+ {'end': 38, 'start': 33},
+ {'attributes': {u'type': u'text/javascript'}, 'end': 69, 'start': 38, 'tag': u'script', 'tag_type': 1},
+ {'end': 130, 'start': 69},
+ {'attributes': {}, 'end': 139, 'start': 130, 'tag': u'script', 'tag_type': 2},
+ {'end': 150, 'start': 139},
+ {'attributes': {}, 'end': 157, 'start': 150, 'tag': u'body', 'tag_type': 2},
+ {'attributes': {}, 'end': 164, 'start': 157, 'tag': u'html', 'tag_type': 2},
+]
+
+# Test sucessive cleaning elements
+PAGE5 = u"""<html><body><script>hello</script><script>brb</script></body><!--commentA--><!--commentB--></html>"""
+
+PARSED5 = [
+ {'attributes': {}, 'end': 6, 'start': 0, 'tag': u'html', 'tag_type': 1},
+ {'attributes': {}, 'end': 12, 'start': 6, 'tag': u'body', 'tag_type': 1},
+ {'attributes': {}, 'end': 20, 'start': 12, 'tag': u'script', 'tag_type': 1},
+ {'end': 25, 'start': 20},
+ {'attributes': {}, 'end': 34, 'start': 25, 'tag': u'script', 'tag_type': 2},
+ {'attributes': {}, 'end': 42, 'start': 34, 'tag': u'script', 'tag_type': 1},
+ {'end': 45, 'start': 42},
+ {'attributes': {}, 'end': 54, 'start': 45, 'tag': u'script', 'tag_type': 2},
+ {'attributes': {}, 'end': 61, 'start': 54, 'tag': u'body', 'tag_type': 2},
+ {'end': 91, 'start': 61},
+ {'attributes': {}, 'end': 98, 'start': 91, 'tag': u'html', 'tag_type': 2},
+]
+ 
+# Test sucessive cleaning elements variant 2
+PAGE6 = u"""<html><body><script>pss<!--comment-->pss</script>all<script>brb</script>\n\n</body></html>"""
+
+PARSED6 = [
+ {'attributes': {}, 'end': 6, 'start': 0, 'tag': u'html', 'tag_type': 1},
+ {'attributes': {}, 'end': 12, 'start': 6, 'tag': u'body', 'tag_type': 1},
+ {'attributes': {}, 'end': 20, 'start': 12, 'tag': u'script', 'tag_type': 1},
+ {'end': 40, 'start': 20},
+ {'attributes': {}, 'end': 49, 'start': 40, 'tag': u'script', 'tag_type': 2},
+ {'end': 52, 'start': 49},
+ {'attributes': {}, 'end': 60, 'start': 52, 'tag': u'script', 'tag_type': 1},
+ {'end': 63, 'start': 60},
+ {'attributes': {}, 'end': 72, 'start': 63, 'tag': u'script', 'tag_type': 2},
+ {'end': 74, 'start': 72},
+ {'attributes': {}, 'end': 81, 'start': 74, 'tag': u'body', 'tag_type': 2},
+ {'attributes': {}, 'end': 88, 'start': 81, 'tag': u'html', 'tag_type': 2},
+]

scrapy/tests/test_contrib_ibl/test_pageparsing.py

@@ -0,0 +1,283 @@
+"""
+Unit tests for pageparsing
+"""
+import os
+from cStringIO import StringIO                                                                                                    
+from gzip import GzipFile
+
+from unittest import TestCase
+from scrapy.utils.python import str_to_unicode
+from scrapy.utils.py26 import json
+
+from scrapy.contrib.ibl.htmlpage import HtmlPage
+from scrapy.contrib.ibl.extraction.pageparsing import (InstanceLearningParser, 
+        TemplatePageParser, ExtractionPageParser)
+from scrapy.contrib.ibl.extraction.pageobjects import TokenDict, TokenType
+from scrapy.tests.test_contrib_ibl import path
+
+SIMPLE_PAGE = u"""
+<html> <p some-attr="foo">this is a test</p> </html>
+"""
+
+LABELLED_PAGE1 = u"""
+<html>
+<h1 data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;name&quot;}}">Some Product</h1>
+<p> some stuff</p>
+<p data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">
+This is such a nice item<br/>
+Everybody likes it.
+</p>
+<p data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;common_prefix&quot;: true, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}"/>
+\xa310.00
+<br/>
+<p data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;short_description&quot;}}">
+Old fashioned product
+<p data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;short_description&quot;}}">
+For exigent individuals
+<p>click here for other items</p>
+</html>
+"""
+
+BROKEN_PAGE = u"""
+<html> <p class="ruleb"align="center">html parser cannot parse this</p></html>
+"""
+
+LABELLED_PAGE2 = u"""
+<html><body>
+<h1>A product</h1>
+<div data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">
+<p>A very nice product for all intelligent people</p>
+<div data-scrapy-ignore="true">
+<img scr="image.jpg" /><br/><a link="back.html">Click here to go back</a>
+</div>
+</div>
+<div data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}">
+\xa310.00<p data-scrapy-ignore="true"> 13 <br></p>
+</div>
+<table data-scrapy-ignore="true">
+<tr><td data-scrapy-ignore="true"></td></tr>
+<tr></tr>
+</table>
+<img data-scrapy-ignore="true" src="image2.jpg"> 
+<img data-scrapy-ignore="true" src="image3.jpg" />
+<img data-scrapy-ignore-beneath="true" src="image2.jpg">
+<img data-scrapy-ignore-beneath="true" src="image3.jpg" />
+</body></html>
+
+"""
+
+LABELLED_PAGE3 = u"""
+<html><body>
+<h1>A product</h1>
+<div data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">
+<p>A very nice product for all intelligent people</p>
+<div data-scrapy-ignore="true">
+<img scr="image.jpg" /><br/><a link="back.html">Click here to go back</a>
+</div>
+</div>
+<div data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">
+\xa310.00<p data-scrapy-ignore="true"> 13 <br></p>
+<table><tr>
+<td>Description 1</td>
+<td data-scrapy-ignore-beneath="true">Description 2</td>
+<td>Description 3</td>
+<td>Description 4</td>
+</tr></table>
+</div>
+</body></html>
+"""
+
+LABELLED_PAGE4 = u"""
+<html><body>
+<h1>A product</h1>
+<div>
+<p>A very nice product for all intelligent people</p>
+<div>
+<img scr="image.jpg" /><br/><a link="back.html">Click here to go back</a>
+</div>
+</div>
+<div data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">
+\xa310.00<p data-scrapy-ignore="true"> 13 <br></p>
+<table><tr>
+<td>Description 1</td>
+<td data-scrapy-ignore-beneath="true">Description 2</td>
+<td>Description 3</td>
+<td data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}">
+Price \xa310.00</td>
+</tr></table>
+</div>
+</body></html>
+"""
+
+LABELLED_PAGE5 = u"""
+<html><body>
+<ul data-scrapy-replacement='select'>
+<li data-scrapy-replacement='option'>Option A</li>
+<li>Option I</li>
+<li data-scrapy-replacement='option'>Option B</li>
+</ul>
+</body></html>
+"""
+
+LABELLED_PAGE6 = u"""
+<html><body>
+Text A
+<p><ins data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;generated&quot;: true, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}">
+65.00</ins>pounds</p>
+<p>Description: <ins data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;generated&quot;: true, &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">
+Text B</ins></p>
+Text C
+</body></html>
+"""
+
+LABELLED_PAGE7 = u"""
+<html><body>
+<div data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">
+<ins data-scrapy-ignore="true" data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;generated&quot;: true, &quot;annotations&quot;: {&quot;content&quot;: &quot;site_id&quot;}}">Item Id</ins>
+Description
+</div>
+</body></html>
+"""
+
+LABELLED_PAGE8 = u"""
+<html><body>
+<div data-scrapy-annotate="{&quot;required&quot;: [&quot;description&quot;], &quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">
+<ins data-scrapy-ignore="true" data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;generated&quot;: true, &quot;annotations&quot;: {&quot;content&quot;: &quot;site_id&quot;}}">Item Id</ins>
+Description
+</div>
+</body></html>
+"""
+
+def _parse_page(parser_class, pagetext):
+    htmlpage = HtmlPage(None, {}, pagetext)
+    parser = parser_class(TokenDict())
+    parser.feed(htmlpage)
+    return parser
+
+def _tags(pp, predicate):
+    return [pp.token_dict.token_string(s) for s in pp.token_list \
+            if predicate(s)]
+
+class TestPageParsing(TestCase):
+
+    def test_instance_parsing(self):
+        pp = _parse_page(InstanceLearningParser, SIMPLE_PAGE)
+        # all tags
+        self.assertEqual(_tags(pp, bool), ['<html>', '<p>', '</p>', '</html>'])
+
+        # open/closing tag handling
+        openp = lambda x: pp.token_dict.token_type(x) == TokenType.OPEN_TAG
+        self.assertEqual(_tags(pp, openp), ['<html>', '<p>'])
+        closep = lambda x: pp.token_dict.token_type(x) == TokenType.CLOSE_TAG
+        self.assertEqual(_tags(pp, closep), ['</p>', '</html>'])
+    
+    def _validate_annotation(self, parser, lable_region, name, start_tag, end_tag):
+        assert lable_region.surrounds_attribute == name
+        start_token = parser.token_list[lable_region.start_index]
+        assert parser.token_dict.token_string(start_token) == start_tag
+        end_token = parser.token_list[lable_region.end_index]
+        assert parser.token_dict.token_string(end_token) == end_tag
+
+    def test_template_parsing(self):
+        lp = _parse_page(TemplatePageParser, LABELLED_PAGE1)
+        self.assertEqual(len(lp.annotations), 5)
+        self._validate_annotation(lp, lp.annotations[0], 
+                'name', '<h1>', '</h1>')
+        self.assertEqual(lp.annotations[0].match_common_prefix, False)
+        self._validate_annotation(lp, lp.annotations[1], 
+                'description', '<p>', '</p>')
+        self.assertEqual(lp.annotations[1].match_common_prefix, False)
+        self._validate_annotation(lp, lp.annotations[2],
+                'price', '<p/>', '<p>')
+        self.assertEqual(lp.annotations[2].match_common_prefix, True)
+        self._validate_annotation(lp, lp.annotations[3],
+                'short_description', '<p>', '<p>')
+        self.assertEqual(lp.annotations[3].match_common_prefix, False)
+        self._validate_annotation(lp, lp.annotations[4],
+                'short_description', '<p>', '<p>')
+        self.assertEqual(lp.annotations[4].match_common_prefix, False)
+        
+        # all tags were closed
+        self.assertEqual(len(lp.labelled_tag_stacks), 0)
+    
+    def test_extraction_page_parsing(self):
+        epp = _parse_page(ExtractionPageParser, SIMPLE_PAGE)
+        ep = epp.to_extraction_page()
+        assert len(ep.page_tokens) == 4
+        assert ep.token_html(0) == '<html>'
+        assert ep.token_html(1) == '<p some-attr="foo">'
+        
+        assert ep.html_between_tokens(1, 2) == 'this is a test'
+        assert ep.html_between_tokens(1, 3) == 'this is a test</p> '
+
+    def test_invalid_html(self):
+        p = _parse_page(InstanceLearningParser, BROKEN_PAGE)
+        assert p
+        
+    def test_ignore_region(self):
+        """Test ignored regions"""
+        p = _parse_page(TemplatePageParser, LABELLED_PAGE2)
+        self.assertEqual(p.ignored_regions, [(7,12),(15,17),(19,26),(21,22),(27,28),(28,29),(29,None),(30,None)])
+        self.assertEqual(len(p.ignored_tag_stacks), 0)
+
+    def test_ignore_regions2(self):
+        """Test ignore-beneath regions"""
+        p = _parse_page(TemplatePageParser, LABELLED_PAGE3)
+        self.assertEqual(p.ignored_regions, [(7,12),(15,17),(22,None)])
+        self.assertEqual(len(p.ignored_tag_stacks), 0)
+        
+    def test_ignore_regions3(self):
+        """Test ignore-beneath with annotation inside region"""
+        p = _parse_page(TemplatePageParser, LABELLED_PAGE4)
+        self.assertEqual(p.ignored_regions, [(15,17),(22,None)])
+        self.assertEqual(len(p.ignored_tag_stacks), 0)
+        
+    def test_replacement(self):
+        """Test parsing of replacement tags"""
+        p = _parse_page(TemplatePageParser, LABELLED_PAGE5)
+        self.assertEqual(_tags(p, bool), ['<html>', '<body>', '<select>', '<option>',
+                    '</option>', '<li>', '</li>', '<option>', '</option>', '</select>', '</body>', '</html>'])
+        
+    def test_partial(self):
+        """Test partial annotation parsing"""
+        p = _parse_page(TemplatePageParser, LABELLED_PAGE6)
+        text = p.annotations[0].annotation_text
+        self.assertEqual(text.start_text, '')
+        self.assertEqual(text.follow_text, 'pounds')
+        text = p.annotations[1].annotation_text
+        self.assertEqual(text.start_text, "Description: ")
+        self.assertEqual(text.follow_text, '')
+        
+    def test_ignored_partial(self):
+        """Test ignored region declared on partial annotation"""
+        p = _parse_page(TemplatePageParser, LABELLED_PAGE7)
+        self.assertEqual(p.ignored_regions, [(2, 3)])
+        
+    def test_extra_required(self):
+        """Test parsing of extra required attributes"""
+        p = _parse_page(TemplatePageParser, LABELLED_PAGE8)
+        self.assertEqual(p.extra_required_attrs, ["description"])
+        
+    def test_site_pages(self):
+        """
+        Tests from real pages. More reliable and easy to build for more complicated structures
+        """
+        samples_file = open(os.path.join(path, "samples_pageparsing.json.gz"), "r")
+        samples = []
+        for line in GzipFile(fileobj=StringIO(samples_file.read())).readlines():
+            samples.append(json.loads(line))
+        for sample in samples:
+            source = sample["annotated"]
+            annotations = sample["annotations"]
+            template = HtmlPage(body=str_to_unicode(source))
+            parser = TemplatePageParser(TokenDict())
+            parser.feed(template)
+            for annotation in parser.annotations:
+                test_annotation = annotations.pop(0)
+                for s in annotation.__slots__:
+                    if s == "tag_attributes":
+                        for pair in getattr(annotation, s):
+                            self.assertEqual(list(pair), test_annotation[s].pop(0))
+                    else:
+                        self.assertEqual(getattr(annotation, s), test_annotation[s])
+            self.assertEqual(annotations, [])