From bc6eee71e325fb2707c5ddad007d94f158d73b2c Mon Sep 17 00:00:00 2001 From: Shane Evans Date: Fri, 11 Mar 2011 20:02:29 +0000 Subject: [PATCH] refactor IBL extraction to allow processing parsed data --HG-- extra : rebase_source : 1a0ec4322702288f6e996d384d45a36deede3868 --- scrapy/contrib/ibl/descriptor.py | 11 +- scrapy/contrib/ibl/extraction/pageobjects.py | 147 ++++++++------- scrapy/contrib/ibl/extraction/pageparsing.py | 37 ++-- .../contrib/ibl/extraction/regionextract.py | 110 +++++------ scrapy/contrib/ibl/extractors.py | 175 +++++++++++++++++- scrapy/contrib/ibl/htmlpage.py | 65 ++++++- .../tests/test_contrib_ibl/test_extraction.py | 62 ++++--- .../test_contrib_ibl/test_pageparsing.py | 8 +- 8 files changed, 416 insertions(+), 199 deletions(-) diff --git a/scrapy/contrib/ibl/descriptor.py b/scrapy/contrib/ibl/descriptor.py index 5b4ae99ce..075920d87 100644 --- a/scrapy/contrib/ibl/descriptor.py +++ b/scrapy/contrib/ibl/descriptor.py @@ -3,25 +3,22 @@ Extended types for IBL extraction """ from itertools import chain -from scrapy.contrib.ibl.extractors import text +from scrapy.contrib.ibl.extractors import text, html class FieldDescriptor(object): """description of a scraped attribute""" - __slots__ = ('name', 'description', 'extractor', 'required', 'allow_markup') + __slots__ = ('name', 'description', 'extractor', 'required') - def __init__(self, name, description, extractor=text, required=False, - allow_markup=False): + def __init__(self, name, description, extractor=text, required=False): self.name = name self.description = description self.extractor = extractor self.required = required - self.allow_markup = allow_markup @classmethod def from_field(cls, name, field): return cls(name, field.get('description'), \ - field.get('ibl_extractor', text), field.get('required', False), \ - field.get('allow_markup', False)) + field.get('ibl_extractor', text), field.get('required', False)) def __str__(self): return "FieldDescriptor(%s)" % self.name diff --git a/scrapy/contrib/ibl/extraction/pageobjects.py b/scrapy/contrib/ibl/extraction/pageobjects.py index fa290d3c7..f7a42417f 100644 --- a/scrapy/contrib/ibl/extraction/pageobjects.py +++ b/scrapy/contrib/ibl/extraction/pageobjects.py @@ -4,16 +4,14 @@ Page objects This module contains objects representing pages and parts of pages (e.g. tokens and annotations) used in the instance based learning algorithm. """ +from itertools import chain from numpy import array, ndarray -from scrapy.contrib.ibl.htmlpage import HtmlTagType +from scrapy.contrib.ibl.htmlpage import HtmlTagType, HtmlPageRegion -class TokenType(object): +class TokenType(HtmlTagType): """constants for token types""" WORD = 0 - OPEN_TAG = HtmlTagType.OPEN_TAG - CLOSE_TAG = HtmlTagType.CLOSE_TAG - NON_PAIRED_TAG = HtmlTagType.UNPAIRED_TAG class TokenDict(object): """Mapping from parse tokens to integers @@ -68,6 +66,36 @@ class TokenDict(object): templates = ["%s", "<%s>", "", "<%s/>"] return templates[tid >> 24] % self.find_token(tid) +class PageRegion(object): + """A region in a page, defined by a start and end index""" + + __slots__ = ('start_index', 'end_index') + + def __init__(self, start, end): + self.start_index = start + self.end_index = end + + def __str__(self): + return "%s(%s, %s)" % (self.__class__.__name__, self.start_index, + self.end_index) + + def __repr__(self): + return str(self) + +class FragmentedHtmlPageRegion(HtmlPageRegion): + """An HtmlPageRegion consisting of possibly non-contiguous sub-regions""" + def __new__(cls, htmlpage, regions): + text = u''.join(regions) + return HtmlPageRegion.__new__(cls, htmlpage, text) + + def __init__(self, htmlpage, regions): + self.htmlpage = htmlpage + self.regions = regions + + @property + def parsed_fragments(self): + return chain(*(r.parsed_fragments for r in self.regions)) + class Page(object): """Basic representation of a page. This consists of a reference to a dictionary of tokens and an array of raw token ids @@ -92,7 +120,8 @@ class TemplatePage(Page): annotations = sorted(annotations, key=lambda x: x.end_index, reverse=True) self.annotations = sorted(annotations, key=lambda x: x.start_index) self.id = template_id - self.ignored_regions = ignored_regions or [] + self.ignored_regions = [i if isinstance(i, PageRegion) else PageRegion(*i) \ + for i in (ignored_regions or [])] self.extra_required_attrs = set(extra_required or []) def __str__(self): @@ -107,66 +136,53 @@ class ExtractionPage(Page): """Parsed data belonging to a web page upon which we wish to perform extraction. """ - __slots__ = ('text', - 'token_start_indexes', # index in text of the start of a token - 'token_follow_indexes', # index in text of data following token - 'tag_attributes' # a map from token index to tag attributes - ) + __slots__ = ('htmlpage', 'token_page_indexes') - def __init__(self, text, token_dict, page_tokens, token_start_indexes, - token_follow_indexes, tag_attributes): + def __init__(self, htmlpage, token_dict, page_tokens, token_page_indexes): + """Construct a new ExtractionPage + + Arguments: + `htmlpage`: The source HtmlPage + `token_dict`: Token Dictionary used for tokenization + `page_tokens': array of page tokens for matching + `token_page_indexes`: indexes of each token in the parsed htmlpage + """ Page.__init__(self, token_dict, page_tokens) - self.text = text - self.token_start_indexes = token_start_indexes - self.token_follow_indexes = token_follow_indexes - self.tag_attributes = tag_attributes - - def token_html(self, token_index): - """The raw html for a page token at the given index in the page_tokens - list - """ - text_start = self.token_start_indexes[token_index] - text_end = self.token_follow_indexes[token_index] - return self.text[text_start:text_end] - - def html_between_tokens(self, start_token_index, end_token_index): - """The raw html between the tokens at the specified indexes in the - page_tokens list - - This assumes start_token_index <= end_token_index - """ - text_start = self.token_follow_indexes[start_token_index] - text_end = self.token_start_indexes[end_token_index] - return self.text[text_start:text_end] + self.htmlpage = htmlpage + self.token_page_indexes = token_page_indexes - def text_between_tokens(self, start_token_index, end_token_index, - tag_replacement=u' '): - """The text between the the tokens at the specified indexes in the - page_tokens list. Tags are replaced by tag_replacement (default one space - character) - """ - return tag_replacement.join([self.text[ - self.token_follow_indexes[i]:self.token_start_indexes[i+1]] \ - for i in xrange(start_token_index, end_token_index)]) + def htmlpage_region(self, start_token_index, end_token_index): + """The region in the HtmlPage corresonding to the area defined by + the start_token_index and the end_token_index - def tag_attribute(self, token_index, attribute): - """The value of a tag attribute. The tag is identified by its - corresponding token index. - - If the tag or attribute is not present, None is returned + This includes the tokens at the specified indexes """ - return self.tag_attributes.get(token_index, {}).get(attribute) + start = self.token_page_indexes[start_token_index] + end = self.token_page_indexes[end_token_index] + return self.htmlpage.subregion(start, end) + + def htmlpage_region_inside(self, start_token_index, end_token_index): + """The region in the HtmlPage corresonding to the area between + the start_token_index and the end_token_index. + + This excludes the tokens at the specified indexes + """ + start = self.token_page_indexes[start_token_index] + 1 + end = self.token_page_indexes[end_token_index] - 1 + return self.htmlpage.subregion(start, end) + def htmlpage_tag(self, token_index): + """The HtmlPage tag at corresponding to the token at token_index""" + return self.htmlpage.parsed_body[self.token_page_indexes[token_index]] + def __str__(self): summary = [] - for (token, start, follow) in zip(self.page_tokens, self.token_start_indexes, - self.token_follow_indexes): - text = "%s %s-%s (%s)" % (self.token_dict.find_token(token), start, follow, - self.text[start:follow]) + for token, tindex in zip(self.page_tokens, self.token_page_indexes): + text = "%s page[%s]: %s" % (self.token_dict.find_token(token), + tindex, self.htmlpage.parsed_body[tindex]) summary.append(text) return "ExtractionPage\n==============\nTokens: %s\n\nRaw text: %s\n\n" \ - "Tag attributes: %s\n" % ('\n'.join(summary), self.text, - self.tag_attributes) + % ('\n'.join(summary), self.htmlpage.body) class AnnotationText(object): __slots__ = ('start_text', 'follow_text') @@ -179,7 +195,8 @@ class AnnotationText(object): return "AnnotationText(%s..%s)" % \ (repr(self.start_text), repr(self.follow_text)) -class AnnotationTag(object): + +class AnnotationTag(PageRegion): """A tag that annotates part of the document It has the following properties: @@ -197,8 +214,7 @@ class AnnotationTag(object): def __init__(self, start_index, end_index, surrounds_attribute=None, annotation_text=None, tag_attributes=None, variant_id=None): - self.start_index = start_index - self.end_index = end_index + PageRegion.__init__(self, start_index, end_index) self.surrounds_attribute = surrounds_attribute self.annotation_text = annotation_text self.tag_attributes = tag_attributes or [] @@ -213,16 +229,3 @@ class AnnotationTag(object): def __repr__(self): return str(self) -class LabelledRegion(object): - __slots__ = ('start_index', 'end_index') - - def __init__(self, start, end): - self.start_index = start - self.end_index = end - - def __str__(self): - return "LabelledRegion (%s, %s)" % (self.start_index, self.end_index) - - def __repr__(self): - return str(self) - diff --git a/scrapy/contrib/ibl/extraction/pageparsing.py b/scrapy/contrib/ibl/extraction/pageparsing.py index 2df63267f..1ce6633db 100644 --- a/scrapy/contrib/ibl/extraction/pageparsing.py +++ b/scrapy/contrib/ibl/extraction/pageparsing.py @@ -9,8 +9,8 @@ from numpy import array from scrapy.utils.py26 import json from scrapy.contrib.ibl.htmlpage import HtmlTagType, HtmlTag, HtmlPage -from scrapy.contrib.ibl.extraction.pageobjects import (AnnotationTag, - TemplatePage, ExtractionPage, AnnotationText, TokenDict) +from scrapy.contrib.ibl.extraction.pageobjects import (AnnotationTag, + TemplatePage, ExtractionPage, AnnotationText, TokenDict, FragmentedHtmlPageRegion) def parse_strings(template_html, extraction_html): """Create a template and extraction page from raw strings @@ -52,18 +52,18 @@ class InstanceLearningParser(object): def feed(self, html_page): self.html_page = html_page self.previous_element_class = None - for data in html_page.parsed_body: + for index, data in enumerate(html_page.parsed_body): if isinstance(data, HtmlTag): self._add_token(data.tag, data.tag_type, data.start, data.end) - self.handle_tag(data) + self.handle_tag(data, index) else: - self.handle_data(data) + self.handle_data(data, index) self.previous_element_class = data.__class__ - def handle_data(self, html_data_fragment): + def handle_data(self, html_data_fragment, index): pass - def handle_tag(self, html_tag): + def handle_tag(self, html_tag, index): pass _END_UNPAIREDTAG_TAGS = ["form", "div", "p", "table", "tr", "td"] @@ -86,7 +86,7 @@ class TemplatePageParser(InstanceLearningParser): self.last_text_region = None self.next_tag_index = 0 - def handle_tag(self, html_tag): + def handle_tag(self, html_tag, index): if self.last_text_region: self._process_text('') @@ -275,7 +275,7 @@ class TemplatePageParser(InstanceLearningParser): if prev != annotation.variant_id: raise ValueError("unbalanced variant annotation tags") - def handle_data(self, html_data_fragment): + def handle_data(self, html_data_fragment, index): fragment_text = self.html_page.fragment_data(html_data_fragment) self._process_text(fragment_text) @@ -300,20 +300,11 @@ class ExtractionPageParser(InstanceLearningParser): """ def __init__(self, token_dict): InstanceLearningParser.__init__(self, token_dict) - self.page_data = [] - self.token_start_index = [] - self.token_follow_index = [] - self.tag_attrs = {} + self._page_token_indexes = [] - def _add_token(self, token, token_type, start, end): - InstanceLearningParser._add_token(self, token, token_type, start, end) - self.token_start_index.append(start) - self.token_follow_index.append(end) - - def handle_tag(self, html_tag): - if html_tag.attributes: - self.tag_attrs[len(self.token_list) - 1] = html_tag.attributes + def handle_tag(self, html_tag, index): + self._page_token_indexes.append(index) def to_extraction_page(self): - return ExtractionPage(self.html_page.body, self.token_dict, array(self.token_list), - self.token_start_index, self.token_follow_index, self.tag_attrs) + return ExtractionPage(self.html_page, self.token_dict, array(self.token_list), + self._page_token_indexes) diff --git a/scrapy/contrib/ibl/extraction/regionextract.py b/scrapy/contrib/ibl/extraction/regionextract.py index c748a6a79..9c1a72674 100644 --- a/scrapy/contrib/ibl/extraction/regionextract.py +++ b/scrapy/contrib/ibl/extraction/regionextract.py @@ -8,14 +8,16 @@ import operator import copy import pprint import cStringIO -from itertools import groupby +from itertools import groupby, izip, starmap from numpy import array from scrapy.contrib.ibl.descriptor import FieldDescriptor +from scrapy.contrib.ibl.htmlpage import HtmlPageRegion from scrapy.contrib.ibl.extraction.similarity import (similar_region, longest_unique_subsequence, common_prefix) -from scrapy.contrib.ibl.extraction.pageobjects import AnnotationTag, LabelledRegion +from scrapy.contrib.ibl.extraction.pageobjects import (AnnotationTag, + PageRegion, FragmentedHtmlPageRegion) def build_extraction_tree(template, type_descriptor, trace=True): """Build a tree of region extractors corresponding to the @@ -33,16 +35,14 @@ def build_extraction_tree(template, type_descriptor, trace=True): return TemplatePageExtractor(template, extractors) -_ID = lambda x: x +_EXTRACT_HTML = lambda x: x _DEFAULT_DESCRIPTOR = FieldDescriptor('none', None) def _labelled(obj): """ Returns labelled element of the object (extractor or labelled region) """ - if hasattr(obj, "annotation"): - return obj.annotation - return obj + return getattr(obj, 'annotation', obj) def _compose(f, g): """given unary functions f and g, return a function that computes f(g(x)) @@ -75,7 +75,7 @@ class BasicTypeExtractor(object): u'
x xx
',\ u'
a name id-9
') >>> ex = BasicTypeExtractor(template.annotations[0]) - >>> ex.extract(page, 0, 3, [LabelledRegion(1, 2)]) + >>> ex.extract(page, 0, 3, [PageRegion(1, 2)]) [(u'name', u'a name')] """ @@ -88,17 +88,15 @@ class BasicTypeExtractor(object): descriptor = attribute_descriptors.get(annotation.surrounds_attribute) if descriptor: self.content_validate = descriptor.extractor - self.allow_markup = descriptor.allow_markup else: - self.content_validate = _ID - self.allow_markup = False + self.content_validate = _EXTRACT_HTML self.extract = self._extract_content if annotation.tag_attributes: self.tag_data = [] for (tag_attr, extraction_attr) in annotation.tag_attributes: descriptor = attribute_descriptors.get(extraction_attr) - extractf = descriptor.extractor if descriptor else _ID + extractf = descriptor.extractor if descriptor else _EXTRACT_HTML self.tag_data.append((extractf, tag_attr, extraction_attr)) self.extract = self._extract_both if \ @@ -109,33 +107,32 @@ class BasicTypeExtractor(object): self._extract_attribute(page, start_index, end_index, ignored_regions) def _extract_content(self, extraction_page, start_index, end_index, ignored_regions=None): - # we might want to add opening/closing ul/ol/table if we have the - # middle of a region. This would require support in the scrapy - # cleansing. - complete_data = "" - start = start_index - end = ignored_regions[0].start_index if ignored_regions else end_index - while start is not None: - if self.allow_markup: - data = extraction_page.html_between_tokens(start, end) - else: - data = extraction_page.text_between_tokens(start, end) - complete_data += data - if ignored_regions: - start = ignored_regions[0].end_index - ignored_regions.pop(0) - end = ignored_regions[0].start_index if ignored_regions else end_index - else: - start = None - complete_data = self.content_validate(complete_data) - return [(self.annotation.surrounds_attribute, complete_data)] if complete_data else [] + # extract content between annotation indexes + if not ignored_regions: + region = extraction_page.htmlpage_region_inside(start_index, end_index) + else: + # assumes ignored_regions are completely contained within start and end index + assert (start_index <= ignored_regions[0].start_index and + end_index >= ignored_regions[-1].end_index) + starts = [start_index] + [i.end_index for i in ignored_regions] + ends = [i.start_index for i in ignored_regions] + if starts[-1] is not None: + ends.append(end_index) + included_regions = izip(starts, ends) + if ends[0] is None: + included_regions.next() + regions = starmap(extraction_page.htmlpage_region_inside, included_regions) + region = FragmentedHtmlPageRegion(extraction_page.htmlpage, list(regions)) + validated = self.content_validate(region) + return [(self.annotation.surrounds_attribute, validated)] if validated else [] def _extract_attribute(self, extraction_page, start_index, end_index, ignored_regions=None): data = [] for (f, ta, ea) in self.tag_data: - tag_value = extraction_page.tag_attribute(start_index, ta) + tag_value = extraction_page.htmlpage_tag(start_index).attributes.get(ta) if tag_value: - extracted = f(tag_value) + region = HtmlPageRegion(extraction_page.htmlpage, tag_value) + extracted = f(region) if extracted is not None: data.append((ea, extracted)) return data @@ -178,10 +175,8 @@ class BasicTypeExtractor(object): def __str__(self): messages = ['BasicTypeExtractor('] if self.annotation.surrounds_attribute: - messages += [self.annotation.surrounds_attribute, ': ', - 'html content' if self.allow_markup else 'text content', - ] - if self.content_validate != _ID: + messages.append(self.annotation.surrounds_attribute) + if self.content_validate != _EXTRACT_HTML: messages += [', extracted with \'', self.content_validate.__name__, '\''] @@ -190,7 +185,7 @@ class BasicTypeExtractor(object): messages.append(';') for (f, ta, ea) in self.tag_data: messages += [ea, ': tag attribute "', ta, '"'] - if f != _ID: + if f != _EXTRACT_HTML: messages += [', validated by ', str(f)] messages.append(", template[%s:%s])" % \ (self.annotation.start_index, self.annotation.end_index)) @@ -340,7 +335,8 @@ class RecordExtractor(object): The region in the page to be extracted from may be specified using start_index and end_index """ - ignored_regions = [i if isinstance(i, LabelledRegion) else LabelledRegion(*i) for i in (ignored_regions or [])] + if ignored_regions is None: + ignored_regions = [] region_elements = sorted(self.extractors + ignored_regions, key=lambda x: _labelled(x).start_index) _, _, attributes = self._doextract(page, region_elements, start_index, end_index) @@ -395,7 +391,7 @@ class RecordExtractor(object): s, p, e = similar_region(page.page_tokens, self.template_tokens, \ i, start, sindex) if s > 0: - similar_ignored_regions.append(LabelledRegion(p, e)) + similar_ignored_regions.append(PageRegion(p, e)) start = e or start extracted_data = first_region.extract(page, pindex, sindex, similar_ignored_regions) if extracted_data: @@ -493,12 +489,12 @@ class TraceExtractor(object): for t in template.page_tokens[tend:tend+5]]) def summarize_trace(self, page, start, end, ret): - text_start = page.token_follow_indexes[start] - text_end = page.token_start_indexes[end or -1] + text_start = page.htmlpage.parsed_body[page.token_page_indexes[start]].start + text_end = page.htmlpage.parsed_body[page.token_page_indexes[end or -1]].end page_snippet = "(...%s)%s(%s...)" % ( - page.text[text_start-50:text_start].replace('\n', ' '), - page.text[text_start:text_end], - page.text[text_end:text_end+50].replace('\n', ' ')) + page.htmlpage.body[text_start-50:text_start].replace('\n', ' '), + page.htmlpage.body[text_start:text_end], + page.htmlpage.body[text_end:text_end+50].replace('\n', ' ')) pre_summary = "\nstart %s page[%s:%s]\n" % (self.traced.__class__.__name__, start, end) post_summary = """ %s page[%s:%s] @@ -572,25 +568,26 @@ class TemplatePageExtractor(object): _tokenize = re.compile(r'\w+|[^\w\s]+', re.UNICODE | re.MULTILINE | re.DOTALL).findall class TextRegionDataExtractor(object): - """Data Extractor for extracting text fragments from within a larger - body of text. It extracts based on the longest unique prefix and suffix. + """Data Extractor for extracting text fragments from an annotation page + fragment or string. It extracts based on the longest unique prefix and + suffix. for example: >>> extractor = TextRegionDataExtractor('designed by ', '.') - >>> extractor.extract("by Marc Newson.") + >>> extractor.extract_text("by Marc Newson.") 'Marc Newson' Both prefix and suffix are optional: >>> extractor = TextRegionDataExtractor('designed by ') - >>> extractor.extract("by Marc Newson.") + >>> extractor.extract_text("by Marc Newson.") 'Marc Newson.' >>> extractor = TextRegionDataExtractor(suffix='.') - >>> extractor.extract("by Marc Newson.") + >>> extractor.extract_text("by Marc Newson.") 'by Marc Newson' It requires a minimum match of at least one word or punctuation character: >>> extractor = TextRegionDataExtractor('designed by') - >>> extractor.extract("y Marc Newson.") is None + >>> extractor.extract_text("y Marc Newson.") is None True """ def __init__(self, prefix=None, suffix=None): @@ -609,8 +606,13 @@ class TextRegionDataExtractor(object): tokens = _tokenize(matchstring or '') return len(tokens[0]) if tokens else 0 - def extract(self, text): - """attempt to extract a substring from the text""" + def extract(self, region): + """Extract a region from the region passed""" + text = self.extract_text(region) + return HtmlPageRegion(region.htmlpage, text) if text else None + + def extract_text(self, text): + """Extract a substring from the text""" pref_index = 0 if self.minprefix > 0: rev_idx, plen = longest_unique_subsequence(text[::-1], self.prefix) @@ -623,5 +625,3 @@ class TextRegionDataExtractor(object): if slen < self.minsuffix: return None return text[pref_index:pref_index + sidx] - - diff --git a/scrapy/contrib/ibl/extractors.py b/scrapy/contrib/ibl/extractors.py index 858aa9d55..41bd4d437 100644 --- a/scrapy/contrib/ibl/extractors.py +++ b/scrapy/contrib/ibl/extractors.py @@ -4,8 +4,9 @@ Extractors for attributes import re import urlparse -from scrapy.utils.markup import remove_entities +from scrapy.utils.markup import remove_entities, remove_comments from scrapy.utils.url import safe_url_string +from scrapy.contrib.ibl.htmlpage import HtmlTag, HtmlTagType #FIXME: the use of "." needs to be localized _NUMERIC_ENTITIES = re.compile("&#([0-9]+)(?:;|\s)", re.U) @@ -22,11 +23,175 @@ _CSS_IMAGERE = re.compile("background(?:-image)?\s*:\s*url\((.*?)\)", re.I) _BASE_PATH_RE = "/?(?:[^/]+/)*(?:.+%s)" _IMAGE_PATH_RE = re.compile(_BASE_PATH_RE % '\.(?:%s)' % _IMAGES_TYPES, re.I) _GENERIC_PATH_RE = re.compile(_BASE_PATH_RE % '', re.I) +_WS = re.compile("\s+", re.U) -def text(txt): - stripped = txt.strip() if txt else None - if stripped: - return stripped +# tags to keep (only for attributes with markup) +_TAGS_TO_KEEP = frozenset(['br', 'p', 'big', 'em', 'small', 'strong', 'sub', + 'sup', 'ins', 'del', 'code', 'kbd', 'samp', 'tt', 'var', 'pre', 'listing', + 'plaintext', 'abbr', 'acronym', 'address', 'bdo', 'blockquote', 'q', + 'cite', 'dfn', 'table', 'tr', 'th', 'td', 'tbody', 'ul', 'ol', 'li', 'dl', + 'dd', 'dt']) + +# tag names to be replaced by other tag names (overrides tags_to_keep) +_TAGS_TO_REPLACE = { + 'h1': 'strong', + 'h2': 'strong', + 'h3': 'strong', + 'h4': 'strong', + 'h5': 'strong', + 'h6': 'strong', + 'b' : 'strong', + 'i' : 'em', +} + +# tags whoose content will be completely removed (recursively) +# (overrides tags_to_keep and tags_to_replace) +_TAGS_TO_PURGE = ('script', 'img', 'input') + +def htmlregion(text): + """convenience function to make an html region from text. + This is useful for testing + """ + from scrapy.contrib.ibl.htmlpage import HtmlPage + return HtmlPage(body=text).subregion() + +def notags(region, tag_replace=u' '): + """Removes all html tags""" + fragments = getattr(region, 'parsed_fragments', None) + if fragments is None: + return region + page = region.htmlpage + data = [page.fragment_data(f) for f in fragments if not isinstance(f, HtmlTag)] + return tag_replace.join(data) + +def text(region): + """Converts HTML to text. There is no attempt at formatting other than + removing excessive whitespace, + + For example: + >>> t = lambda s: text(htmlregion(s)) + >>> t(u'

test

') + u'test' + + Leading and trailing whitespace are removed + >>> t(u'

test

') + u'test' + + Comments are removed + >>> t(u'test me') + u'test me' + + Text between script tags is ignored + >>> t(u"scripts are ignored") + u'scripts are ignored' + + HTML entities are converted to text + >>> t(u"only £42") + u'only \\xa342' + """ + chunks = _process_markup(region, + lambda text: remove_entities(text, encoding=region.htmlpage.encoding), + lambda tag: u' ' + ) + text = u''.join(chunks) + return _WS.sub(u' ', text).strip() + +def safehtml(region, allowed_tags=_TAGS_TO_KEEP, replace_tags=_TAGS_TO_REPLACE): + """Creates an HTML subset, using a whitelist of HTML tags. + + The HTML generated is safe for display on a website,without escaping and + should not cause formatting problems. + + Allowed_tags is a set of tags that are allowed and replace_tags is a mapping of + tags to alternative tags to substitute. + + For example: + >>> t = lambda s: safehtml(htmlregion(s)) + >>> t(u'test test') + u'test test' + + Some tags, like script, are completely removed + >>> t(u'test') + u'test' + + replace_tags define tags that are converted. By default all headers, bold and indenting + are converted to strong and em. + >>> t(u'

header

test bold indent') + u'header test bold indent' + + Comments are stripped, but entities are not converted + >>> t(u' only £42') + u'only £42' + + Paired tags are closed + >>> t(u'

test') + u'

test

' + + >>> t(u'

test
test

') + u'

test
test

' + + """ + tagstack = [] + def _process_tag(tag): + tagstr = replace_tags.get(tag.tag, tag.tag) + if tagstr not in allowed_tags: + return + if tag.tag_type == HtmlTagType.OPEN_TAG: + tagstack.append(tagstr) + return u"<%s>" % tagstr + elif tag.tag_type == HtmlTagType.CLOSE_TAG: + try: + last = tagstack.pop() + # common case of matching tag + if last == tagstr: + return u"" % last + # output all preceeding tags (if present) + revtags = tagstack[::-1] + tindex = revtags.index(tagstr) + del tagstack[-tindex-1:] + return u"" % (last, u">" % tag.tag + chunks = list(_process_markup(region, lambda text: text, _process_tag)) + \ + ["" % t for t in reversed(tagstack)] + return u''.join(chunks).strip() + +def _process_markup(region, textf, tagf): + fragments = getattr(region, 'parsed_fragments', None) + if fragments is None: + textf(region) + return + fiter = iter(fragments) + for fragment in fiter: + if isinstance(fragment, HtmlTag): + # skip forward to closing script tags + tag = fragment.tag + if tag in _TAGS_TO_PURGE: + # if opening, keep going until closed + if fragment.tag_type == HtmlTagType.OPEN_TAG: + for probe in fiter: + if isinstance(probe, HtmlTag) and \ + probe.tag == tag and \ + probe.tag_type == HtmlTagType.CLOSE_TAG: + break + else: + output = tagf(fragment) + if output: + yield output + else: + text = region.htmlpage.fragment_data(fragment) + text = remove_comments(text) + text = textf(text) + if text: + yield text + +def html(pageregion): + """A page region is already html, so this is the identity function""" + return pageregion def contains_any_numbers(txt): """text that must contain at least one number diff --git a/scrapy/contrib/ibl/htmlpage.py b/scrapy/contrib/ibl/htmlpage.py index c86ec9670..e16ac2c7a 100644 --- a/scrapy/contrib/ibl/htmlpage.py +++ b/scrapy/contrib/ibl/htmlpage.py @@ -1,8 +1,9 @@ """ htmlpage -Container object for representing html pages in the IBL system. This -encapsulates page related information and prevents parsing multiple times. +Container objects for representing html pages and their parts in the IBL +system. This encapsulates page related information and prevents parsing +multiple times. """ import re import hashlib @@ -26,26 +27,82 @@ def create_page_from_jsonpage(jsonpage, body_key): return HtmlPage(url, headers, body, page_id) class HtmlPage(object): - def __init__(self, url=None, headers=None, body=None, page_id=None): + """HtmlPage + + This is a parsed HTML page. It contains the page headers, url, raw body and parsed + body. + + The parsed body is a list of HtmlDataFragment objects. + """ + def __init__(self, url=None, headers=None, body=None, page_id=None, encoding='utf-8'): assert isinstance(body, unicode), "unicode expected, got: %s" % type(body).__name__ self.headers = headers or {} self.body = body self.url = url or u'' + self.encoding = encoding if page_id is None and url: self.page_id = hashlib.sha1(url).hexdigest() else: self.page_id = page_id + @classmethod + def from_response(cls, response, page_id=None): + """Create an HtmlPage from a scrapy response""" + return HtmlPage(response.url, response.headers, + response.body_as_unicode(), page_id, response.encoding) def _set_body(self, body): self._body = body self.parsed_body = list(parse_html(body)) - body = property(lambda x: x._body, _set_body) + body = property(lambda x: x._body, _set_body, doc="raw html for the page") + def subregion(self, start=0, end=None): + """HtmlPageRegion constructed from the start and end index (inclusive) + into the parsed page + """ + return HtmlPageParsedRegion(self, start, end) + def fragment_data(self, data_fragment): + """portion of the body corresponding to the HtmlDataFragment""" return self.body[data_fragment.start:data_fragment.end] + +class HtmlPageRegion(unicode): + """A Region of an HtmlPage that has been extracted + """ + def __new__(cls, htmlpage, data): + return unicode.__new__(cls, data) + + def __init__(self, htmlpage, data): + """Construct a new HtmlPageRegion object. + + htmlpage is the original page and data is the raw html + """ + self.htmlpage = htmlpage +class HtmlPageParsedRegion(HtmlPageRegion): + """A region of an HtmlPage that has been extracted + + This has a parsed_fragments property that contains the parsed html + fragments contained within this region + """ + def __new__(cls, htmlpage, start_index, end_index): + text_start = htmlpage.parsed_body[start_index].start + text_end = htmlpage.parsed_body[end_index or -1].end + text = htmlpage.body[text_start:text_end] + return HtmlPageRegion.__new__(cls, htmlpage, text) + + def __init__(self, htmlpage, start_index, end_index): + self.htmlpage = htmlpage + self.start_index = start_index + self.end_index = end_index + + @property + def parsed_fragments(self): + """HtmlDataFragment or HtmlTag objects for this parsed region""" + end = self.end_index + 1 if self.end_index is not None else None + return self.htmlpage.parsed_body[self.start_index:end] + class HtmlTagType(object): OPEN_TAG = 1 CLOSE_TAG = 2 diff --git a/scrapy/tests/test_contrib_ibl/test_extraction.py b/scrapy/tests/test_contrib_ibl/test_extraction.py index e9ce86f7e..3a7bdd393 100644 --- a/scrapy/tests/test_contrib_ibl/test_extraction.py +++ b/scrapy/tests/test_contrib_ibl/test_extraction.py @@ -9,7 +9,7 @@ from scrapy.contrib.ibl.htmlpage import HtmlPage from scrapy.contrib.ibl.descriptor import (FieldDescriptor as A, ItemDescriptor) from scrapy.contrib.ibl.extractors import (contains_any_numbers, - image_url) + image_url, html, notags) try: import numpy @@ -719,19 +719,23 @@ EXTRACT_PAGE23 = u""" """ +DEFAULT_DESCRIPTOR = ItemDescriptor('test', + 'item test, removes tags from description attribute', + [A('description', 'description field without tags', notags)]) + SAMPLE_DESCRIPTOR1 = ItemDescriptor('test', 'product test', [ A('name', "Product name", required=True), A('price', "Product price, including any discounts and tax or vat", contains_any_numbers, True), A('image_urls', "URLs for one or more images", image_url, True), - A('description', "The full description of the product", allow_markup=True), + A('description', "The full description of the product", html), ] ) # A list of (test name, [templates], page, extractors, expected_result) TEST_DATA = [ # extract from a similar page - ('similar page extraction', [ANNOTATED_PAGE1], EXTRACT_PAGE1, None, + ('similar page extraction', [ANNOTATED_PAGE1], EXTRACT_PAGE1, DEFAULT_DESCRIPTOR, {u'title': [u'Nice Product'], u'description': [u'wonderful product'], u'image_url': [u'nice_product.jpg']} ), @@ -743,20 +747,20 @@ TEST_DATA = [ u'image_url': [u'nice_product.jpg']} ), # compilicated tag (multiple attributes and annotation) - ('multiple attributes and annotation', [ANNOTATED_PAGE2], EXTRACT_PAGE2, None, + ('multiple attributes and annotation', [ANNOTATED_PAGE2], EXTRACT_PAGE2, DEFAULT_DESCRIPTOR, {'name': [u'product 1'], 'image_url': [u'http://example.com/product1.jpg'], 'description': [u'product 1 is great']} ), # can only work out correct placement by matching the second attribute first - ('ambiguous description', [ANNOTATED_PAGE3], EXTRACT_PAGE3, None, + ('ambiguous description', [ANNOTATED_PAGE3], EXTRACT_PAGE3, DEFAULT_DESCRIPTOR, {'description': [u'description'], 'delivery': [u'delivery']} ), # infer a repeated structure - ('repeated elements', [ANNOTATED_PAGE4], EXTRACT_PAGE4, None, + ('repeated elements', [ANNOTATED_PAGE4], EXTRACT_PAGE4, DEFAULT_DESCRIPTOR, {'features': [u'feature1', u'feature2', u'feature3']} ), # identical variants with a repeated structure - ('repeated identical variants', [ANNOTATED_PAGE5], EXTRACT_PAGE5, None, + ('repeated identical variants', [ANNOTATED_PAGE5], EXTRACT_PAGE5, DEFAULT_DESCRIPTOR, { 'description': [u'description'], 'variants': [ @@ -767,7 +771,7 @@ TEST_DATA = [ } ), # variants with an irregular structure - ('irregular variants', [ANNOTATED_PAGE6], EXTRACT_PAGE6, None, + ('irregular variants', [ANNOTATED_PAGE6], EXTRACT_PAGE6, DEFAULT_DESCRIPTOR, { 'description': [u'description'], 'variants': [ @@ -779,7 +783,7 @@ TEST_DATA = [ ), # discovering repeated variants in table columns -# ('variants in table columns', [ANNOTATED_PAGE7], EXTRACT_PAGE7, None, +# ('variants in table columns', [ANNOTATED_PAGE7], EXTRACT_PAGE7, DEFAULT_DESCRIPTOR, # {'variants': [ # {u'colour': [u'colour 1'], u'price': [u'price 1']}, # {u'colour': [u'colour 2'], u'price': [u'price 2']}, @@ -790,66 +794,66 @@ TEST_DATA = [ # ignored regions ( - 'ignored_regions', [ANNOTATED_PAGE8], EXTRACT_PAGE8, None, + 'ignored_regions', [ANNOTATED_PAGE8], EXTRACT_PAGE8, DEFAULT_DESCRIPTOR, { - 'description': [u'\n A very nice product for all intelligent people \n\n'], + 'description': [u'\n A very nice product for all intelligent people \n \n'], 'price': [u'\n12.00\n(VAT exc.)'], } ), # shifted ignored regions (detected by region similarity) ( - 'shifted_ignored_regions', [ANNOTATED_PAGE9], EXTRACT_PAGE9, None, + 'shifted_ignored_regions', [ANNOTATED_PAGE9], EXTRACT_PAGE9, DEFAULT_DESCRIPTOR, { - 'description': [u'\n A very nice product for all intelligent people \n\n'], + 'description': [u'\n A very nice product for all intelligent people \n \n'], 'price': [u'\n12.00\n(VAT exc.)'], } ), (# special case with partial annotations - 'special_partial_annotation', [ANNOTATED_PAGE11], EXTRACT_PAGE11, None, + 'special_partial_annotation', [ANNOTATED_PAGE11], EXTRACT_PAGE11, DEFAULT_DESCRIPTOR, { 'name': [u'SL342'], - 'description': [u'\nSL342\n \nNice product for ladies\n \n£s;85.00\n'], + 'description': ['\nSL342\n \nNice product for ladies\n \n£s;85.00\n'], 'price': [u'£s;85.00'], 'price_before_discount': [u'£s;100.00'], } ), (# with ignore-beneath feature - 'ignore-beneath', [ANNOTATED_PAGE12], EXTRACT_PAGE12a, None, + 'ignore-beneath', [ANNOTATED_PAGE12], EXTRACT_PAGE12a, DEFAULT_DESCRIPTOR, { 'description': [u'\n A very nice product for all intelligent people \n'], } ), (# ignore-beneath with extra tags - 'ignore-beneath with extra tags', [ANNOTATED_PAGE12], EXTRACT_PAGE12b, None, + 'ignore-beneath with extra tags', [ANNOTATED_PAGE12], EXTRACT_PAGE12b, DEFAULT_DESCRIPTOR, { 'description': [u'\n A very nice product for all intelligent people \n'], } ), - ('nested annotation with replica outside', [ANNOTATED_PAGE13a], EXTRACT_PAGE13a, None, + ('nested annotation with replica outside', [ANNOTATED_PAGE13a], EXTRACT_PAGE13a, DEFAULT_DESCRIPTOR, {'description': [u'\n A product \n $50.00 \nThis product is excelent. Buy it!\n \n'], 'price': ["$50.00"], 'name': [u'A product']} ), - ('outside annotation with nested replica', [ANNOTATED_PAGE13b], EXTRACT_PAGE13b, None, + ('outside annotation with nested replica', [ANNOTATED_PAGE13b], EXTRACT_PAGE13b, DEFAULT_DESCRIPTOR, {'description': [u'\n A product \n $50.00 \nThis product is excelent. Buy it!\n'], 'price': ["$45.00"], 'name': [u'A product']} ), - ('consistency check', [ANNOTATED_PAGE14], EXTRACT_PAGE14, None, + ('consistency check', [ANNOTATED_PAGE14], EXTRACT_PAGE14, DEFAULT_DESCRIPTOR, {}, ), - ('consecutive nesting', [ANNOTATED_PAGE15], EXTRACT_PAGE15, None, - {'description': [u'Description\n\n'], + ('consecutive nesting', [ANNOTATED_PAGE15], EXTRACT_PAGE15, DEFAULT_DESCRIPTOR, + {'description': [u'Description\n \n'], 'price': [u'80.00']}, ), - ('nested inside not found', [ANNOTATED_PAGE16], EXTRACT_PAGE16, None, + ('nested inside not found', [ANNOTATED_PAGE16], EXTRACT_PAGE16, DEFAULT_DESCRIPTOR, {'price': [u'90.00'], 'name': [u'product name']}, ), - ('ignored region helps to find attributes', [ANNOTATED_PAGE17], EXTRACT_PAGE17, None, + ('ignored region helps to find attributes', [ANNOTATED_PAGE17], EXTRACT_PAGE17, DEFAULT_DESCRIPTOR, {'description': [u'\nThis product is excelent. Buy it!\n']}, ), - ('ignored region in partial annotation', [ANNOTATED_PAGE18], EXTRACT_PAGE18, None, + ('ignored region in partial annotation', [ANNOTATED_PAGE18], EXTRACT_PAGE18, DEFAULT_DESCRIPTOR, {u'site_id': [u'Item Id'], u'description': [u'\nDescription\n']}, ), @@ -864,13 +868,13 @@ TEST_DATA = [ SAMPLE_DESCRIPTOR1, None, ), - ('repeated partial annotations with variants', [ANNOTATED_PAGE20], EXTRACT_PAGE20, None, + ('repeated partial annotations with variants', [ANNOTATED_PAGE20], EXTRACT_PAGE20, DEFAULT_DESCRIPTOR, {u'variants': [ {'price': ['270'], 'name': ['Twin']}, {'price': ['330'], 'name': ['Queen']}, ]}, ), - ('variants with swatches', [ANNOTATED_PAGE21], EXTRACT_PAGE21, None, + ('variants with swatches', [ANNOTATED_PAGE21], EXTRACT_PAGE21, DEFAULT_DESCRIPTOR, {u'category': [u'chairs'], u'image_urls': [u'image.jpg'], u'variants': [ @@ -881,7 +885,7 @@ TEST_DATA = [ ] }, ), - ('variants with swatches complete', [ANNOTATED_PAGE22], EXTRACT_PAGE22, None, + ('variants with swatches complete', [ANNOTATED_PAGE22], EXTRACT_PAGE22, DEFAULT_DESCRIPTOR, {u'category': [u'chairs'], u'variants': [ {u'swatches': [u'swatch1.jpg'], @@ -899,7 +903,7 @@ TEST_DATA = [ ], u'image_urls': [u'image.jpg']}, ), - ('repeated (variants) with ignore annotations', [ANNOTATED_PAGE23], EXTRACT_PAGE23, None, + ('repeated (variants) with ignore annotations', [ANNOTATED_PAGE23], EXTRACT_PAGE23, DEFAULT_DESCRIPTOR, {'variants': [ {u'price': [u'300'], u'name': [u'Variant 1']}, {u'price': [u'320'], u'name': [u'Variant 2']}, diff --git a/scrapy/tests/test_contrib_ibl/test_pageparsing.py b/scrapy/tests/test_contrib_ibl/test_pageparsing.py index 1e3d610bb..d694d08a1 100644 --- a/scrapy/tests/test_contrib_ibl/test_pageparsing.py +++ b/scrapy/tests/test_contrib_ibl/test_pageparsing.py @@ -228,11 +228,11 @@ class TestPageParsing(TestCase): epp = _parse_page(ExtractionPageParser, SIMPLE_PAGE) ep = epp.to_extraction_page() assert len(ep.page_tokens) == 4 - assert ep.token_html(0) == '' - assert ep.token_html(1) == '

' + assert ep.htmlpage.fragment_data(ep.htmlpage_tag(0)) == '' + assert ep.htmlpage.fragment_data(ep.htmlpage_tag(1)) == '

' - assert ep.html_between_tokens(1, 2) == 'this is a test' - assert ep.html_between_tokens(1, 3) == 'this is a test

' + assert ep.htmlpage_region_inside(1, 2) == 'this is a test' + assert ep.htmlpage_region_inside(1, 3) == 'this is a test

' def test_invalid_html(self): p = _parse_page(InstanceLearningParser, BROKEN_PAGE)