diff --git a/scrapy/contrib/ibl/descriptor.py b/scrapy/contrib/ibl/descriptor.py
index 5b4ae99ce..075920d87 100644
--- a/scrapy/contrib/ibl/descriptor.py
+++ b/scrapy/contrib/ibl/descriptor.py
@@ -3,25 +3,22 @@ Extended types for IBL extraction
"""
from itertools import chain
-from scrapy.contrib.ibl.extractors import text
+from scrapy.contrib.ibl.extractors import text, html
class FieldDescriptor(object):
"""description of a scraped attribute"""
- __slots__ = ('name', 'description', 'extractor', 'required', 'allow_markup')
+ __slots__ = ('name', 'description', 'extractor', 'required')
- def __init__(self, name, description, extractor=text, required=False,
- allow_markup=False):
+ def __init__(self, name, description, extractor=text, required=False):
self.name = name
self.description = description
self.extractor = extractor
self.required = required
- self.allow_markup = allow_markup
@classmethod
def from_field(cls, name, field):
return cls(name, field.get('description'), \
- field.get('ibl_extractor', text), field.get('required', False), \
- field.get('allow_markup', False))
+ field.get('ibl_extractor', text), field.get('required', False))
def __str__(self):
return "FieldDescriptor(%s)" % self.name
diff --git a/scrapy/contrib/ibl/extraction/pageobjects.py b/scrapy/contrib/ibl/extraction/pageobjects.py
index fa290d3c7..f7a42417f 100644
--- a/scrapy/contrib/ibl/extraction/pageobjects.py
+++ b/scrapy/contrib/ibl/extraction/pageobjects.py
@@ -4,16 +4,14 @@ Page objects
This module contains objects representing pages and parts of pages (e.g. tokens
and annotations) used in the instance based learning algorithm.
"""
+from itertools import chain
from numpy import array, ndarray
-from scrapy.contrib.ibl.htmlpage import HtmlTagType
+from scrapy.contrib.ibl.htmlpage import HtmlTagType, HtmlPageRegion
-class TokenType(object):
+class TokenType(HtmlTagType):
"""constants for token types"""
WORD = 0
- OPEN_TAG = HtmlTagType.OPEN_TAG
- CLOSE_TAG = HtmlTagType.CLOSE_TAG
- NON_PAIRED_TAG = HtmlTagType.UNPAIRED_TAG
class TokenDict(object):
"""Mapping from parse tokens to integers
@@ -68,6 +66,36 @@ class TokenDict(object):
templates = ["%s", "<%s>", "", "<%s/>"]
return templates[tid >> 24] % self.find_token(tid)
+class PageRegion(object):
+ """A region in a page, defined by a start and end index"""
+
+ __slots__ = ('start_index', 'end_index')
+
+ def __init__(self, start, end):
+ self.start_index = start
+ self.end_index = end
+
+ def __str__(self):
+ return "%s(%s, %s)" % (self.__class__.__name__, self.start_index,
+ self.end_index)
+
+ def __repr__(self):
+ return str(self)
+
+class FragmentedHtmlPageRegion(HtmlPageRegion):
+ """An HtmlPageRegion consisting of possibly non-contiguous sub-regions"""
+ def __new__(cls, htmlpage, regions):
+ text = u''.join(regions)
+ return HtmlPageRegion.__new__(cls, htmlpage, text)
+
+ def __init__(self, htmlpage, regions):
+ self.htmlpage = htmlpage
+ self.regions = regions
+
+ @property
+ def parsed_fragments(self):
+ return chain(*(r.parsed_fragments for r in self.regions))
+
class Page(object):
"""Basic representation of a page. This consists of a reference to a
dictionary of tokens and an array of raw token ids
@@ -92,7 +120,8 @@ class TemplatePage(Page):
annotations = sorted(annotations, key=lambda x: x.end_index, reverse=True)
self.annotations = sorted(annotations, key=lambda x: x.start_index)
self.id = template_id
- self.ignored_regions = ignored_regions or []
+ self.ignored_regions = [i if isinstance(i, PageRegion) else PageRegion(*i) \
+ for i in (ignored_regions or [])]
self.extra_required_attrs = set(extra_required or [])
def __str__(self):
@@ -107,66 +136,53 @@ class ExtractionPage(Page):
"""Parsed data belonging to a web page upon which we wish to perform
extraction.
"""
- __slots__ = ('text',
- 'token_start_indexes', # index in text of the start of a token
- 'token_follow_indexes', # index in text of data following token
- 'tag_attributes' # a map from token index to tag attributes
- )
+ __slots__ = ('htmlpage', 'token_page_indexes')
- def __init__(self, text, token_dict, page_tokens, token_start_indexes,
- token_follow_indexes, tag_attributes):
+ def __init__(self, htmlpage, token_dict, page_tokens, token_page_indexes):
+ """Construct a new ExtractionPage
+
+ Arguments:
+ `htmlpage`: The source HtmlPage
+ `token_dict`: Token Dictionary used for tokenization
+ `page_tokens': array of page tokens for matching
+ `token_page_indexes`: indexes of each token in the parsed htmlpage
+ """
Page.__init__(self, token_dict, page_tokens)
- self.text = text
- self.token_start_indexes = token_start_indexes
- self.token_follow_indexes = token_follow_indexes
- self.tag_attributes = tag_attributes
-
- def token_html(self, token_index):
- """The raw html for a page token at the given index in the page_tokens
- list
- """
- text_start = self.token_start_indexes[token_index]
- text_end = self.token_follow_indexes[token_index]
- return self.text[text_start:text_end]
-
- def html_between_tokens(self, start_token_index, end_token_index):
- """The raw html between the tokens at the specified indexes in the
- page_tokens list
-
- This assumes start_token_index <= end_token_index
- """
- text_start = self.token_follow_indexes[start_token_index]
- text_end = self.token_start_indexes[end_token_index]
- return self.text[text_start:text_end]
+ self.htmlpage = htmlpage
+ self.token_page_indexes = token_page_indexes
- def text_between_tokens(self, start_token_index, end_token_index,
- tag_replacement=u' '):
- """The text between the the tokens at the specified indexes in the
- page_tokens list. Tags are replaced by tag_replacement (default one space
- character)
- """
- return tag_replacement.join([self.text[
- self.token_follow_indexes[i]:self.token_start_indexes[i+1]] \
- for i in xrange(start_token_index, end_token_index)])
+ def htmlpage_region(self, start_token_index, end_token_index):
+ """The region in the HtmlPage corresonding to the area defined by
+ the start_token_index and the end_token_index
- def tag_attribute(self, token_index, attribute):
- """The value of a tag attribute. The tag is identified by its
- corresponding token index.
-
- If the tag or attribute is not present, None is returned
+ This includes the tokens at the specified indexes
"""
- return self.tag_attributes.get(token_index, {}).get(attribute)
+ start = self.token_page_indexes[start_token_index]
+ end = self.token_page_indexes[end_token_index]
+ return self.htmlpage.subregion(start, end)
+
+ def htmlpage_region_inside(self, start_token_index, end_token_index):
+ """The region in the HtmlPage corresonding to the area between
+ the start_token_index and the end_token_index.
+
+ This excludes the tokens at the specified indexes
+ """
+ start = self.token_page_indexes[start_token_index] + 1
+ end = self.token_page_indexes[end_token_index] - 1
+ return self.htmlpage.subregion(start, end)
+ def htmlpage_tag(self, token_index):
+ """The HtmlPage tag at corresponding to the token at token_index"""
+ return self.htmlpage.parsed_body[self.token_page_indexes[token_index]]
+
def __str__(self):
summary = []
- for (token, start, follow) in zip(self.page_tokens, self.token_start_indexes,
- self.token_follow_indexes):
- text = "%s %s-%s (%s)" % (self.token_dict.find_token(token), start, follow,
- self.text[start:follow])
+ for token, tindex in zip(self.page_tokens, self.token_page_indexes):
+ text = "%s page[%s]: %s" % (self.token_dict.find_token(token),
+ tindex, self.htmlpage.parsed_body[tindex])
summary.append(text)
return "ExtractionPage\n==============\nTokens: %s\n\nRaw text: %s\n\n" \
- "Tag attributes: %s\n" % ('\n'.join(summary), self.text,
- self.tag_attributes)
+ % ('\n'.join(summary), self.htmlpage.body)
class AnnotationText(object):
__slots__ = ('start_text', 'follow_text')
@@ -179,7 +195,8 @@ class AnnotationText(object):
return "AnnotationText(%s..%s)" % \
(repr(self.start_text), repr(self.follow_text))
-class AnnotationTag(object):
+
+class AnnotationTag(PageRegion):
"""A tag that annotates part of the document
It has the following properties:
@@ -197,8 +214,7 @@ class AnnotationTag(object):
def __init__(self, start_index, end_index, surrounds_attribute=None,
annotation_text=None, tag_attributes=None, variant_id=None):
- self.start_index = start_index
- self.end_index = end_index
+ PageRegion.__init__(self, start_index, end_index)
self.surrounds_attribute = surrounds_attribute
self.annotation_text = annotation_text
self.tag_attributes = tag_attributes or []
@@ -213,16 +229,3 @@ class AnnotationTag(object):
def __repr__(self):
return str(self)
-class LabelledRegion(object):
- __slots__ = ('start_index', 'end_index')
-
- def __init__(self, start, end):
- self.start_index = start
- self.end_index = end
-
- def __str__(self):
- return "LabelledRegion (%s, %s)" % (self.start_index, self.end_index)
-
- def __repr__(self):
- return str(self)
-
diff --git a/scrapy/contrib/ibl/extraction/pageparsing.py b/scrapy/contrib/ibl/extraction/pageparsing.py
index 2df63267f..1ce6633db 100644
--- a/scrapy/contrib/ibl/extraction/pageparsing.py
+++ b/scrapy/contrib/ibl/extraction/pageparsing.py
@@ -9,8 +9,8 @@ from numpy import array
from scrapy.utils.py26 import json
from scrapy.contrib.ibl.htmlpage import HtmlTagType, HtmlTag, HtmlPage
-from scrapy.contrib.ibl.extraction.pageobjects import (AnnotationTag,
- TemplatePage, ExtractionPage, AnnotationText, TokenDict)
+from scrapy.contrib.ibl.extraction.pageobjects import (AnnotationTag,
+ TemplatePage, ExtractionPage, AnnotationText, TokenDict, FragmentedHtmlPageRegion)
def parse_strings(template_html, extraction_html):
"""Create a template and extraction page from raw strings
@@ -52,18 +52,18 @@ class InstanceLearningParser(object):
def feed(self, html_page):
self.html_page = html_page
self.previous_element_class = None
- for data in html_page.parsed_body:
+ for index, data in enumerate(html_page.parsed_body):
if isinstance(data, HtmlTag):
self._add_token(data.tag, data.tag_type, data.start, data.end)
- self.handle_tag(data)
+ self.handle_tag(data, index)
else:
- self.handle_data(data)
+ self.handle_data(data, index)
self.previous_element_class = data.__class__
- def handle_data(self, html_data_fragment):
+ def handle_data(self, html_data_fragment, index):
pass
- def handle_tag(self, html_tag):
+ def handle_tag(self, html_tag, index):
pass
_END_UNPAIREDTAG_TAGS = ["form", "div", "p", "table", "tr", "td"]
@@ -86,7 +86,7 @@ class TemplatePageParser(InstanceLearningParser):
self.last_text_region = None
self.next_tag_index = 0
- def handle_tag(self, html_tag):
+ def handle_tag(self, html_tag, index):
if self.last_text_region:
self._process_text('')
@@ -275,7 +275,7 @@ class TemplatePageParser(InstanceLearningParser):
if prev != annotation.variant_id:
raise ValueError("unbalanced variant annotation tags")
- def handle_data(self, html_data_fragment):
+ def handle_data(self, html_data_fragment, index):
fragment_text = self.html_page.fragment_data(html_data_fragment)
self._process_text(fragment_text)
@@ -300,20 +300,11 @@ class ExtractionPageParser(InstanceLearningParser):
"""
def __init__(self, token_dict):
InstanceLearningParser.__init__(self, token_dict)
- self.page_data = []
- self.token_start_index = []
- self.token_follow_index = []
- self.tag_attrs = {}
+ self._page_token_indexes = []
- def _add_token(self, token, token_type, start, end):
- InstanceLearningParser._add_token(self, token, token_type, start, end)
- self.token_start_index.append(start)
- self.token_follow_index.append(end)
-
- def handle_tag(self, html_tag):
- if html_tag.attributes:
- self.tag_attrs[len(self.token_list) - 1] = html_tag.attributes
+ def handle_tag(self, html_tag, index):
+ self._page_token_indexes.append(index)
def to_extraction_page(self):
- return ExtractionPage(self.html_page.body, self.token_dict, array(self.token_list),
- self.token_start_index, self.token_follow_index, self.tag_attrs)
+ return ExtractionPage(self.html_page, self.token_dict, array(self.token_list),
+ self._page_token_indexes)
diff --git a/scrapy/contrib/ibl/extraction/regionextract.py b/scrapy/contrib/ibl/extraction/regionextract.py
index c748a6a79..9c1a72674 100644
--- a/scrapy/contrib/ibl/extraction/regionextract.py
+++ b/scrapy/contrib/ibl/extraction/regionextract.py
@@ -8,14 +8,16 @@ import operator
import copy
import pprint
import cStringIO
-from itertools import groupby
+from itertools import groupby, izip, starmap
from numpy import array
from scrapy.contrib.ibl.descriptor import FieldDescriptor
+from scrapy.contrib.ibl.htmlpage import HtmlPageRegion
from scrapy.contrib.ibl.extraction.similarity import (similar_region,
longest_unique_subsequence, common_prefix)
-from scrapy.contrib.ibl.extraction.pageobjects import AnnotationTag, LabelledRegion
+from scrapy.contrib.ibl.extraction.pageobjects import (AnnotationTag,
+ PageRegion, FragmentedHtmlPageRegion)
def build_extraction_tree(template, type_descriptor, trace=True):
"""Build a tree of region extractors corresponding to the
@@ -33,16 +35,14 @@ def build_extraction_tree(template, type_descriptor, trace=True):
return TemplatePageExtractor(template, extractors)
-_ID = lambda x: x
+_EXTRACT_HTML = lambda x: x
_DEFAULT_DESCRIPTOR = FieldDescriptor('none', None)
def _labelled(obj):
"""
Returns labelled element of the object (extractor or labelled region)
"""
- if hasattr(obj, "annotation"):
- return obj.annotation
- return obj
+ return getattr(obj, 'annotation', obj)
def _compose(f, g):
"""given unary functions f and g, return a function that computes f(g(x))
@@ -75,7 +75,7 @@ class BasicTypeExtractor(object):
u'x xx
',\
u'a name id-9
')
>>> ex = BasicTypeExtractor(template.annotations[0])
- >>> ex.extract(page, 0, 3, [LabelledRegion(1, 2)])
+ >>> ex.extract(page, 0, 3, [PageRegion(1, 2)])
[(u'name', u'a name')]
"""
@@ -88,17 +88,15 @@ class BasicTypeExtractor(object):
descriptor = attribute_descriptors.get(annotation.surrounds_attribute)
if descriptor:
self.content_validate = descriptor.extractor
- self.allow_markup = descriptor.allow_markup
else:
- self.content_validate = _ID
- self.allow_markup = False
+ self.content_validate = _EXTRACT_HTML
self.extract = self._extract_content
if annotation.tag_attributes:
self.tag_data = []
for (tag_attr, extraction_attr) in annotation.tag_attributes:
descriptor = attribute_descriptors.get(extraction_attr)
- extractf = descriptor.extractor if descriptor else _ID
+ extractf = descriptor.extractor if descriptor else _EXTRACT_HTML
self.tag_data.append((extractf, tag_attr, extraction_attr))
self.extract = self._extract_both if \
@@ -109,33 +107,32 @@ class BasicTypeExtractor(object):
self._extract_attribute(page, start_index, end_index, ignored_regions)
def _extract_content(self, extraction_page, start_index, end_index, ignored_regions=None):
- # we might want to add opening/closing ul/ol/table if we have the
- # middle of a region. This would require support in the scrapy
- # cleansing.
- complete_data = ""
- start = start_index
- end = ignored_regions[0].start_index if ignored_regions else end_index
- while start is not None:
- if self.allow_markup:
- data = extraction_page.html_between_tokens(start, end)
- else:
- data = extraction_page.text_between_tokens(start, end)
- complete_data += data
- if ignored_regions:
- start = ignored_regions[0].end_index
- ignored_regions.pop(0)
- end = ignored_regions[0].start_index if ignored_regions else end_index
- else:
- start = None
- complete_data = self.content_validate(complete_data)
- return [(self.annotation.surrounds_attribute, complete_data)] if complete_data else []
+ # extract content between annotation indexes
+ if not ignored_regions:
+ region = extraction_page.htmlpage_region_inside(start_index, end_index)
+ else:
+ # assumes ignored_regions are completely contained within start and end index
+ assert (start_index <= ignored_regions[0].start_index and
+ end_index >= ignored_regions[-1].end_index)
+ starts = [start_index] + [i.end_index for i in ignored_regions]
+ ends = [i.start_index for i in ignored_regions]
+ if starts[-1] is not None:
+ ends.append(end_index)
+ included_regions = izip(starts, ends)
+ if ends[0] is None:
+ included_regions.next()
+ regions = starmap(extraction_page.htmlpage_region_inside, included_regions)
+ region = FragmentedHtmlPageRegion(extraction_page.htmlpage, list(regions))
+ validated = self.content_validate(region)
+ return [(self.annotation.surrounds_attribute, validated)] if validated else []
def _extract_attribute(self, extraction_page, start_index, end_index, ignored_regions=None):
data = []
for (f, ta, ea) in self.tag_data:
- tag_value = extraction_page.tag_attribute(start_index, ta)
+ tag_value = extraction_page.htmlpage_tag(start_index).attributes.get(ta)
if tag_value:
- extracted = f(tag_value)
+ region = HtmlPageRegion(extraction_page.htmlpage, tag_value)
+ extracted = f(region)
if extracted is not None:
data.append((ea, extracted))
return data
@@ -178,10 +175,8 @@ class BasicTypeExtractor(object):
def __str__(self):
messages = ['BasicTypeExtractor(']
if self.annotation.surrounds_attribute:
- messages += [self.annotation.surrounds_attribute, ': ',
- 'html content' if self.allow_markup else 'text content',
- ]
- if self.content_validate != _ID:
+ messages.append(self.annotation.surrounds_attribute)
+ if self.content_validate != _EXTRACT_HTML:
messages += [', extracted with \'',
self.content_validate.__name__, '\'']
@@ -190,7 +185,7 @@ class BasicTypeExtractor(object):
messages.append(';')
for (f, ta, ea) in self.tag_data:
messages += [ea, ': tag attribute "', ta, '"']
- if f != _ID:
+ if f != _EXTRACT_HTML:
messages += [', validated by ', str(f)]
messages.append(", template[%s:%s])" % \
(self.annotation.start_index, self.annotation.end_index))
@@ -340,7 +335,8 @@ class RecordExtractor(object):
The region in the page to be extracted from may be specified using
start_index and end_index
"""
- ignored_regions = [i if isinstance(i, LabelledRegion) else LabelledRegion(*i) for i in (ignored_regions or [])]
+ if ignored_regions is None:
+ ignored_regions = []
region_elements = sorted(self.extractors + ignored_regions, key=lambda x: _labelled(x).start_index)
_, _, attributes = self._doextract(page, region_elements, start_index,
end_index)
@@ -395,7 +391,7 @@ class RecordExtractor(object):
s, p, e = similar_region(page.page_tokens, self.template_tokens, \
i, start, sindex)
if s > 0:
- similar_ignored_regions.append(LabelledRegion(p, e))
+ similar_ignored_regions.append(PageRegion(p, e))
start = e or start
extracted_data = first_region.extract(page, pindex, sindex, similar_ignored_regions)
if extracted_data:
@@ -493,12 +489,12 @@ class TraceExtractor(object):
for t in template.page_tokens[tend:tend+5]])
def summarize_trace(self, page, start, end, ret):
- text_start = page.token_follow_indexes[start]
- text_end = page.token_start_indexes[end or -1]
+ text_start = page.htmlpage.parsed_body[page.token_page_indexes[start]].start
+ text_end = page.htmlpage.parsed_body[page.token_page_indexes[end or -1]].end
page_snippet = "(...%s)%s(%s...)" % (
- page.text[text_start-50:text_start].replace('\n', ' '),
- page.text[text_start:text_end],
- page.text[text_end:text_end+50].replace('\n', ' '))
+ page.htmlpage.body[text_start-50:text_start].replace('\n', ' '),
+ page.htmlpage.body[text_start:text_end],
+ page.htmlpage.body[text_end:text_end+50].replace('\n', ' '))
pre_summary = "\nstart %s page[%s:%s]\n" % (self.traced.__class__.__name__, start, end)
post_summary = """
%s page[%s:%s]
@@ -572,25 +568,26 @@ class TemplatePageExtractor(object):
_tokenize = re.compile(r'\w+|[^\w\s]+', re.UNICODE | re.MULTILINE | re.DOTALL).findall
class TextRegionDataExtractor(object):
- """Data Extractor for extracting text fragments from within a larger
- body of text. It extracts based on the longest unique prefix and suffix.
+ """Data Extractor for extracting text fragments from an annotation page
+ fragment or string. It extracts based on the longest unique prefix and
+ suffix.
for example:
>>> extractor = TextRegionDataExtractor('designed by ', '.')
- >>> extractor.extract("by Marc Newson.")
+ >>> extractor.extract_text("by Marc Newson.")
'Marc Newson'
Both prefix and suffix are optional:
>>> extractor = TextRegionDataExtractor('designed by ')
- >>> extractor.extract("by Marc Newson.")
+ >>> extractor.extract_text("by Marc Newson.")
'Marc Newson.'
>>> extractor = TextRegionDataExtractor(suffix='.')
- >>> extractor.extract("by Marc Newson.")
+ >>> extractor.extract_text("by Marc Newson.")
'by Marc Newson'
It requires a minimum match of at least one word or punctuation character:
>>> extractor = TextRegionDataExtractor('designed by')
- >>> extractor.extract("y Marc Newson.") is None
+ >>> extractor.extract_text("y Marc Newson.") is None
True
"""
def __init__(self, prefix=None, suffix=None):
@@ -609,8 +606,13 @@ class TextRegionDataExtractor(object):
tokens = _tokenize(matchstring or '')
return len(tokens[0]) if tokens else 0
- def extract(self, text):
- """attempt to extract a substring from the text"""
+ def extract(self, region):
+ """Extract a region from the region passed"""
+ text = self.extract_text(region)
+ return HtmlPageRegion(region.htmlpage, text) if text else None
+
+ def extract_text(self, text):
+ """Extract a substring from the text"""
pref_index = 0
if self.minprefix > 0:
rev_idx, plen = longest_unique_subsequence(text[::-1], self.prefix)
@@ -623,5 +625,3 @@ class TextRegionDataExtractor(object):
if slen < self.minsuffix:
return None
return text[pref_index:pref_index + sidx]
-
-
diff --git a/scrapy/contrib/ibl/extractors.py b/scrapy/contrib/ibl/extractors.py
index 858aa9d55..41bd4d437 100644
--- a/scrapy/contrib/ibl/extractors.py
+++ b/scrapy/contrib/ibl/extractors.py
@@ -4,8 +4,9 @@ Extractors for attributes
import re
import urlparse
-from scrapy.utils.markup import remove_entities
+from scrapy.utils.markup import remove_entities, remove_comments
from scrapy.utils.url import safe_url_string
+from scrapy.contrib.ibl.htmlpage import HtmlTag, HtmlTagType
#FIXME: the use of "." needs to be localized
_NUMERIC_ENTITIES = re.compile("&#([0-9]+)(?:;|\s)", re.U)
@@ -22,11 +23,175 @@ _CSS_IMAGERE = re.compile("background(?:-image)?\s*:\s*url\((.*?)\)", re.I)
_BASE_PATH_RE = "/?(?:[^/]+/)*(?:.+%s)"
_IMAGE_PATH_RE = re.compile(_BASE_PATH_RE % '\.(?:%s)' % _IMAGES_TYPES, re.I)
_GENERIC_PATH_RE = re.compile(_BASE_PATH_RE % '', re.I)
+_WS = re.compile("\s+", re.U)
-def text(txt):
- stripped = txt.strip() if txt else None
- if stripped:
- return stripped
+# tags to keep (only for attributes with markup)
+_TAGS_TO_KEEP = frozenset(['br', 'p', 'big', 'em', 'small', 'strong', 'sub',
+ 'sup', 'ins', 'del', 'code', 'kbd', 'samp', 'tt', 'var', 'pre', 'listing',
+ 'plaintext', 'abbr', 'acronym', 'address', 'bdo', 'blockquote', 'q',
+ 'cite', 'dfn', 'table', 'tr', 'th', 'td', 'tbody', 'ul', 'ol', 'li', 'dl',
+ 'dd', 'dt'])
+
+# tag names to be replaced by other tag names (overrides tags_to_keep)
+_TAGS_TO_REPLACE = {
+ 'h1': 'strong',
+ 'h2': 'strong',
+ 'h3': 'strong',
+ 'h4': 'strong',
+ 'h5': 'strong',
+ 'h6': 'strong',
+ 'b' : 'strong',
+ 'i' : 'em',
+}
+
+# tags whoose content will be completely removed (recursively)
+# (overrides tags_to_keep and tags_to_replace)
+_TAGS_TO_PURGE = ('script', 'img', 'input')
+
+def htmlregion(text):
+ """convenience function to make an html region from text.
+ This is useful for testing
+ """
+ from scrapy.contrib.ibl.htmlpage import HtmlPage
+ return HtmlPage(body=text).subregion()
+
+def notags(region, tag_replace=u' '):
+ """Removes all html tags"""
+ fragments = getattr(region, 'parsed_fragments', None)
+ if fragments is None:
+ return region
+ page = region.htmlpage
+ data = [page.fragment_data(f) for f in fragments if not isinstance(f, HtmlTag)]
+ return tag_replace.join(data)
+
+def text(region):
+ """Converts HTML to text. There is no attempt at formatting other than
+ removing excessive whitespace,
+
+ For example:
+ >>> t = lambda s: text(htmlregion(s))
+ >>> t(u'test
')
+ u'test'
+
+ Leading and trailing whitespace are removed
+ >>> t(u' test
')
+ u'test'
+
+ Comments are removed
+ >>> t(u'test me')
+ u'test me'
+
+ Text between script tags is ignored
+ >>> t(u"scripts are ignored")
+ u'scripts are ignored'
+
+ HTML entities are converted to text
+ >>> t(u"only £42")
+ u'only \\xa342'
+ """
+ chunks = _process_markup(region,
+ lambda text: remove_entities(text, encoding=region.htmlpage.encoding),
+ lambda tag: u' '
+ )
+ text = u''.join(chunks)
+ return _WS.sub(u' ', text).strip()
+
+def safehtml(region, allowed_tags=_TAGS_TO_KEEP, replace_tags=_TAGS_TO_REPLACE):
+ """Creates an HTML subset, using a whitelist of HTML tags.
+
+ The HTML generated is safe for display on a website,without escaping and
+ should not cause formatting problems.
+
+ Allowed_tags is a set of tags that are allowed and replace_tags is a mapping of
+ tags to alternative tags to substitute.
+
+ For example:
+ >>> t = lambda s: safehtml(htmlregion(s))
+ >>> t(u'test ')
+ u'test test'
+
+ Some tags, like script, are completely removed
+ >>> t(u'test')
+ u'test'
+
+ replace_tags define tags that are converted. By default all headers, bold and indenting
+ are converted to strong and em.
+ >>> t(u'header
test bold indent')
+ u'header test bold indent'
+
+ Comments are stripped, but entities are not converted
+ >>> t(u' only £42')
+ u'only £42'
+
+ Paired tags are closed
+ >>> t(u'test')
+ u'
test
'
+
+ >>> t(u'test
test
')
+ u'test
test
'
+
+ """
+ tagstack = []
+ def _process_tag(tag):
+ tagstr = replace_tags.get(tag.tag, tag.tag)
+ if tagstr not in allowed_tags:
+ return
+ if tag.tag_type == HtmlTagType.OPEN_TAG:
+ tagstack.append(tagstr)
+ return u"<%s>" % tagstr
+ elif tag.tag_type == HtmlTagType.CLOSE_TAG:
+ try:
+ last = tagstack.pop()
+ # common case of matching tag
+ if last == tagstr:
+ return u"" % last
+ # output all preceeding tags (if present)
+ revtags = tagstack[::-1]
+ tindex = revtags.index(tagstr)
+ del tagstack[-tindex-1:]
+ return u"" % (last, u">" % tag.tag
+ chunks = list(_process_markup(region, lambda text: text, _process_tag)) + \
+ ["" % t for t in reversed(tagstack)]
+ return u''.join(chunks).strip()
+
+def _process_markup(region, textf, tagf):
+ fragments = getattr(region, 'parsed_fragments', None)
+ if fragments is None:
+ textf(region)
+ return
+ fiter = iter(fragments)
+ for fragment in fiter:
+ if isinstance(fragment, HtmlTag):
+ # skip forward to closing script tags
+ tag = fragment.tag
+ if tag in _TAGS_TO_PURGE:
+ # if opening, keep going until closed
+ if fragment.tag_type == HtmlTagType.OPEN_TAG:
+ for probe in fiter:
+ if isinstance(probe, HtmlTag) and \
+ probe.tag == tag and \
+ probe.tag_type == HtmlTagType.CLOSE_TAG:
+ break
+ else:
+ output = tagf(fragment)
+ if output:
+ yield output
+ else:
+ text = region.htmlpage.fragment_data(fragment)
+ text = remove_comments(text)
+ text = textf(text)
+ if text:
+ yield text
+
+def html(pageregion):
+ """A page region is already html, so this is the identity function"""
+ return pageregion
def contains_any_numbers(txt):
"""text that must contain at least one number
diff --git a/scrapy/contrib/ibl/htmlpage.py b/scrapy/contrib/ibl/htmlpage.py
index c86ec9670..e16ac2c7a 100644
--- a/scrapy/contrib/ibl/htmlpage.py
+++ b/scrapy/contrib/ibl/htmlpage.py
@@ -1,8 +1,9 @@
"""
htmlpage
-Container object for representing html pages in the IBL system. This
-encapsulates page related information and prevents parsing multiple times.
+Container objects for representing html pages and their parts in the IBL
+system. This encapsulates page related information and prevents parsing
+multiple times.
"""
import re
import hashlib
@@ -26,26 +27,82 @@ def create_page_from_jsonpage(jsonpage, body_key):
return HtmlPage(url, headers, body, page_id)
class HtmlPage(object):
- def __init__(self, url=None, headers=None, body=None, page_id=None):
+ """HtmlPage
+
+ This is a parsed HTML page. It contains the page headers, url, raw body and parsed
+ body.
+
+ The parsed body is a list of HtmlDataFragment objects.
+ """
+ def __init__(self, url=None, headers=None, body=None, page_id=None, encoding='utf-8'):
assert isinstance(body, unicode), "unicode expected, got: %s" % type(body).__name__
self.headers = headers or {}
self.body = body
self.url = url or u''
+ self.encoding = encoding
if page_id is None and url:
self.page_id = hashlib.sha1(url).hexdigest()
else:
self.page_id = page_id
+ @classmethod
+ def from_response(cls, response, page_id=None):
+ """Create an HtmlPage from a scrapy response"""
+ return HtmlPage(response.url, response.headers,
+ response.body_as_unicode(), page_id, response.encoding)
def _set_body(self, body):
self._body = body
self.parsed_body = list(parse_html(body))
- body = property(lambda x: x._body, _set_body)
+ body = property(lambda x: x._body, _set_body, doc="raw html for the page")
+ def subregion(self, start=0, end=None):
+ """HtmlPageRegion constructed from the start and end index (inclusive)
+ into the parsed page
+ """
+ return HtmlPageParsedRegion(self, start, end)
+
def fragment_data(self, data_fragment):
+ """portion of the body corresponding to the HtmlDataFragment"""
return self.body[data_fragment.start:data_fragment.end]
+
+class HtmlPageRegion(unicode):
+ """A Region of an HtmlPage that has been extracted
+ """
+ def __new__(cls, htmlpage, data):
+ return unicode.__new__(cls, data)
+
+ def __init__(self, htmlpage, data):
+ """Construct a new HtmlPageRegion object.
+
+ htmlpage is the original page and data is the raw html
+ """
+ self.htmlpage = htmlpage
+class HtmlPageParsedRegion(HtmlPageRegion):
+ """A region of an HtmlPage that has been extracted
+
+ This has a parsed_fragments property that contains the parsed html
+ fragments contained within this region
+ """
+ def __new__(cls, htmlpage, start_index, end_index):
+ text_start = htmlpage.parsed_body[start_index].start
+ text_end = htmlpage.parsed_body[end_index or -1].end
+ text = htmlpage.body[text_start:text_end]
+ return HtmlPageRegion.__new__(cls, htmlpage, text)
+
+ def __init__(self, htmlpage, start_index, end_index):
+ self.htmlpage = htmlpage
+ self.start_index = start_index
+ self.end_index = end_index
+
+ @property
+ def parsed_fragments(self):
+ """HtmlDataFragment or HtmlTag objects for this parsed region"""
+ end = self.end_index + 1 if self.end_index is not None else None
+ return self.htmlpage.parsed_body[self.start_index:end]
+
class HtmlTagType(object):
OPEN_TAG = 1
CLOSE_TAG = 2
diff --git a/scrapy/tests/test_contrib_ibl/test_extraction.py b/scrapy/tests/test_contrib_ibl/test_extraction.py
index e9ce86f7e..3a7bdd393 100644
--- a/scrapy/tests/test_contrib_ibl/test_extraction.py
+++ b/scrapy/tests/test_contrib_ibl/test_extraction.py
@@ -9,7 +9,7 @@ from scrapy.contrib.ibl.htmlpage import HtmlPage
from scrapy.contrib.ibl.descriptor import (FieldDescriptor as A,
ItemDescriptor)
from scrapy.contrib.ibl.extractors import (contains_any_numbers,
- image_url)
+ image_url, html, notags)
try:
import numpy
@@ -719,19 +719,23 @@ EXTRACT_PAGE23 = u"""