mirror of https://github.com/scrapy/scrapy.git
refactor IBL extraction to allow processing parsed data
--HG-- extra : rebase_source : 1a0ec4322702288f6e996d384d45a36deede3868
This commit is contained in:
Shane Evans
parent
9591413d9d
commit
bc6eee71e3
|
|
@ -3,25 +3,22 @@ Extended types for IBL extraction
|
|||
"""
|
||||
from itertools import chain
|
||||
|
||||
from scrapy.contrib.ibl.extractors import text
|
||||
from scrapy.contrib.ibl.extractors import text, html
|
||||
|
||||
class FieldDescriptor(object):
|
||||
"""description of a scraped attribute"""
|
||||
__slots__ = ('name', 'description', 'extractor', 'required', 'allow_markup')
|
||||
__slots__ = ('name', 'description', 'extractor', 'required')
|
||||
|
||||
def __init__(self, name, description, extractor=text, required=False,
|
||||
allow_markup=False):
|
||||
def __init__(self, name, description, extractor=text, required=False):
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.extractor = extractor
|
||||
self.required = required
|
||||
self.allow_markup = allow_markup
|
||||
|
||||
@classmethod
|
||||
def from_field(cls, name, field):
|
||||
return cls(name, field.get('description'), \
|
||||
field.get('ibl_extractor', text), field.get('required', False), \
|
||||
field.get('allow_markup', False))
|
||||
field.get('ibl_extractor', text), field.get('required', False))
|
||||
|
||||
def __str__(self):
|
||||
return "FieldDescriptor(%s)" % self.name
|
||||
|
|
|
|||
|
|
@ -4,16 +4,14 @@ Page objects
|
|||
This module contains objects representing pages and parts of pages (e.g. tokens
|
||||
and annotations) used in the instance based learning algorithm.
|
||||
"""
|
||||
from itertools import chain
|
||||
from numpy import array, ndarray
|
||||
|
||||
from scrapy.contrib.ibl.htmlpage import HtmlTagType
|
||||
from scrapy.contrib.ibl.htmlpage import HtmlTagType, HtmlPageRegion
|
||||
|
||||
class TokenType(object):
|
||||
class TokenType(HtmlTagType):
|
||||
"""constants for token types"""
|
||||
WORD = 0
|
||||
OPEN_TAG = HtmlTagType.OPEN_TAG
|
||||
CLOSE_TAG = HtmlTagType.CLOSE_TAG
|
||||
NON_PAIRED_TAG = HtmlTagType.UNPAIRED_TAG
|
||||
|
||||
class TokenDict(object):
|
||||
"""Mapping from parse tokens to integers
|
||||
|
|
@ -68,6 +66,36 @@ class TokenDict(object):
|
|||
templates = ["%s", "<%s>", "</%s>", "<%s/>"]
|
||||
return templates[tid >> 24] % self.find_token(tid)
|
||||
|
||||
class PageRegion(object):
|
||||
"""A region in a page, defined by a start and end index"""
|
||||
|
||||
__slots__ = ('start_index', 'end_index')
|
||||
|
||||
def __init__(self, start, end):
|
||||
self.start_index = start
|
||||
self.end_index = end
|
||||
|
||||
def __str__(self):
|
||||
return "%s(%s, %s)" % (self.__class__.__name__, self.start_index,
|
||||
self.end_index)
|
||||
|
||||
def __repr__(self):
|
||||
return str(self)
|
||||
|
||||
class FragmentedHtmlPageRegion(HtmlPageRegion):
|
||||
"""An HtmlPageRegion consisting of possibly non-contiguous sub-regions"""
|
||||
def __new__(cls, htmlpage, regions):
|
||||
text = u''.join(regions)
|
||||
return HtmlPageRegion.__new__(cls, htmlpage, text)
|
||||
|
||||
def __init__(self, htmlpage, regions):
|
||||
self.htmlpage = htmlpage
|
||||
self.regions = regions
|
||||
|
||||
@property
|
||||
def parsed_fragments(self):
|
||||
return chain(*(r.parsed_fragments for r in self.regions))
|
||||
|
||||
class Page(object):
|
||||
"""Basic representation of a page. This consists of a reference to a
|
||||
dictionary of tokens and an array of raw token ids
|
||||
|
|
@ -92,7 +120,8 @@ class TemplatePage(Page):
|
|||
annotations = sorted(annotations, key=lambda x: x.end_index, reverse=True)
|
||||
self.annotations = sorted(annotations, key=lambda x: x.start_index)
|
||||
self.id = template_id
|
||||
self.ignored_regions = ignored_regions or []
|
||||
self.ignored_regions = [i if isinstance(i, PageRegion) else PageRegion(*i) \
|
||||
for i in (ignored_regions or [])]
|
||||
self.extra_required_attrs = set(extra_required or [])
|
||||
|
||||
def __str__(self):
|
||||
|
|
@ -107,66 +136,53 @@ class ExtractionPage(Page):
|
|||
"""Parsed data belonging to a web page upon which we wish to perform
|
||||
extraction.
|
||||
"""
|
||||
__slots__ = ('text',
|
||||
'token_start_indexes', # index in text of the start of a token
|
||||
'token_follow_indexes', # index in text of data following token
|
||||
'tag_attributes' # a map from token index to tag attributes
|
||||
)
|
||||
__slots__ = ('htmlpage', 'token_page_indexes')
|
||||
|
||||
def __init__(self, text, token_dict, page_tokens, token_start_indexes,
|
||||
token_follow_indexes, tag_attributes):
|
||||
def __init__(self, htmlpage, token_dict, page_tokens, token_page_indexes):
|
||||
"""Construct a new ExtractionPage
|
||||
|
||||
Arguments:
|
||||
`htmlpage`: The source HtmlPage
|
||||
`token_dict`: Token Dictionary used for tokenization
|
||||
`page_tokens': array of page tokens for matching
|
||||
`token_page_indexes`: indexes of each token in the parsed htmlpage
|
||||
"""
|
||||
Page.__init__(self, token_dict, page_tokens)
|
||||
self.text = text
|
||||
self.token_start_indexes = token_start_indexes
|
||||
self.token_follow_indexes = token_follow_indexes
|
||||
self.tag_attributes = tag_attributes
|
||||
|
||||
def token_html(self, token_index):
|
||||
"""The raw html for a page token at the given index in the page_tokens
|
||||
list
|
||||
"""
|
||||
text_start = self.token_start_indexes[token_index]
|
||||
text_end = self.token_follow_indexes[token_index]
|
||||
return self.text[text_start:text_end]
|
||||
|
||||
def html_between_tokens(self, start_token_index, end_token_index):
|
||||
"""The raw html between the tokens at the specified indexes in the
|
||||
page_tokens list
|
||||
|
||||
This assumes start_token_index <= end_token_index
|
||||
"""
|
||||
text_start = self.token_follow_indexes[start_token_index]
|
||||
text_end = self.token_start_indexes[end_token_index]
|
||||
return self.text[text_start:text_end]
|
||||
self.htmlpage = htmlpage
|
||||
self.token_page_indexes = token_page_indexes
|
||||
|
||||
def text_between_tokens(self, start_token_index, end_token_index,
|
||||
tag_replacement=u' '):
|
||||
"""The text between the the tokens at the specified indexes in the
|
||||
page_tokens list. Tags are replaced by tag_replacement (default one space
|
||||
character)
|
||||
"""
|
||||
return tag_replacement.join([self.text[
|
||||
self.token_follow_indexes[i]:self.token_start_indexes[i+1]] \
|
||||
for i in xrange(start_token_index, end_token_index)])
|
||||
def htmlpage_region(self, start_token_index, end_token_index):
|
||||
"""The region in the HtmlPage corresonding to the area defined by
|
||||
the start_token_index and the end_token_index
|
||||
|
||||
def tag_attribute(self, token_index, attribute):
|
||||
"""The value of a tag attribute. The tag is identified by its
|
||||
corresponding token index.
|
||||
|
||||
If the tag or attribute is not present, None is returned
|
||||
This includes the tokens at the specified indexes
|
||||
"""
|
||||
return self.tag_attributes.get(token_index, {}).get(attribute)
|
||||
start = self.token_page_indexes[start_token_index]
|
||||
end = self.token_page_indexes[end_token_index]
|
||||
return self.htmlpage.subregion(start, end)
|
||||
|
||||
def htmlpage_region_inside(self, start_token_index, end_token_index):
|
||||
"""The region in the HtmlPage corresonding to the area between
|
||||
the start_token_index and the end_token_index.
|
||||
|
||||
This excludes the tokens at the specified indexes
|
||||
"""
|
||||
start = self.token_page_indexes[start_token_index] + 1
|
||||
end = self.token_page_indexes[end_token_index] - 1
|
||||
return self.htmlpage.subregion(start, end)
|
||||
|
||||
def htmlpage_tag(self, token_index):
|
||||
"""The HtmlPage tag at corresponding to the token at token_index"""
|
||||
return self.htmlpage.parsed_body[self.token_page_indexes[token_index]]
|
||||
|
||||
def __str__(self):
|
||||
summary = []
|
||||
for (token, start, follow) in zip(self.page_tokens, self.token_start_indexes,
|
||||
self.token_follow_indexes):
|
||||
text = "%s %s-%s (%s)" % (self.token_dict.find_token(token), start, follow,
|
||||
self.text[start:follow])
|
||||
for token, tindex in zip(self.page_tokens, self.token_page_indexes):
|
||||
text = "%s page[%s]: %s" % (self.token_dict.find_token(token),
|
||||
tindex, self.htmlpage.parsed_body[tindex])
|
||||
summary.append(text)
|
||||
return "ExtractionPage\n==============\nTokens: %s\n\nRaw text: %s\n\n" \
|
||||
"Tag attributes: %s\n" % ('\n'.join(summary), self.text,
|
||||
self.tag_attributes)
|
||||
% ('\n'.join(summary), self.htmlpage.body)
|
||||
|
||||
class AnnotationText(object):
|
||||
__slots__ = ('start_text', 'follow_text')
|
||||
|
|
@ -179,7 +195,8 @@ class AnnotationText(object):
|
|||
return "AnnotationText(%s..%s)" % \
|
||||
(repr(self.start_text), repr(self.follow_text))
|
||||
|
||||
class AnnotationTag(object):
|
||||
|
||||
class AnnotationTag(PageRegion):
|
||||
"""A tag that annotates part of the document
|
||||
|
||||
It has the following properties:
|
||||
|
|
@ -197,8 +214,7 @@ class AnnotationTag(object):
|
|||
|
||||
def __init__(self, start_index, end_index, surrounds_attribute=None,
|
||||
annotation_text=None, tag_attributes=None, variant_id=None):
|
||||
self.start_index = start_index
|
||||
self.end_index = end_index
|
||||
PageRegion.__init__(self, start_index, end_index)
|
||||
self.surrounds_attribute = surrounds_attribute
|
||||
self.annotation_text = annotation_text
|
||||
self.tag_attributes = tag_attributes or []
|
||||
|
|
@ -213,16 +229,3 @@ class AnnotationTag(object):
|
|||
def __repr__(self):
|
||||
return str(self)
|
||||
|
||||
class LabelledRegion(object):
|
||||
__slots__ = ('start_index', 'end_index')
|
||||
|
||||
def __init__(self, start, end):
|
||||
self.start_index = start
|
||||
self.end_index = end
|
||||
|
||||
def __str__(self):
|
||||
return "LabelledRegion (%s, %s)" % (self.start_index, self.end_index)
|
||||
|
||||
def __repr__(self):
|
||||
return str(self)
|
||||
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ from numpy import array
|
|||
from scrapy.utils.py26 import json
|
||||
|
||||
from scrapy.contrib.ibl.htmlpage import HtmlTagType, HtmlTag, HtmlPage
|
||||
from scrapy.contrib.ibl.extraction.pageobjects import (AnnotationTag,
|
||||
TemplatePage, ExtractionPage, AnnotationText, TokenDict)
|
||||
from scrapy.contrib.ibl.extraction.pageobjects import (AnnotationTag,
|
||||
TemplatePage, ExtractionPage, AnnotationText, TokenDict, FragmentedHtmlPageRegion)
|
||||
|
||||
def parse_strings(template_html, extraction_html):
|
||||
"""Create a template and extraction page from raw strings
|
||||
|
|
@ -52,18 +52,18 @@ class InstanceLearningParser(object):
|
|||
def feed(self, html_page):
|
||||
self.html_page = html_page
|
||||
self.previous_element_class = None
|
||||
for data in html_page.parsed_body:
|
||||
for index, data in enumerate(html_page.parsed_body):
|
||||
if isinstance(data, HtmlTag):
|
||||
self._add_token(data.tag, data.tag_type, data.start, data.end)
|
||||
self.handle_tag(data)
|
||||
self.handle_tag(data, index)
|
||||
else:
|
||||
self.handle_data(data)
|
||||
self.handle_data(data, index)
|
||||
self.previous_element_class = data.__class__
|
||||
|
||||
def handle_data(self, html_data_fragment):
|
||||
def handle_data(self, html_data_fragment, index):
|
||||
pass
|
||||
|
||||
def handle_tag(self, html_tag):
|
||||
def handle_tag(self, html_tag, index):
|
||||
pass
|
||||
|
||||
_END_UNPAIREDTAG_TAGS = ["form", "div", "p", "table", "tr", "td"]
|
||||
|
|
@ -86,7 +86,7 @@ class TemplatePageParser(InstanceLearningParser):
|
|||
self.last_text_region = None
|
||||
self.next_tag_index = 0
|
||||
|
||||
def handle_tag(self, html_tag):
|
||||
def handle_tag(self, html_tag, index):
|
||||
if self.last_text_region:
|
||||
self._process_text('')
|
||||
|
||||
|
|
@ -275,7 +275,7 @@ class TemplatePageParser(InstanceLearningParser):
|
|||
if prev != annotation.variant_id:
|
||||
raise ValueError("unbalanced variant annotation tags")
|
||||
|
||||
def handle_data(self, html_data_fragment):
|
||||
def handle_data(self, html_data_fragment, index):
|
||||
fragment_text = self.html_page.fragment_data(html_data_fragment)
|
||||
self._process_text(fragment_text)
|
||||
|
||||
|
|
@ -300,20 +300,11 @@ class ExtractionPageParser(InstanceLearningParser):
|
|||
"""
|
||||
def __init__(self, token_dict):
|
||||
InstanceLearningParser.__init__(self, token_dict)
|
||||
self.page_data = []
|
||||
self.token_start_index = []
|
||||
self.token_follow_index = []
|
||||
self.tag_attrs = {}
|
||||
self._page_token_indexes = []
|
||||
|
||||
def _add_token(self, token, token_type, start, end):
|
||||
InstanceLearningParser._add_token(self, token, token_type, start, end)
|
||||
self.token_start_index.append(start)
|
||||
self.token_follow_index.append(end)
|
||||
|
||||
def handle_tag(self, html_tag):
|
||||
if html_tag.attributes:
|
||||
self.tag_attrs[len(self.token_list) - 1] = html_tag.attributes
|
||||
def handle_tag(self, html_tag, index):
|
||||
self._page_token_indexes.append(index)
|
||||
|
||||
def to_extraction_page(self):
|
||||
return ExtractionPage(self.html_page.body, self.token_dict, array(self.token_list),
|
||||
self.token_start_index, self.token_follow_index, self.tag_attrs)
|
||||
return ExtractionPage(self.html_page, self.token_dict, array(self.token_list),
|
||||
self._page_token_indexes)
|
||||
|
|
|
|||
|
|
@ -8,14 +8,16 @@ import operator
|
|||
import copy
|
||||
import pprint
|
||||
import cStringIO
|
||||
from itertools import groupby
|
||||
from itertools import groupby, izip, starmap
|
||||
|
||||
from numpy import array
|
||||
|
||||
from scrapy.contrib.ibl.descriptor import FieldDescriptor
|
||||
from scrapy.contrib.ibl.htmlpage import HtmlPageRegion
|
||||
from scrapy.contrib.ibl.extraction.similarity import (similar_region,
|
||||
longest_unique_subsequence, common_prefix)
|
||||
from scrapy.contrib.ibl.extraction.pageobjects import AnnotationTag, LabelledRegion
|
||||
from scrapy.contrib.ibl.extraction.pageobjects import (AnnotationTag,
|
||||
PageRegion, FragmentedHtmlPageRegion)
|
||||
|
||||
def build_extraction_tree(template, type_descriptor, trace=True):
|
||||
"""Build a tree of region extractors corresponding to the
|
||||
|
|
@ -33,16 +35,14 @@ def build_extraction_tree(template, type_descriptor, trace=True):
|
|||
|
||||
return TemplatePageExtractor(template, extractors)
|
||||
|
||||
_ID = lambda x: x
|
||||
_EXTRACT_HTML = lambda x: x
|
||||
_DEFAULT_DESCRIPTOR = FieldDescriptor('none', None)
|
||||
|
||||
def _labelled(obj):
|
||||
"""
|
||||
Returns labelled element of the object (extractor or labelled region)
|
||||
"""
|
||||
if hasattr(obj, "annotation"):
|
||||
return obj.annotation
|
||||
return obj
|
||||
return getattr(obj, 'annotation', obj)
|
||||
|
||||
def _compose(f, g):
|
||||
"""given unary functions f and g, return a function that computes f(g(x))
|
||||
|
|
@ -75,7 +75,7 @@ class BasicTypeExtractor(object):
|
|||
u'<div data-scrapy-annotate="{"annotations": {"content": "name"}}">x<b> xx</b></div>',\
|
||||
u'<div>a name<b> id-9</b></div>')
|
||||
>>> ex = BasicTypeExtractor(template.annotations[0])
|
||||
>>> ex.extract(page, 0, 3, [LabelledRegion(1, 2)])
|
||||
>>> ex.extract(page, 0, 3, [PageRegion(1, 2)])
|
||||
[(u'name', u'a name')]
|
||||
"""
|
||||
|
||||
|
|
@ -88,17 +88,15 @@ class BasicTypeExtractor(object):
|
|||
descriptor = attribute_descriptors.get(annotation.surrounds_attribute)
|
||||
if descriptor:
|
||||
self.content_validate = descriptor.extractor
|
||||
self.allow_markup = descriptor.allow_markup
|
||||
else:
|
||||
self.content_validate = _ID
|
||||
self.allow_markup = False
|
||||
self.content_validate = _EXTRACT_HTML
|
||||
self.extract = self._extract_content
|
||||
|
||||
if annotation.tag_attributes:
|
||||
self.tag_data = []
|
||||
for (tag_attr, extraction_attr) in annotation.tag_attributes:
|
||||
descriptor = attribute_descriptors.get(extraction_attr)
|
||||
extractf = descriptor.extractor if descriptor else _ID
|
||||
extractf = descriptor.extractor if descriptor else _EXTRACT_HTML
|
||||
self.tag_data.append((extractf, tag_attr, extraction_attr))
|
||||
|
||||
self.extract = self._extract_both if \
|
||||
|
|
@ -109,33 +107,32 @@ class BasicTypeExtractor(object):
|
|||
self._extract_attribute(page, start_index, end_index, ignored_regions)
|
||||
|
||||
def _extract_content(self, extraction_page, start_index, end_index, ignored_regions=None):
|
||||
# we might want to add opening/closing ul/ol/table if we have the
|
||||
# middle of a region. This would require support in the scrapy
|
||||
# cleansing.
|
||||
complete_data = ""
|
||||
start = start_index
|
||||
end = ignored_regions[0].start_index if ignored_regions else end_index
|
||||
while start is not None:
|
||||
if self.allow_markup:
|
||||
data = extraction_page.html_between_tokens(start, end)
|
||||
else:
|
||||
data = extraction_page.text_between_tokens(start, end)
|
||||
complete_data += data
|
||||
if ignored_regions:
|
||||
start = ignored_regions[0].end_index
|
||||
ignored_regions.pop(0)
|
||||
end = ignored_regions[0].start_index if ignored_regions else end_index
|
||||
else:
|
||||
start = None
|
||||
complete_data = self.content_validate(complete_data)
|
||||
return [(self.annotation.surrounds_attribute, complete_data)] if complete_data else []
|
||||
# extract content between annotation indexes
|
||||
if not ignored_regions:
|
||||
region = extraction_page.htmlpage_region_inside(start_index, end_index)
|
||||
else:
|
||||
# assumes ignored_regions are completely contained within start and end index
|
||||
assert (start_index <= ignored_regions[0].start_index and
|
||||
end_index >= ignored_regions[-1].end_index)
|
||||
starts = [start_index] + [i.end_index for i in ignored_regions]
|
||||
ends = [i.start_index for i in ignored_regions]
|
||||
if starts[-1] is not None:
|
||||
ends.append(end_index)
|
||||
included_regions = izip(starts, ends)
|
||||
if ends[0] is None:
|
||||
included_regions.next()
|
||||
regions = starmap(extraction_page.htmlpage_region_inside, included_regions)
|
||||
region = FragmentedHtmlPageRegion(extraction_page.htmlpage, list(regions))
|
||||
validated = self.content_validate(region)
|
||||
return [(self.annotation.surrounds_attribute, validated)] if validated else []
|
||||
|
||||
def _extract_attribute(self, extraction_page, start_index, end_index, ignored_regions=None):
|
||||
data = []
|
||||
for (f, ta, ea) in self.tag_data:
|
||||
tag_value = extraction_page.tag_attribute(start_index, ta)
|
||||
tag_value = extraction_page.htmlpage_tag(start_index).attributes.get(ta)
|
||||
if tag_value:
|
||||
extracted = f(tag_value)
|
||||
region = HtmlPageRegion(extraction_page.htmlpage, tag_value)
|
||||
extracted = f(region)
|
||||
if extracted is not None:
|
||||
data.append((ea, extracted))
|
||||
return data
|
||||
|
|
@ -178,10 +175,8 @@ class BasicTypeExtractor(object):
|
|||
def __str__(self):
|
||||
messages = ['BasicTypeExtractor(']
|
||||
if self.annotation.surrounds_attribute:
|
||||
messages += [self.annotation.surrounds_attribute, ': ',
|
||||
'html content' if self.allow_markup else 'text content',
|
||||
]
|
||||
if self.content_validate != _ID:
|
||||
messages.append(self.annotation.surrounds_attribute)
|
||||
if self.content_validate != _EXTRACT_HTML:
|
||||
messages += [', extracted with \'',
|
||||
self.content_validate.__name__, '\'']
|
||||
|
||||
|
|
@ -190,7 +185,7 @@ class BasicTypeExtractor(object):
|
|||
messages.append(';')
|
||||
for (f, ta, ea) in self.tag_data:
|
||||
messages += [ea, ': tag attribute "', ta, '"']
|
||||
if f != _ID:
|
||||
if f != _EXTRACT_HTML:
|
||||
messages += [', validated by ', str(f)]
|
||||
messages.append(", template[%s:%s])" % \
|
||||
(self.annotation.start_index, self.annotation.end_index))
|
||||
|
|
@ -340,7 +335,8 @@ class RecordExtractor(object):
|
|||
The region in the page to be extracted from may be specified using
|
||||
start_index and end_index
|
||||
"""
|
||||
ignored_regions = [i if isinstance(i, LabelledRegion) else LabelledRegion(*i) for i in (ignored_regions or [])]
|
||||
if ignored_regions is None:
|
||||
ignored_regions = []
|
||||
region_elements = sorted(self.extractors + ignored_regions, key=lambda x: _labelled(x).start_index)
|
||||
_, _, attributes = self._doextract(page, region_elements, start_index,
|
||||
end_index)
|
||||
|
|
@ -395,7 +391,7 @@ class RecordExtractor(object):
|
|||
s, p, e = similar_region(page.page_tokens, self.template_tokens, \
|
||||
i, start, sindex)
|
||||
if s > 0:
|
||||
similar_ignored_regions.append(LabelledRegion(p, e))
|
||||
similar_ignored_regions.append(PageRegion(p, e))
|
||||
start = e or start
|
||||
extracted_data = first_region.extract(page, pindex, sindex, similar_ignored_regions)
|
||||
if extracted_data:
|
||||
|
|
@ -493,12 +489,12 @@ class TraceExtractor(object):
|
|||
for t in template.page_tokens[tend:tend+5]])
|
||||
|
||||
def summarize_trace(self, page, start, end, ret):
|
||||
text_start = page.token_follow_indexes[start]
|
||||
text_end = page.token_start_indexes[end or -1]
|
||||
text_start = page.htmlpage.parsed_body[page.token_page_indexes[start]].start
|
||||
text_end = page.htmlpage.parsed_body[page.token_page_indexes[end or -1]].end
|
||||
page_snippet = "(...%s)%s(%s...)" % (
|
||||
page.text[text_start-50:text_start].replace('\n', ' '),
|
||||
page.text[text_start:text_end],
|
||||
page.text[text_end:text_end+50].replace('\n', ' '))
|
||||
page.htmlpage.body[text_start-50:text_start].replace('\n', ' '),
|
||||
page.htmlpage.body[text_start:text_end],
|
||||
page.htmlpage.body[text_end:text_end+50].replace('\n', ' '))
|
||||
pre_summary = "\nstart %s page[%s:%s]\n" % (self.traced.__class__.__name__, start, end)
|
||||
post_summary = """
|
||||
%s page[%s:%s]
|
||||
|
|
@ -572,25 +568,26 @@ class TemplatePageExtractor(object):
|
|||
_tokenize = re.compile(r'\w+|[^\w\s]+', re.UNICODE | re.MULTILINE | re.DOTALL).findall
|
||||
|
||||
class TextRegionDataExtractor(object):
|
||||
"""Data Extractor for extracting text fragments from within a larger
|
||||
body of text. It extracts based on the longest unique prefix and suffix.
|
||||
"""Data Extractor for extracting text fragments from an annotation page
|
||||
fragment or string. It extracts based on the longest unique prefix and
|
||||
suffix.
|
||||
|
||||
for example:
|
||||
>>> extractor = TextRegionDataExtractor('designed by ', '.')
|
||||
>>> extractor.extract("by Marc Newson.")
|
||||
>>> extractor.extract_text("by Marc Newson.")
|
||||
'Marc Newson'
|
||||
|
||||
Both prefix and suffix are optional:
|
||||
>>> extractor = TextRegionDataExtractor('designed by ')
|
||||
>>> extractor.extract("by Marc Newson.")
|
||||
>>> extractor.extract_text("by Marc Newson.")
|
||||
'Marc Newson.'
|
||||
>>> extractor = TextRegionDataExtractor(suffix='.')
|
||||
>>> extractor.extract("by Marc Newson.")
|
||||
>>> extractor.extract_text("by Marc Newson.")
|
||||
'by Marc Newson'
|
||||
|
||||
It requires a minimum match of at least one word or punctuation character:
|
||||
>>> extractor = TextRegionDataExtractor('designed by')
|
||||
>>> extractor.extract("y Marc Newson.") is None
|
||||
>>> extractor.extract_text("y Marc Newson.") is None
|
||||
True
|
||||
"""
|
||||
def __init__(self, prefix=None, suffix=None):
|
||||
|
|
@ -609,8 +606,13 @@ class TextRegionDataExtractor(object):
|
|||
tokens = _tokenize(matchstring or '')
|
||||
return len(tokens[0]) if tokens else 0
|
||||
|
||||
def extract(self, text):
|
||||
"""attempt to extract a substring from the text"""
|
||||
def extract(self, region):
|
||||
"""Extract a region from the region passed"""
|
||||
text = self.extract_text(region)
|
||||
return HtmlPageRegion(region.htmlpage, text) if text else None
|
||||
|
||||
def extract_text(self, text):
|
||||
"""Extract a substring from the text"""
|
||||
pref_index = 0
|
||||
if self.minprefix > 0:
|
||||
rev_idx, plen = longest_unique_subsequence(text[::-1], self.prefix)
|
||||
|
|
@ -623,5 +625,3 @@ class TextRegionDataExtractor(object):
|
|||
if slen < self.minsuffix:
|
||||
return None
|
||||
return text[pref_index:pref_index + sidx]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,8 +4,9 @@ Extractors for attributes
|
|||
import re
|
||||
import urlparse
|
||||
|
||||
from scrapy.utils.markup import remove_entities
|
||||
from scrapy.utils.markup import remove_entities, remove_comments
|
||||
from scrapy.utils.url import safe_url_string
|
||||
from scrapy.contrib.ibl.htmlpage import HtmlTag, HtmlTagType
|
||||
|
||||
#FIXME: the use of "." needs to be localized
|
||||
_NUMERIC_ENTITIES = re.compile("&#([0-9]+)(?:;|\s)", re.U)
|
||||
|
|
@ -22,11 +23,175 @@ _CSS_IMAGERE = re.compile("background(?:-image)?\s*:\s*url\((.*?)\)", re.I)
|
|||
_BASE_PATH_RE = "/?(?:[^/]+/)*(?:.+%s)"
|
||||
_IMAGE_PATH_RE = re.compile(_BASE_PATH_RE % '\.(?:%s)' % _IMAGES_TYPES, re.I)
|
||||
_GENERIC_PATH_RE = re.compile(_BASE_PATH_RE % '', re.I)
|
||||
_WS = re.compile("\s+", re.U)
|
||||
|
||||
def text(txt):
|
||||
stripped = txt.strip() if txt else None
|
||||
if stripped:
|
||||
return stripped
|
||||
# tags to keep (only for attributes with markup)
|
||||
_TAGS_TO_KEEP = frozenset(['br', 'p', 'big', 'em', 'small', 'strong', 'sub',
|
||||
'sup', 'ins', 'del', 'code', 'kbd', 'samp', 'tt', 'var', 'pre', 'listing',
|
||||
'plaintext', 'abbr', 'acronym', 'address', 'bdo', 'blockquote', 'q',
|
||||
'cite', 'dfn', 'table', 'tr', 'th', 'td', 'tbody', 'ul', 'ol', 'li', 'dl',
|
||||
'dd', 'dt'])
|
||||
|
||||
# tag names to be replaced by other tag names (overrides tags_to_keep)
|
||||
_TAGS_TO_REPLACE = {
|
||||
'h1': 'strong',
|
||||
'h2': 'strong',
|
||||
'h3': 'strong',
|
||||
'h4': 'strong',
|
||||
'h5': 'strong',
|
||||
'h6': 'strong',
|
||||
'b' : 'strong',
|
||||
'i' : 'em',
|
||||
}
|
||||
|
||||
# tags whoose content will be completely removed (recursively)
|
||||
# (overrides tags_to_keep and tags_to_replace)
|
||||
_TAGS_TO_PURGE = ('script', 'img', 'input')
|
||||
|
||||
def htmlregion(text):
|
||||
"""convenience function to make an html region from text.
|
||||
This is useful for testing
|
||||
"""
|
||||
from scrapy.contrib.ibl.htmlpage import HtmlPage
|
||||
return HtmlPage(body=text).subregion()
|
||||
|
||||
def notags(region, tag_replace=u' '):
|
||||
"""Removes all html tags"""
|
||||
fragments = getattr(region, 'parsed_fragments', None)
|
||||
if fragments is None:
|
||||
return region
|
||||
page = region.htmlpage
|
||||
data = [page.fragment_data(f) for f in fragments if not isinstance(f, HtmlTag)]
|
||||
return tag_replace.join(data)
|
||||
|
||||
def text(region):
|
||||
"""Converts HTML to text. There is no attempt at formatting other than
|
||||
removing excessive whitespace,
|
||||
|
||||
For example:
|
||||
>>> t = lambda s: text(htmlregion(s))
|
||||
>>> t(u'<h1>test</h1>')
|
||||
u'test'
|
||||
|
||||
Leading and trailing whitespace are removed
|
||||
>>> t(u'<h1> test</h1> ')
|
||||
u'test'
|
||||
|
||||
Comments are removed
|
||||
>>> t(u'test <!-- this is a comment --> me')
|
||||
u'test me'
|
||||
|
||||
Text between script tags is ignored
|
||||
>>> t(u"scripts are<script>n't</script> ignored")
|
||||
u'scripts are ignored'
|
||||
|
||||
HTML entities are converted to text
|
||||
>>> t(u"only £42")
|
||||
u'only \\xa342'
|
||||
"""
|
||||
chunks = _process_markup(region,
|
||||
lambda text: remove_entities(text, encoding=region.htmlpage.encoding),
|
||||
lambda tag: u' '
|
||||
)
|
||||
text = u''.join(chunks)
|
||||
return _WS.sub(u' ', text).strip()
|
||||
|
||||
def safehtml(region, allowed_tags=_TAGS_TO_KEEP, replace_tags=_TAGS_TO_REPLACE):
|
||||
"""Creates an HTML subset, using a whitelist of HTML tags.
|
||||
|
||||
The HTML generated is safe for display on a website,without escaping and
|
||||
should not cause formatting problems.
|
||||
|
||||
Allowed_tags is a set of tags that are allowed and replace_tags is a mapping of
|
||||
tags to alternative tags to substitute.
|
||||
|
||||
For example:
|
||||
>>> t = lambda s: safehtml(htmlregion(s))
|
||||
>>> t(u'<strong>test <blink>test</blink></strong>')
|
||||
u'<strong>test test</strong>'
|
||||
|
||||
Some tags, like script, are completely removed
|
||||
>>> t(u'<script>test </script>test')
|
||||
u'test'
|
||||
|
||||
replace_tags define tags that are converted. By default all headers, bold and indenting
|
||||
are converted to strong and em.
|
||||
>>> t(u'<h2>header</h2> test <b>bold</b> <i>indent</i>')
|
||||
u'<strong>header</strong> test <strong>bold</strong> <em>indent</em>'
|
||||
|
||||
Comments are stripped, but entities are not converted
|
||||
>>> t(u'<!-- comment --> only £42')
|
||||
u'only £42'
|
||||
|
||||
Paired tags are closed
|
||||
>>> t(u'<p>test')
|
||||
u'<p>test</p>'
|
||||
|
||||
>>> t(u'<p>test <i><br/><b>test</p>')
|
||||
u'<p>test <em><br/><strong>test</strong></em></p>'
|
||||
|
||||
"""
|
||||
tagstack = []
|
||||
def _process_tag(tag):
|
||||
tagstr = replace_tags.get(tag.tag, tag.tag)
|
||||
if tagstr not in allowed_tags:
|
||||
return
|
||||
if tag.tag_type == HtmlTagType.OPEN_TAG:
|
||||
tagstack.append(tagstr)
|
||||
return u"<%s>" % tagstr
|
||||
elif tag.tag_type == HtmlTagType.CLOSE_TAG:
|
||||
try:
|
||||
last = tagstack.pop()
|
||||
# common case of matching tag
|
||||
if last == tagstr:
|
||||
return u"</%s>" % last
|
||||
# output all preceeding tags (if present)
|
||||
revtags = tagstack[::-1]
|
||||
tindex = revtags.index(tagstr)
|
||||
del tagstack[-tindex-1:]
|
||||
return u"</%s></%s>" % (last, u"></".join(revtags[:tindex+1]))
|
||||
except (ValueError, IndexError):
|
||||
# popped from empty stack or failed to find the tag
|
||||
pass
|
||||
else:
|
||||
assert tag.tag_type == HtmlTagType.UNPAIRED_TAG, "unrecognised tag type"
|
||||
return u"<%s/>" % tag.tag
|
||||
chunks = list(_process_markup(region, lambda text: text, _process_tag)) + \
|
||||
["</%s>" % t for t in reversed(tagstack)]
|
||||
return u''.join(chunks).strip()
|
||||
|
||||
def _process_markup(region, textf, tagf):
|
||||
fragments = getattr(region, 'parsed_fragments', None)
|
||||
if fragments is None:
|
||||
textf(region)
|
||||
return
|
||||
fiter = iter(fragments)
|
||||
for fragment in fiter:
|
||||
if isinstance(fragment, HtmlTag):
|
||||
# skip forward to closing script tags
|
||||
tag = fragment.tag
|
||||
if tag in _TAGS_TO_PURGE:
|
||||
# if opening, keep going until closed
|
||||
if fragment.tag_type == HtmlTagType.OPEN_TAG:
|
||||
for probe in fiter:
|
||||
if isinstance(probe, HtmlTag) and \
|
||||
probe.tag == tag and \
|
||||
probe.tag_type == HtmlTagType.CLOSE_TAG:
|
||||
break
|
||||
else:
|
||||
output = tagf(fragment)
|
||||
if output:
|
||||
yield output
|
||||
else:
|
||||
text = region.htmlpage.fragment_data(fragment)
|
||||
text = remove_comments(text)
|
||||
text = textf(text)
|
||||
if text:
|
||||
yield text
|
||||
|
||||
def html(pageregion):
|
||||
"""A page region is already html, so this is the identity function"""
|
||||
return pageregion
|
||||
|
||||
def contains_any_numbers(txt):
|
||||
"""text that must contain at least one number
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
"""
|
||||
htmlpage
|
||||
|
||||
Container object for representing html pages in the IBL system. This
|
||||
encapsulates page related information and prevents parsing multiple times.
|
||||
Container objects for representing html pages and their parts in the IBL
|
||||
system. This encapsulates page related information and prevents parsing
|
||||
multiple times.
|
||||
"""
|
||||
import re
|
||||
import hashlib
|
||||
|
|
@ -26,26 +27,82 @@ def create_page_from_jsonpage(jsonpage, body_key):
|
|||
return HtmlPage(url, headers, body, page_id)
|
||||
|
||||
class HtmlPage(object):
|
||||
def __init__(self, url=None, headers=None, body=None, page_id=None):
|
||||
"""HtmlPage
|
||||
|
||||
This is a parsed HTML page. It contains the page headers, url, raw body and parsed
|
||||
body.
|
||||
|
||||
The parsed body is a list of HtmlDataFragment objects.
|
||||
"""
|
||||
def __init__(self, url=None, headers=None, body=None, page_id=None, encoding='utf-8'):
|
||||
assert isinstance(body, unicode), "unicode expected, got: %s" % type(body).__name__
|
||||
self.headers = headers or {}
|
||||
self.body = body
|
||||
self.url = url or u''
|
||||
self.encoding = encoding
|
||||
if page_id is None and url:
|
||||
self.page_id = hashlib.sha1(url).hexdigest()
|
||||
else:
|
||||
self.page_id = page_id
|
||||
|
||||
@classmethod
|
||||
def from_response(cls, response, page_id=None):
|
||||
"""Create an HtmlPage from a scrapy response"""
|
||||
return HtmlPage(response.url, response.headers,
|
||||
response.body_as_unicode(), page_id, response.encoding)
|
||||
|
||||
def _set_body(self, body):
|
||||
self._body = body
|
||||
self.parsed_body = list(parse_html(body))
|
||||
|
||||
body = property(lambda x: x._body, _set_body)
|
||||
body = property(lambda x: x._body, _set_body, doc="raw html for the page")
|
||||
|
||||
def subregion(self, start=0, end=None):
|
||||
"""HtmlPageRegion constructed from the start and end index (inclusive)
|
||||
into the parsed page
|
||||
"""
|
||||
return HtmlPageParsedRegion(self, start, end)
|
||||
|
||||
def fragment_data(self, data_fragment):
|
||||
"""portion of the body corresponding to the HtmlDataFragment"""
|
||||
return self.body[data_fragment.start:data_fragment.end]
|
||||
|
||||
class HtmlPageRegion(unicode):
|
||||
"""A Region of an HtmlPage that has been extracted
|
||||
"""
|
||||
def __new__(cls, htmlpage, data):
|
||||
return unicode.__new__(cls, data)
|
||||
|
||||
def __init__(self, htmlpage, data):
|
||||
"""Construct a new HtmlPageRegion object.
|
||||
|
||||
htmlpage is the original page and data is the raw html
|
||||
"""
|
||||
self.htmlpage = htmlpage
|
||||
|
||||
class HtmlPageParsedRegion(HtmlPageRegion):
|
||||
"""A region of an HtmlPage that has been extracted
|
||||
|
||||
This has a parsed_fragments property that contains the parsed html
|
||||
fragments contained within this region
|
||||
"""
|
||||
def __new__(cls, htmlpage, start_index, end_index):
|
||||
text_start = htmlpage.parsed_body[start_index].start
|
||||
text_end = htmlpage.parsed_body[end_index or -1].end
|
||||
text = htmlpage.body[text_start:text_end]
|
||||
return HtmlPageRegion.__new__(cls, htmlpage, text)
|
||||
|
||||
def __init__(self, htmlpage, start_index, end_index):
|
||||
self.htmlpage = htmlpage
|
||||
self.start_index = start_index
|
||||
self.end_index = end_index
|
||||
|
||||
@property
|
||||
def parsed_fragments(self):
|
||||
"""HtmlDataFragment or HtmlTag objects for this parsed region"""
|
||||
end = self.end_index + 1 if self.end_index is not None else None
|
||||
return self.htmlpage.parsed_body[self.start_index:end]
|
||||
|
||||
class HtmlTagType(object):
|
||||
OPEN_TAG = 1
|
||||
CLOSE_TAG = 2
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from scrapy.contrib.ibl.htmlpage import HtmlPage
|
|||
from scrapy.contrib.ibl.descriptor import (FieldDescriptor as A,
|
||||
ItemDescriptor)
|
||||
from scrapy.contrib.ibl.extractors import (contains_any_numbers,
|
||||
image_url)
|
||||
image_url, html, notags)
|
||||
|
||||
try:
|
||||
import numpy
|
||||
|
|
@ -719,19 +719,23 @@ EXTRACT_PAGE23 = u"""
|
|||
</body></html>
|
||||
"""
|
||||
|
||||
DEFAULT_DESCRIPTOR = ItemDescriptor('test',
|
||||
'item test, removes tags from description attribute',
|
||||
[A('description', 'description field without tags', notags)])
|
||||
|
||||
SAMPLE_DESCRIPTOR1 = ItemDescriptor('test', 'product test', [
|
||||
A('name', "Product name", required=True),
|
||||
A('price', "Product price, including any discounts and tax or vat",
|
||||
contains_any_numbers, True),
|
||||
A('image_urls', "URLs for one or more images", image_url, True),
|
||||
A('description', "The full description of the product", allow_markup=True),
|
||||
A('description', "The full description of the product", html),
|
||||
]
|
||||
)
|
||||
|
||||
# A list of (test name, [templates], page, extractors, expected_result)
|
||||
TEST_DATA = [
|
||||
# extract from a similar page
|
||||
('similar page extraction', [ANNOTATED_PAGE1], EXTRACT_PAGE1, None,
|
||||
('similar page extraction', [ANNOTATED_PAGE1], EXTRACT_PAGE1, DEFAULT_DESCRIPTOR,
|
||||
{u'title': [u'Nice Product'], u'description': [u'wonderful product'],
|
||||
u'image_url': [u'nice_product.jpg']}
|
||||
),
|
||||
|
|
@ -743,20 +747,20 @@ TEST_DATA = [
|
|||
u'image_url': [u'nice_product.jpg']}
|
||||
),
|
||||
# compilicated tag (multiple attributes and annotation)
|
||||
('multiple attributes and annotation', [ANNOTATED_PAGE2], EXTRACT_PAGE2, None,
|
||||
('multiple attributes and annotation', [ANNOTATED_PAGE2], EXTRACT_PAGE2, DEFAULT_DESCRIPTOR,
|
||||
{'name': [u'product 1'], 'image_url': [u'http://example.com/product1.jpg'],
|
||||
'description': [u'product 1 is great']}
|
||||
),
|
||||
# can only work out correct placement by matching the second attribute first
|
||||
('ambiguous description', [ANNOTATED_PAGE3], EXTRACT_PAGE3, None,
|
||||
('ambiguous description', [ANNOTATED_PAGE3], EXTRACT_PAGE3, DEFAULT_DESCRIPTOR,
|
||||
{'description': [u'description'], 'delivery': [u'delivery']}
|
||||
),
|
||||
# infer a repeated structure
|
||||
('repeated elements', [ANNOTATED_PAGE4], EXTRACT_PAGE4, None,
|
||||
('repeated elements', [ANNOTATED_PAGE4], EXTRACT_PAGE4, DEFAULT_DESCRIPTOR,
|
||||
{'features': [u'feature1', u'feature2', u'feature3']}
|
||||
),
|
||||
# identical variants with a repeated structure
|
||||
('repeated identical variants', [ANNOTATED_PAGE5], EXTRACT_PAGE5, None,
|
||||
('repeated identical variants', [ANNOTATED_PAGE5], EXTRACT_PAGE5, DEFAULT_DESCRIPTOR,
|
||||
{
|
||||
'description': [u'description'],
|
||||
'variants': [
|
||||
|
|
@ -767,7 +771,7 @@ TEST_DATA = [
|
|||
}
|
||||
),
|
||||
# variants with an irregular structure
|
||||
('irregular variants', [ANNOTATED_PAGE6], EXTRACT_PAGE6, None,
|
||||
('irregular variants', [ANNOTATED_PAGE6], EXTRACT_PAGE6, DEFAULT_DESCRIPTOR,
|
||||
{
|
||||
'description': [u'description'],
|
||||
'variants': [
|
||||
|
|
@ -779,7 +783,7 @@ TEST_DATA = [
|
|||
),
|
||||
|
||||
# discovering repeated variants in table columns
|
||||
# ('variants in table columns', [ANNOTATED_PAGE7], EXTRACT_PAGE7, None,
|
||||
# ('variants in table columns', [ANNOTATED_PAGE7], EXTRACT_PAGE7, DEFAULT_DESCRIPTOR,
|
||||
# {'variants': [
|
||||
# {u'colour': [u'colour 1'], u'price': [u'price 1']},
|
||||
# {u'colour': [u'colour 2'], u'price': [u'price 2']},
|
||||
|
|
@ -790,66 +794,66 @@ TEST_DATA = [
|
|||
|
||||
# ignored regions
|
||||
(
|
||||
'ignored_regions', [ANNOTATED_PAGE8], EXTRACT_PAGE8, None,
|
||||
'ignored_regions', [ANNOTATED_PAGE8], EXTRACT_PAGE8, DEFAULT_DESCRIPTOR,
|
||||
{
|
||||
'description': [u'\n A very nice product for all intelligent people \n\n'],
|
||||
'description': [u'\n A very nice product for all intelligent people \n \n'],
|
||||
'price': [u'\n12.00\n(VAT exc.)'],
|
||||
}
|
||||
),
|
||||
# shifted ignored regions (detected by region similarity)
|
||||
(
|
||||
'shifted_ignored_regions', [ANNOTATED_PAGE9], EXTRACT_PAGE9, None,
|
||||
'shifted_ignored_regions', [ANNOTATED_PAGE9], EXTRACT_PAGE9, DEFAULT_DESCRIPTOR,
|
||||
{
|
||||
'description': [u'\n A very nice product for all intelligent people \n\n'],
|
||||
'description': [u'\n A very nice product for all intelligent people \n \n'],
|
||||
'price': [u'\n12.00\n(VAT exc.)'],
|
||||
}
|
||||
),
|
||||
(# special case with partial annotations
|
||||
'special_partial_annotation', [ANNOTATED_PAGE11], EXTRACT_PAGE11, None,
|
||||
'special_partial_annotation', [ANNOTATED_PAGE11], EXTRACT_PAGE11, DEFAULT_DESCRIPTOR,
|
||||
{
|
||||
'name': [u'SL342'],
|
||||
'description': [u'\nSL342\n \nNice product for ladies\n \n£s;85.00\n'],
|
||||
'description': ['\nSL342\n \nNice product for ladies\n \n£s;85.00\n'],
|
||||
'price': [u'£s;85.00'],
|
||||
'price_before_discount': [u'£s;100.00'],
|
||||
}
|
||||
),
|
||||
(# with ignore-beneath feature
|
||||
'ignore-beneath', [ANNOTATED_PAGE12], EXTRACT_PAGE12a, None,
|
||||
'ignore-beneath', [ANNOTATED_PAGE12], EXTRACT_PAGE12a, DEFAULT_DESCRIPTOR,
|
||||
{
|
||||
'description': [u'\n A very nice product for all intelligent people \n'],
|
||||
}
|
||||
),
|
||||
(# ignore-beneath with extra tags
|
||||
'ignore-beneath with extra tags', [ANNOTATED_PAGE12], EXTRACT_PAGE12b, None,
|
||||
'ignore-beneath with extra tags', [ANNOTATED_PAGE12], EXTRACT_PAGE12b, DEFAULT_DESCRIPTOR,
|
||||
{
|
||||
'description': [u'\n A very nice product for all intelligent people \n'],
|
||||
}
|
||||
),
|
||||
('nested annotation with replica outside', [ANNOTATED_PAGE13a], EXTRACT_PAGE13a, None,
|
||||
('nested annotation with replica outside', [ANNOTATED_PAGE13a], EXTRACT_PAGE13a, DEFAULT_DESCRIPTOR,
|
||||
{'description': [u'\n A product \n $50.00 \nThis product is excelent. Buy it!\n \n'],
|
||||
'price': ["$50.00"],
|
||||
'name': [u'A product']}
|
||||
),
|
||||
('outside annotation with nested replica', [ANNOTATED_PAGE13b], EXTRACT_PAGE13b, None,
|
||||
('outside annotation with nested replica', [ANNOTATED_PAGE13b], EXTRACT_PAGE13b, DEFAULT_DESCRIPTOR,
|
||||
{'description': [u'\n A product \n $50.00 \nThis product is excelent. Buy it!\n'],
|
||||
'price': ["$45.00"],
|
||||
'name': [u'A product']}
|
||||
),
|
||||
('consistency check', [ANNOTATED_PAGE14], EXTRACT_PAGE14, None,
|
||||
('consistency check', [ANNOTATED_PAGE14], EXTRACT_PAGE14, DEFAULT_DESCRIPTOR,
|
||||
{},
|
||||
),
|
||||
('consecutive nesting', [ANNOTATED_PAGE15], EXTRACT_PAGE15, None,
|
||||
{'description': [u'Description\n\n'],
|
||||
('consecutive nesting', [ANNOTATED_PAGE15], EXTRACT_PAGE15, DEFAULT_DESCRIPTOR,
|
||||
{'description': [u'Description\n \n'],
|
||||
'price': [u'80.00']},
|
||||
),
|
||||
('nested inside not found', [ANNOTATED_PAGE16], EXTRACT_PAGE16, None,
|
||||
('nested inside not found', [ANNOTATED_PAGE16], EXTRACT_PAGE16, DEFAULT_DESCRIPTOR,
|
||||
{'price': [u'90.00'],
|
||||
'name': [u'product name']},
|
||||
),
|
||||
('ignored region helps to find attributes', [ANNOTATED_PAGE17], EXTRACT_PAGE17, None,
|
||||
('ignored region helps to find attributes', [ANNOTATED_PAGE17], EXTRACT_PAGE17, DEFAULT_DESCRIPTOR,
|
||||
{'description': [u'\nThis product is excelent. Buy it!\n']},
|
||||
),
|
||||
('ignored region in partial annotation', [ANNOTATED_PAGE18], EXTRACT_PAGE18, None,
|
||||
('ignored region in partial annotation', [ANNOTATED_PAGE18], EXTRACT_PAGE18, DEFAULT_DESCRIPTOR,
|
||||
{u'site_id': [u'Item Id'],
|
||||
u'description': [u'\nDescription\n']},
|
||||
),
|
||||
|
|
@ -864,13 +868,13 @@ TEST_DATA = [
|
|||
SAMPLE_DESCRIPTOR1,
|
||||
None,
|
||||
),
|
||||
('repeated partial annotations with variants', [ANNOTATED_PAGE20], EXTRACT_PAGE20, None,
|
||||
('repeated partial annotations with variants', [ANNOTATED_PAGE20], EXTRACT_PAGE20, DEFAULT_DESCRIPTOR,
|
||||
{u'variants': [
|
||||
{'price': ['270'], 'name': ['Twin']},
|
||||
{'price': ['330'], 'name': ['Queen']},
|
||||
]},
|
||||
),
|
||||
('variants with swatches', [ANNOTATED_PAGE21], EXTRACT_PAGE21, None,
|
||||
('variants with swatches', [ANNOTATED_PAGE21], EXTRACT_PAGE21, DEFAULT_DESCRIPTOR,
|
||||
{u'category': [u'chairs'],
|
||||
u'image_urls': [u'image.jpg'],
|
||||
u'variants': [
|
||||
|
|
@ -881,7 +885,7 @@ TEST_DATA = [
|
|||
]
|
||||
},
|
||||
),
|
||||
('variants with swatches complete', [ANNOTATED_PAGE22], EXTRACT_PAGE22, None,
|
||||
('variants with swatches complete', [ANNOTATED_PAGE22], EXTRACT_PAGE22, DEFAULT_DESCRIPTOR,
|
||||
{u'category': [u'chairs'],
|
||||
u'variants': [
|
||||
{u'swatches': [u'swatch1.jpg'],
|
||||
|
|
@ -899,7 +903,7 @@ TEST_DATA = [
|
|||
],
|
||||
u'image_urls': [u'image.jpg']},
|
||||
),
|
||||
('repeated (variants) with ignore annotations', [ANNOTATED_PAGE23], EXTRACT_PAGE23, None,
|
||||
('repeated (variants) with ignore annotations', [ANNOTATED_PAGE23], EXTRACT_PAGE23, DEFAULT_DESCRIPTOR,
|
||||
{'variants': [
|
||||
{u'price': [u'300'], u'name': [u'Variant 1']},
|
||||
{u'price': [u'320'], u'name': [u'Variant 2']},
|
||||
|
|
|
|||
|
|
@ -228,11 +228,11 @@ class TestPageParsing(TestCase):
|
|||
epp = _parse_page(ExtractionPageParser, SIMPLE_PAGE)
|
||||
ep = epp.to_extraction_page()
|
||||
assert len(ep.page_tokens) == 4
|
||||
assert ep.token_html(0) == '<html>'
|
||||
assert ep.token_html(1) == '<p some-attr="foo">'
|
||||
assert ep.htmlpage.fragment_data(ep.htmlpage_tag(0)) == '<html>'
|
||||
assert ep.htmlpage.fragment_data(ep.htmlpage_tag(1)) == '<p some-attr="foo">'
|
||||
|
||||
assert ep.html_between_tokens(1, 2) == 'this is a test'
|
||||
assert ep.html_between_tokens(1, 3) == 'this is a test</p> '
|
||||
assert ep.htmlpage_region_inside(1, 2) == 'this is a test'
|
||||
assert ep.htmlpage_region_inside(1, 3) == 'this is a test</p> '
|
||||
|
||||
def test_invalid_html(self):
|
||||
p = _parse_page(InstanceLearningParser, BROKEN_PAGE)
|
||||
|
|
|
|||
Loading…
Reference in New Issue