RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added support for variants when applied to tag attributes. fixed handling of variants with single attribute. Removed unneeded object attribute surrounds_variant. Added new tests cases for fixes.

This commit is contained in:
Martin Olveyra 2010-10-26 16:11:04 -02:00
parent a5a6c2ae31
commit 41a85f9d14
5 changed files with 209 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scrapy/contrib/ibl/extraction/pageobjects.py
View File

@ -194,18 +194,17 @@ class AnnotationTag(object):
"""
__slots__ = ('surrounds_attribute', 'start_index', 'end_index',
'tag_attributes', 'annotation_text', 'variant_id',
'surrounds_variant','match_common_prefix', 'metadata')
'match_common_prefix', 'metadata')
def __init__(self, start_index, end_index, surrounds_attribute=None,
annotation_text=None, tag_attributes=None, variant_id=None,
surrounds_variant=None, match_common_prefix=False):
match_common_prefix=False):
self.start_index = start_index
self.end_index = end_index
self.surrounds_attribute = surrounds_attribute
self.annotation_text = annotation_text
self.tag_attributes = tag_attributes or []
self.variant_id = variant_id
self.surrounds_variant = surrounds_variant
self.match_common_prefix = match_common_prefix
self.metadata = {}

21
scrapy/contrib/ibl/extraction/pageparsing.py
View File

@ -201,29 +201,32 @@ class TemplatePageParser(InstanceLearningParser):
self.extra_required_attrs.extend(jannotation.pop('required', []))
variant_id = jannotation.pop('variant', 0)
if variant_id > 0:
self.variant_stack.append(variant_id)
annotation.surrounds_variant = variant_id
attribute_annotations = jannotation.pop('annotations', {}).items()
for extract_attribute, tag_value in attribute_annotations:
if extract_attribute == 'content':
annotation.surrounds_attribute = tag_value
else:
annotation.tag_attributes.append((extract_attribute, tag_value))
variant_id = jannotation.pop('variant', 0)
if variant_id > 0:
if annotation.surrounds_attribute is not None:
self.variant_stack.append(variant_id)
else:
annotation.variant_id = variant_id
annotation.metadata = jannotation
if annotation.annotation_text is None:
self.next_tag_index += 1
if self.variant_stack:
if self.variant_stack and annotation.variant_id is None:
variant_id = self.variant_stack[-1]
if variant_id == '0':
variant_id = None
annotation.variant_id = variant_id
# look for a closing tag if the content is important
if annotation.surrounds_attribute or annotation.surrounds_variant:
if annotation.surrounds_attribute:
self.labelled_tag_stacks[html_tag.tag].append(annotation)
else:
annotation.end_index = annotation.start_index + 1
@ -272,9 +275,9 @@ class TemplatePageParser(InstanceLearningParser):
self.next_tag_index += 1
if len(labelled_tags) == 0:
del self.labelled_tag_stacks[html_tag.tag]
if annotation.surrounds_variant and self.variant_stack:
if annotation.variant_id and self.variant_stack:
prev = self.variant_stack.pop()
if prev != annotation.surrounds_variant:
if prev != annotation.variant_id:
raise ValueError("unbalanced variant annotation tags")
def handle_data(self, html_data_fragment):

5
scrapy/contrib/ibl/extraction/regionextract.py
View File

@ -25,7 +25,7 @@ def build_extraction_tree(template, type_descriptor, trace=True):
extractors = BasicTypeExtractor.create(template.annotations, attribute_map)
if trace:
extractors = TraceExtractor.apply(template, extractors)
for cls in (RepeatedDataExtractor, AdjacentVariantExtractor, RepeatedDataExtractor,
for cls in (AdjacentVariantExtractor, RepeatedDataExtractor, AdjacentVariantExtractor, RepeatedDataExtractor,
RecordExtractor):
extractors = cls.apply(template, extractors)
if trace:
@ -465,9 +465,8 @@ class AdjacentVariantExtractor(RecordExtractor):
continue
if vid in adjacent_variants:
adjacent_variants.remove(vid)
elif len(list(egroup)) > 1:
else:
adjacent_variants.add(vid)
new_extractors = []
for variant, group_seq in groupby(extractors, variantf):
group_seq = list(group_seq)

144
scrapy/tests/test_contrib_ibl/test_extraction.py
View File

@ -633,6 +633,121 @@ EXTRACT_PAGE20 = u"""
</body></html>
"""
ANNOTATED_PAGE21 = u"""
<html><body>
<img src="image.jpg" data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;src&quot;: &quot;image_urls&quot;}}">
<p>
<table>
<tr><td><img src="swatch1.jpg" data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 1, &quot;annotations&quot;: {&quot;src&quot;: &quot;swatches&quot;}}"></td></tr>
<tr><td><img src="swatch2.jpg"></td></tr>
<tr><td><img src="swatch3.jpg"></td></tr>
<tr><td><img src="swatch4.jpg" data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 2, &quot;annotations&quot;: {&quot;src&quot;: &quot;swatches&quot;}}"></td></tr>
</table>
<div data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;category&quot;}}">tables</div>
</body></html>
"""
EXTRACT_PAGE21 = u"""
<html><body>
<img src="image.jpg">
<p>
<table>
<tr><td><img src="swatch1.jpg"></td></tr>
<tr><td><img src="swatch2.jpg"></td></tr>
<tr><td><img src="swatch3.jpg"></td></tr>
<tr><td><img src="swatch4.jpg"></td></tr>
</table>
<div>chairs</div>
</body></html>
"""
ANNOTATED_PAGE22 = u"""
<html><body>
<img src="image.jpg" data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;src&quot;: &quot;image_urls&quot;}}">
<p>
<table>
<tr><td>
<p data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 1, &quot;annotations&quot;: {&quot;content&quot;: &quot;name&quot;}}">product 1</p>
<b data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 1, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}">$67</b>
<img src="swatch1.jpg" data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 1, &quot;annotations&quot;: {&quot;src&quot;: &quot;swatches&quot;}}">
</td></tr>
<tr><td>
<p>product 2</p>
<b>$70</b>
<img src="swatch2.jpg">
</td></tr>
<tr><td>
<p>product 3</p>
<b>$73</b>
<img src="swatch3.jpg">
</td></tr>
<tr><td>
<p data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 2, &quot;annotations&quot;: {&quot;content&quot;: &quot;name&quot;}}">product 4</p>
<b data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 2, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}">$80</b>
<img src="swatch4.jpg" data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 2, &quot;annotations&quot;: {&quot;src&quot;: &quot;swatches&quot;}}">
</td></tr>
</table>
<div data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;category&quot;}}">tables</div>
</body></html>
"""
EXTRACT_PAGE22 = u"""
<html><body>
<img src="image.jpg">
<p>
<table>
<tr><td>
<p>product 1</p>
<b>$70</b>
<img src="swatch1.jpg">
</td></tr>
<tr><td>
<p>product 2</p>
<b>$80</b>
<img src="swatch2.jpg">
</td></tr>
<tr><td>
<p>product 3</p>
<b>$90</b>
<img src="swatch3.jpg">
</td></tr>
<tr><td>
<p>product 4</p>
<b>$100</b>
<img src="swatch4.jpg">
</td></tr>
</table>
<div>chairs</div>
</body></html>
"""
SAMPLE_DESCRIPTOR1 = ItemDescriptor('test', 'product test', [
A('name', "Product name", required=True),
A('price', "Product price, including any discounts and tax or vat",
@ -813,6 +928,35 @@ TEST_DATA = [
{'price': ['330'], 'name': ['Queen']},
]},
),
('variants with swatches', [ANNOTATED_PAGE21], EXTRACT_PAGE21, None,
{u'category': [u'chairs'],
u'image_urls': [u'image.jpg'],
u'variants': [
{'swatches': ['swatch1.jpg']},
{'swatches': ['swatch2.jpg']},
{'swatches': ['swatch3.jpg']},
{'swatches': ['swatch4.jpg']},
]
},
),
('variants with swatches complete', [ANNOTATED_PAGE22], EXTRACT_PAGE22, None,
{u'category': [u'chairs'],
u'variants': [
{u'swatches': [u'swatch1.jpg'],
u'price': [u'$70'],
u'name': [u'product 1']},
{u'swatches': [u'swatch2.jpg'],\
u'price': [u'$80'],
u'name': [u'product 2']},
{u'swatches': [u'swatch3.jpg'],
u'price': [u'$90'],
u'name': [u'product 3']},
{u'swatches': [u'swatch4.jpg'],
u'price': [u'$100'],
u'name': [u'product 4']}
],
u'image_urls': [u'image.jpg']},
),
]
class TestExtraction(TestCase):

50
scrapy/tests/test_contrib_ibl/test_pageparsing.py
View File

@ -155,6 +155,32 @@ Description
</body></html>
"""
LABELLED_PAGE9 = u"""
<html><body>
<img src="image.jpg" data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;src&quot;: &quot;image_urls&quot;}}">
<p data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 1, &quot;annotations&quot;: {&quot;content&quot;: &quot;name&quot;}}">product 1</p>
<b data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 1, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}">$67</b>
<p data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 2, &quot;annotations&quot;: {&quot;content&quot;: &quot;name&quot;}}">product 2</p>
<b data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 2, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}">$70</b>
<div data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;category&quot;}}">tables</div>
</body></html>
"""
LABELLED_PAGE10 = u"""
<html><body>
<img src="image.jpg" data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;src&quot;: &quot;image_urls&quot;}}">
<p data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 1, &quot;annotations&quot;: {&quot;content&quot;: &quot;name&quot;}}">product 1</p>
<b data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 1, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}">$67</b>
<img src="swatch1.jpg" data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 1, &quot;annotations&quot;: {&quot;src&quot;: &quot;swatches&quot;}}">
<p data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 2, &quot;annotations&quot;: {&quot;content&quot;: &quot;name&quot;}}">product 2</p>
<b data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 2, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}">$70</b>
<img src="swatch2.jpg" data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 2, &quot;annotations&quot;: {&quot;src&quot;: &quot;swatches&quot;}}">
<div data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;category&quot;}}">tables</div>
</body></html>
"""
def _parse_page(parser_class, pagetext):
htmlpage = HtmlPage(None, {}, pagetext)
parser = parser_class(TokenDict())
@ -268,7 +294,29 @@ class TestPageParsing(TestCase):
"""Test parsing of extra required attributes"""
p = _parse_page(TemplatePageParser, LABELLED_PAGE8)
self.assertEqual(p.extra_required_attrs, ["description"])
def test_variants(self):
"""Test parsing of variant annotations"""
annotations = _parse_page(TemplatePageParser, LABELLED_PAGE9).annotations
self.assertEqual(annotations[0].variant_id, None)
self.assertEqual(annotations[1].variant_id, 1)
self.assertEqual(annotations[2].variant_id, 1)
self.assertEqual(annotations[3].variant_id, 2)
self.assertEqual(annotations[4].variant_id, 2)
self.assertEqual(annotations[5].variant_id, None)
def test_variants_in_attributes(self):
"""Test parsing of variant annotations in attributes"""
annotations = _parse_page(TemplatePageParser, LABELLED_PAGE10).annotations
self.assertEqual(annotations[0].variant_id, None)
self.assertEqual(annotations[1].variant_id, 1)
self.assertEqual(annotations[2].variant_id, 1)
self.assertEqual(annotations[3].variant_id, 1)
self.assertEqual(annotations[4].variant_id, 2)
self.assertEqual(annotations[5].variant_id, 2)
self.assertEqual(annotations[6].variant_id, 2)
self.assertEqual(annotations[7].variant_id, None)
def test_site_pages(self):
"""
Tests from real pages. More reliable and easy to build for more complicated structures