RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow to reapply a labelled region so to allow to use ignored regions inside repeated variants

This commit is contained in:
Martin Olveyra 2010-11-12 13:30:39 -02:00
parent 08bbbc2f82
commit b4cc2d91f4
2 changed files with 41 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/contrib/ibl/extraction/regionextract.py
View File

@ -343,7 +343,7 @@ class RecordExtractor(object):
The region in the page to be extracted from may be specified using
start_index and end_index
"""
ignored_regions = [LabelledRegion(*i) for i in (ignored_regions or [])]
ignored_regions = [i if isinstance(i, LabelledRegion) else LabelledRegion(*i) for i in (ignored_regions or [])]
region_elements = sorted(self.extractors + ignored_regions, key=lambda x: _labelled(x).start_index)
_, _, attributes = self._doextract(page, region_elements, start_index,
end_index)

40
scrapy/tests/test_contrib_ibl/test_extraction.py
View File

@ -749,6 +749,39 @@ EXTRACT_PAGE22 = u"""
</body></html>
"""
ANNOTATED_PAGE23 = u"""
<html><body>
<h4>Product</h4>
<table>
<tr><td>
<p data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 1, &quot;annotations&quot;: {&quot;content&quot;: &quot;name&quot;}}">Variant 1<b data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 1, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}" data-scrapy-ignore="true">560</b></p>
</td></tr>
<tr><td>
<p>Variant 2<b>570</b></p>
</td></tr>
<tr><td>
<p data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 2, &quot;annotations&quot;: {&quot;content&quot;: &quot;name&quot;}}">Variant 3<b data-scrapy-annotate="{&quot;required&quot;: [], &quot;variant&quot;: 2, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}" data-scrapy-ignore="true">580</b></p>
</td></tr>
</table>
</body></html>
"""
EXTRACT_PAGE23 = u"""
<html><body>
<h4>Product</h4>
<table>
<tr><td>
<p>Variant 1<b>300</b></p>
</td></tr>
<tr><td>
<p>Variant 2<b>320</b></p>
</td></tr>
<tr><td>
<p>Variant 3<b>340</b></p>
</td></tr>
</table>
</body></html>
"""
SAMPLE_DESCRIPTOR1 = ItemDescriptor('test', 'product test', [
A('name', "Product name", required=True),
@ -959,6 +992,13 @@ TEST_DATA = [
],
u'image_urls': [u'image.jpg']},
),
('repeated (variants) with ignore annotations', [ANNOTATED_PAGE23], EXTRACT_PAGE23, None,
{'variants': [
{u'price': [u'300'], u'name': [u'Variant 1']},
{u'price': [u'320'], u'name': [u'Variant 2']},
{u'price': [u'340'], u'name': [u'Variant 3']}
]},
),
]
class TestExtraction(TestCase):