diff --git a/scrapy/contrib/ibl/extraction/regionextract.py b/scrapy/contrib/ibl/extraction/regionextract.py index fc5470f5b..5fb368f5e 100644 --- a/scrapy/contrib/ibl/extraction/regionextract.py +++ b/scrapy/contrib/ibl/extraction/regionextract.py @@ -343,7 +343,7 @@ class RecordExtractor(object): The region in the page to be extracted from may be specified using start_index and end_index """ - ignored_regions = [LabelledRegion(*i) for i in (ignored_regions or [])] + ignored_regions = [i if isinstance(i, LabelledRegion) else LabelledRegion(*i) for i in (ignored_regions or [])] region_elements = sorted(self.extractors + ignored_regions, key=lambda x: _labelled(x).start_index) _, _, attributes = self._doextract(page, region_elements, start_index, end_index) diff --git a/scrapy/tests/test_contrib_ibl/test_extraction.py b/scrapy/tests/test_contrib_ibl/test_extraction.py index 97027114b..0dabccd5b 100644 --- a/scrapy/tests/test_contrib_ibl/test_extraction.py +++ b/scrapy/tests/test_contrib_ibl/test_extraction.py @@ -749,6 +749,39 @@ EXTRACT_PAGE22 = u""" """ +ANNOTATED_PAGE23 = u""" + +

Product

+ + + + +
+

Variant 1560

+
+

Variant 2570

+
+

Variant 3580

+
+ +""" + +EXTRACT_PAGE23 = u""" + +

Product

+ + + + +
+

Variant 1300

+
+

Variant 2320

+
+

Variant 3340

+
+ +""" SAMPLE_DESCRIPTOR1 = ItemDescriptor('test', 'product test', [ A('name', "Product name", required=True), @@ -959,6 +992,13 @@ TEST_DATA = [ ], u'image_urls': [u'image.jpg']}, ), + ('repeated (variants) with ignore annotations', [ANNOTATED_PAGE23], EXTRACT_PAGE23, None, + {'variants': [ + {u'price': [u'300'], u'name': [u'Variant 1']}, + {u'price': [u'320'], u'name': [u'Variant 2']}, + {u'price': [u'340'], u'name': [u'Variant 3']} + ]}, + ), ] class TestExtraction(TestCase):