diff --git a/scrapy/contrib/ibl/extraction/regionextract.py b/scrapy/contrib/ibl/extraction/regionextract.py index dd4bc6256..e84c82cd3 100644 --- a/scrapy/contrib/ibl/extraction/regionextract.py +++ b/scrapy/contrib/ibl/extraction/regionextract.py @@ -409,16 +409,16 @@ class RecordExtractor(object): _, _, nested_data = self._doextract(page, nested_regions, pindex, sindex) extracted_data += nested_data if following_regions: - _, _, following_data = self._doextract(page, following_regions, sindex or start_index, end_region) + _, _, following_data = self._doextract(page, following_regions, sindex or start_index, end_index) extracted_data += following_data elif following_regions: - end_index, _, following_data = self._doextract(page, following_regions, start_index, end_region) + end_index, _, following_data = self._doextract(page, following_regions, start_index, end_index) if end_index is not None: pindex, sindex, extracted_data = self._doextract(page, [first_region], start_index, end_index - 1, nested_regions, ignored_regions) extracted_data += following_data elif nested_regions: - _, _, nested_data = self._doextract(page, nested_regions, start_index, end_region) + _, _, nested_data = self._doextract(page, nested_regions, start_index, end_index) extracted_data += nested_data return pindex, sindex, extracted_data diff --git a/scrapy/tests/test_contrib_ibl/test_extraction.py b/scrapy/tests/test_contrib_ibl/test_extraction.py index d4acbf5d5..2c5cbb30a 100644 --- a/scrapy/tests/test_contrib_ibl/test_extraction.py +++ b/scrapy/tests/test_contrib_ibl/test_extraction.py @@ -607,6 +607,32 @@ EXTRACT_PAGE19b = u""" """ +ANNOTATED_PAGE20 = u""" + +

Product Name

+ +
+Twin: $270 - November 2010
+Queen: $330 - In stock
+
+ +""" + +EXTRACT_PAGE20 = u""" + +

Product Name

+ +
+Twin: $270 - November 2010
+Queen: $330 - Movember 2010
+
+ +""" + SAMPLE_DESCRIPTOR1 = ItemDescriptor('test', 'product test', [ A('name', "Product name", required=True), A('price', "Product price, including any discounts and tax or vat", @@ -781,6 +807,12 @@ TEST_DATA = [ SAMPLE_DESCRIPTOR1, None, ), + ('repeated partial annotations with variants', [ANNOTATED_PAGE20], EXTRACT_PAGE20, None, + {u'variants': [ + {'price': ['270'], 'name': ['Twin']}, + {'price': ['330'], 'name': ['Queen']}, + ]}, + ), ] class TestExtraction(TestCase):