From 4bda0976b28917405f54c781afda5ea55b65b16b Mon Sep 17 00:00:00 2001 From: Laerte <5853172+Laerte@users.noreply.github.com> Date: Tue, 8 Feb 2022 10:57:19 -0300 Subject: [PATCH] Fix csviter call, add parse_rows test (#5394) --- scrapy/spiders/feed.py | 2 +- tests/test_spider.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index bef2d6b24..79e12e030 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -123,7 +123,7 @@ class CSVFeedSpider(Spider): process_results methods for pre and post-processing purposes. """ - for row in csviter(response, self.delimiter, self.headers, self.quotechar): + for row in csviter(response, self.delimiter, self.headers, quotechar=self.quotechar): ret = iterate_spider_output(self.parse_row(response, row)) for result_item in self.process_results(response, ret): yield result_item diff --git a/tests/test_spider.py b/tests/test_spider.py index a7c3ee048..689349999 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -21,6 +21,7 @@ from scrapy.spiders import ( ) from scrapy.linkextractors import LinkExtractor from scrapy.utils.test import get_crawler +from tests import get_testdata class SpiderTest(unittest.TestCase): @@ -167,6 +168,23 @@ class CSVFeedSpiderTest(SpiderTest): spider_class = CSVFeedSpider + def test_parse_rows(self): + body = get_testdata('feeds', 'feed-sample6.csv') + response = Response("http://example.org/dummy.csv", body=body) + + class _CrawlSpider(self.spider_class): + name = "test" + delimiter = "," + quotechar = "'" + + def parse_row(self, response, row): + return row + + spider = _CrawlSpider() + rows = list(spider.parse_rows(response)) + assert rows[0] == {'id': '1', 'name': 'alpha', 'value': 'foobar'} + assert len(rows) == 4 + class CrawlSpiderTest(SpiderTest):