From f4f39057cbbfa4daf66f82061e57101b88d88d05 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:46:45 +0300 Subject: [PATCH] Make csviter work on python 3.7 PEP 479 does not allow for StopIteration in generators. Instead, handle it explicitly, also use a for loop which looks simpler. --- scrapy/utils/iterators.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 73857b410..a12e14005 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -98,8 +98,9 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): """ encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or 'utf-8' - def _getrow(csv_r): - return [to_unicode(field, encoding) for field in next(csv_r)] + + def row_to_unicode(row_): + return [to_unicode(field, encoding) for field in row_] # Python 3 csv reader input object needs to return strings if six.PY3: @@ -113,10 +114,14 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): csv_r = csv.reader(lines, **kwargs) if not headers: - headers = _getrow(csv_r) + try: + row = next(csv_r) + except StopIteration: + return + headers = row_to_unicode(row) - while True: - row = _getrow(csv_r) + for row in csv_r: + row = row_to_unicode(row) if len(row) != len(headers): logger.warning("ignoring row %(csvlnum)d (length: %(csvrow)d, " "should be: %(csvheader)d)",