From aca2655c12d806759c6e0821a40d0277d200e0ea Mon Sep 17 00:00:00 2001 From: Patience Shyu Date: Fri, 2 Mar 2018 14:57:39 +0100 Subject: [PATCH 01/13] [WIP] Run tests for Python 3.7 --- .travis.yml | 2 ++ tox.ini | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6635f5d3b..e4df22139 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,8 @@ matrix: env: TOXENV=py36 - python: 3.6 env: TOXENV=docs + - python: 3.7 + env: TOXENV=py37 install: - | if [ "$TOXENV" = "pypy" ]; then diff --git a/tox.ini b/tox.ini index 60ff8c15e..5301624ee 100644 --- a/tox.ini +++ b/tox.ini @@ -79,6 +79,10 @@ deps = {[testenv:py34]deps} basepython = python3.6 deps = {[testenv:py34]deps} +[testenv:py37] +basepython = python3.7 +deps = {[testenv:py34]deps} + [testenv:pypy3] basepython = pypy3 deps = {[testenv:py34]deps} From fab68ff6260b9ce4f55ca7b211a1aeb3e8e6df3d Mon Sep 17 00:00:00 2001 From: Patience Shyu Date: Fri, 2 Mar 2018 17:05:14 +0100 Subject: [PATCH 02/13] Use 3.7-dev version for travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e4df22139..aa1a3c4c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ matrix: env: TOXENV=py36 - python: 3.6 env: TOXENV=docs - - python: 3.7 + - python: 3.7-dev env: TOXENV=py37 install: - | From ca7d79c29a55be4482bf2d4f704fc145ba801337 Mon Sep 17 00:00:00 2001 From: Patience Shyu Date: Mon, 5 Mar 2018 10:46:51 +0100 Subject: [PATCH 03/13] Install Twisted from branch to bypass syntax issue --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2a94d742d..47eddf1fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Twisted>=13.1.0 +git+https://github.com/twisted/twisted.git@dcaf946 lxml pyOpenSSL cssselect>=0.9 From 5d1f5245f2699745e73449b013a29cc370f424c9 Mon Sep 17 00:00:00 2001 From: Patience Shyu Date: Mon, 5 Mar 2018 11:14:50 +0100 Subject: [PATCH 04/13] [WIP] Install Twisted from branch to bypass syntax issue --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 47eddf1fc..95cd37772 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -git+https://github.com/twisted/twisted.git@dcaf946 +git+https://github.com/lopuhin/twisted.git@9384-remove-async-param lxml pyOpenSSL cssselect>=0.9 From f10a43d562dee32f324703fddb19bee5266912ce Mon Sep 17 00:00:00 2001 From: Patience Shyu Date: Mon, 5 Mar 2018 11:43:39 +0100 Subject: [PATCH 05/13] [WIP] Install Twisted from branch for py3.7 --- requirements-py3.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index 2aae3ae65..c3357e970 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,4 +1,4 @@ -Twisted >= 17.9.0 +git+https://github.com/lopuhin/twisted.git@9384-remove-async-param lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9 diff --git a/requirements.txt b/requirements.txt index 95cd37772..2a94d742d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -git+https://github.com/lopuhin/twisted.git@9384-remove-async-param +Twisted>=13.1.0 lxml pyOpenSSL cssselect>=0.9 From 722e1afcdb337bf11652167f02435c81fc68ecfb Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:21:19 +0300 Subject: [PATCH 06/13] Update ancient pytest on python 3 2.9 gives collection errors on python 3.7 due to PEP 479. --- tests/requirements-py3.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 51a25f5e5..8d9ce5231 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,6 +1,6 @@ -pytest==2.9.2 +pytest==3.6.3 pytest-twisted -pytest-cov==2.2.1 +pytest-cov==2.5.1 testfixtures jmespath leveldb From 17e9914b8a12e5a96cc40016b74f301fb9835cbf Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:26:09 +0300 Subject: [PATCH 07/13] Catch SyntaxError as well when importing manhole Also give a more detailed reason why telnet is not enabled (for the future). --- requirements-py3.txt | 2 +- scrapy/extensions/telnet.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index b941fd867..b38c4cc09 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,4 +1,4 @@ -git+https://github.com/lopuhin/twisted.git@9384-remove-async-param +Twisted>=17.9.0 lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9 diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index e78afa1fc..7cc8f823a 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -12,7 +12,7 @@ try: from twisted.conch import manhole, telnet from twisted.conch.insults import insults TWISTED_CONCH_AVAILABLE = True -except ImportError: +except (ImportError, SyntaxError): TWISTED_CONCH_AVAILABLE = False from scrapy.exceptions import NotConfigured @@ -40,7 +40,8 @@ class TelnetConsole(protocol.ServerFactory): if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'): raise NotConfigured if not TWISTED_CONCH_AVAILABLE: - raise NotConfigured + raise NotConfigured('TelnetConsole not enabled: failed to import ' + 'required twisted modules.') self.crawler = crawler self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] From cf9399acc149cf5eafb2d00d310416ab2ba185e5 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:26:56 +0300 Subject: [PATCH 08/13] Use python 3.7 on travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 29f9f0065..f6ea670ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ matrix: env: TOXENV=py36 - python: 3.6 env: TOXENV=docs - - python: 3.7-dev + - python: 3.7 env: TOXENV=py37 install: - | From 2773fe09e4b4fac51dbc06725f1fafb2e5d9a271 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:36:58 +0300 Subject: [PATCH 09/13] Make "docs" the last build, even though it still uses python3.6 for now --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f6ea670ae..88c72b08e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,10 +21,10 @@ matrix: env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 - - python: 3.6 - env: TOXENV=docs - python: 3.7 env: TOXENV=py37 + - python: 3.6 + env: TOXENV=docs install: - | if [ "$TOXENV" = "pypy" ]; then From f4f39057cbbfa4daf66f82061e57101b88d88d05 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:46:45 +0300 Subject: [PATCH 10/13] Make csviter work on python 3.7 PEP 479 does not allow for StopIteration in generators. Instead, handle it explicitly, also use a for loop which looks simpler. --- scrapy/utils/iterators.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 73857b410..a12e14005 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -98,8 +98,9 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): """ encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or 'utf-8' - def _getrow(csv_r): - return [to_unicode(field, encoding) for field in next(csv_r)] + + def row_to_unicode(row_): + return [to_unicode(field, encoding) for field in row_] # Python 3 csv reader input object needs to return strings if six.PY3: @@ -113,10 +114,14 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): csv_r = csv.reader(lines, **kwargs) if not headers: - headers = _getrow(csv_r) + try: + row = next(csv_r) + except StopIteration: + return + headers = row_to_unicode(row) - while True: - row = _getrow(csv_r) + for row in csv_r: + row = row_to_unicode(row) if len(row) != len(headers): logger.warning("ignoring row %(csvlnum)d (length: %(csvrow)d, " "should be: %(csvheader)d)", From b3cd12dc48592fb8b1d4c6883315fe2b341ca5c2 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:53:40 +0300 Subject: [PATCH 11/13] Try to get python3.7 by using xenial base and sudo See https://github.com/travis-ci/travis-ci/issues/9815#issuecomment-401756442 and https://github.com/travis-ci/travis-ci/issues/9815#issuecomment-402045581 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 88c72b08e..4218d13bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,8 @@ matrix: env: TOXENV=py36 - python: 3.7 env: TOXENV=py37 + dist: xenial + sudo: true - python: 3.6 env: TOXENV=docs install: From 92b504eae5379dadade2d78efba4e54b201cbd93 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 13:43:36 +0300 Subject: [PATCH 12/13] Fix telnet warnings in tests Disable telnet console if it's not available, else we'll get an extra warning about failure to enable it, and tests will fail. --- tests/test_crawler.py | 6 ++++-- tests/test_utils_log.py | 7 ++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index d3b80f460..6a8e11363 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,5 +1,4 @@ import logging -import os import tempfile import warnings import unittest @@ -14,8 +13,9 @@ from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider from scrapy.utils.misc import load_object -from scrapy.utils.test import get_crawler from scrapy.extensions.throttle import AutoThrottle +from scrapy.extensions import telnet + class BaseCrawlerTest(unittest.TestCase): @@ -100,6 +100,8 @@ class CrawlerLoggingTestCase(unittest.TestCase): custom_settings = { 'LOG_LEVEL': 'INFO', 'LOG_FILE': log_file.name, + # disable telnet if not available to avoid an extra warning + 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, } configure_logging() diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 45527b03b..742e04803 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -10,6 +10,7 @@ from twisted.python.failure import Failure from scrapy.utils.log import (failure_to_exc_info, TopLevelFormatter, LogCounterHandler, StreamLogger) from scrapy.utils.test import get_crawler +from scrapy.extensions import telnet class FailureToExcInfoTest(unittest.TestCase): @@ -65,10 +66,14 @@ class TopLevelFormatterTest(unittest.TestCase): class LogCounterHandlerTest(unittest.TestCase): def setUp(self): + settings = {'LOG_LEVEL': 'WARNING'} + if not telnet.TWISTED_CONCH_AVAILABLE: + # disable it to avoid the extra warning + settings['TELNETCONSOLE_ENABLED'] = False self.logger = logging.getLogger('test') self.logger.setLevel(logging.NOTSET) self.logger.propagate = False - self.crawler = get_crawler(settings_dict={'LOG_LEVEL': 'WARNING'}) + self.crawler = get_crawler(settings_dict=settings) self.handler = LogCounterHandler(self.crawler) self.logger.addHandler(self.handler) From 9428a4a3aa4c679c15eb2c606de7b49ad832ee6e Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 21:03:26 +0300 Subject: [PATCH 13/13] More visible telnet conch message Capture traceback when trying to import required twisted modules, print it in case telnet is enabled, and mention settings variable that can be used to supress the message. Thanks @kmike! --- scrapy/extensions/telnet.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 7cc8f823a..3024ddfaa 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -6,6 +6,7 @@ See documentation in docs/topics/telnetconsole.rst import pprint import logging +import traceback from twisted.internet import protocol try: @@ -13,6 +14,7 @@ try: from twisted.conch.insults import insults TWISTED_CONCH_AVAILABLE = True except (ImportError, SyntaxError): + _TWISTED_CONCH_TRACEBACK = traceback.format_exc() TWISTED_CONCH_AVAILABLE = False from scrapy.exceptions import NotConfigured @@ -40,8 +42,9 @@ class TelnetConsole(protocol.ServerFactory): if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'): raise NotConfigured if not TWISTED_CONCH_AVAILABLE: - raise NotConfigured('TelnetConsole not enabled: failed to import ' - 'required twisted modules.') + raise NotConfigured( + 'TELNETCONSOLE_ENABLED setting is True but required twisted ' + 'modules failed to import:\n' + _TWISTED_CONCH_TRACEBACK) self.crawler = crawler self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')]