From 15d96ab8b5fa6d349a45920f93d349b8ea1d0372 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 28 Apr 2020 17:09:05 +0200 Subject: [PATCH 1/6] Test the latest Ubuntu along the latest Python --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 66e1a9617..02c8885e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,10 +24,13 @@ matrix: python: 3.7 - env: TOXENV=py38 python: 3.8 + dist: bionic - env: TOXENV=extra-deps python: 3.8 + dist: bionic - env: TOXENV=py38-asyncio python: 3.8 + dist: bionic - env: TOXENV=docs python: 3.7 # Keep in sync with .readthedocs.yml install: From f787b8483ceb37ad8c9764d5a28be07028d85f70 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 20 Apr 2020 12:05:15 -0300 Subject: [PATCH 2/6] IPv6 test: check for the absence of DNSLookupError --- tests/test_crawler.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index b4144ea1d..9151278a5 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -311,14 +311,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_ipv6_alternative_name_resolver(self): log = self.run_script('alternative_name_resolver.py') self.assertIn('Spider closed (finished)', log) - self.assertTrue(any([ - "twisted.internet.error.ConnectionRefusedError" in log, - "twisted.internet.error.ConnectError" in log, - ])) - self.assertTrue(any([ - "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log, - "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log, - ])) + self.assertNotIn("twisted.internet.error.DNSLookupError", log) def test_reactor_select(self): log = self.run_script("twisted_reactor_select.py") From 418b9b5f5222e05bec497a83af8c7dfcca30c6cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 6 May 2020 11:15:02 +0200 Subject: [PATCH 3/6] Travis CI: do not run security and Flake8 on multiple jobs --- .travis.yml | 11 +++++++---- tox.ini | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6bde973f4..75d3c5a98 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,14 +15,17 @@ matrix: python: 3.7 # Keep in sync with .readthedocs.yml - env: TOXENV=pypy3 - - python: 3.5 + - env: TOXENV=py + python: 3.5 - env: TOXENV=pinned python: 3.5 - env: TOXENV=asyncio python: 3.5.2 - - python: 3.6 - - python: 3.7 - - env: PYPI_RELEASE_JOB=true + - env: TOXENV=py + python: 3.6 + - env: TOXENV=py + python: 3.7 + - env: TOXENV=py PYPI_RELEASE_JOB=true python: 3.8 - env: TOXENV=extra-deps python: 3.8 diff --git a/tox.ini b/tox.ini index 697328ebd..2102fc602 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = security,flake8,py3 +envlist = security,flake8,py minversion = 1.7.0 [testenv] From 88efc988473fb0db8ca8fb512b2aab834c5aa7af Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 6 May 2020 16:42:47 -0300 Subject: [PATCH 4/6] Flake8: remove E129 --- pytest.ini | 4 ++-- tests/test_utils_iterators.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pytest.ini b/pytest.ini index 1a73b41be..ff0bb010f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -110,7 +110,7 @@ flake8-ignore = # scrapy/spidermiddlewares scrapy/spidermiddlewares/httperror.py E501 scrapy/spidermiddlewares/offsite.py E501 - scrapy/spidermiddlewares/referer.py E501 E129 + scrapy/spidermiddlewares/referer.py E501 scrapy/spidermiddlewares/urllength.py E501 # scrapy/spiders scrapy/spiders/__init__.py E501 E402 @@ -235,7 +235,7 @@ flake8-ignore = tests/test_utils_defer.py E501 F841 tests/test_utils_deprecate.py F841 E501 tests/test_utils_http.py E501 E128 - tests/test_utils_iterators.py E501 E128 E129 + tests/test_utils_iterators.py E501 E128 tests/test_utils_log.py E741 tests/test_utils_python.py E501 tests/test_utils_reqser.py E501 E128 diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index a85087619..46aaaecbc 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -93,8 +93,8 @@ class XmliterTestCase(unittest.TestCase): # with bytes XmlResponse(url="http://example.com", body=body.encode('utf-8')), # Unicode body needs encoding information - XmlResponse(url="http://example.com", body=body, encoding='utf-8')): - + XmlResponse(url="http://example.com", body=body, encoding='utf-8'), + ): attrs = [] for x in self.xmliter(r, u'þingflokkur'): attrs.append((x.attrib['id'], From 422e6429b56e42b8344a0e46c45f4106d374d024 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 7 May 2020 09:22:14 -0300 Subject: [PATCH 5/6] Add mising len check in spiderloader --- scrapy/spiderloader.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 8dc89c2e9..92aed9b8e 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from collections import defaultdict import traceback import warnings +from collections import defaultdict from zope.interface import implementer @@ -16,6 +16,7 @@ class SpiderLoader: SpiderLoader is a class which locates and loads spiders in a Scrapy project. """ + def __init__(self, settings): self.spider_modules = settings.getlist('SPIDER_MODULES') self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY') @@ -29,6 +30,7 @@ class SpiderLoader: dupes.extend([ " {cls} named {name!r} (in {module})".format(module=mod, cls=cls, name=name) for mod, cls in locations + if len(locations) > 1 ]) if dupes: @@ -49,10 +51,9 @@ class SpiderLoader: self._load_spiders(module) except ImportError: if self.warn_only: - msg = ( - "\n{tb}Could not load spiders from module '{modname}'. " - "See above traceback for details.".format(modname=name, tb=traceback.format_exc()) - ) + msg = ("\n{tb}Could not load spiders from module '{modname}'. " + "See above traceback for details.".format( + modname=name, tb=traceback.format_exc())) warnings.warn(msg, RuntimeWarning) else: raise @@ -76,8 +77,10 @@ class SpiderLoader: """ Return the list of spider names that can handle the given request. """ - return [name for name, cls in self._spiders.items() - if cls.handles_request(request)] + return [ + name for name, cls in self._spiders.items() + if cls.handles_request(request) + ] def list(self): """ From e0127a31230d4be13b1bd29e62d75c2954b47d9e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 7 May 2020 12:48:43 -0300 Subject: [PATCH 6/6] Refactor warnings in spiderloader --- scrapy/spiderloader.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 92aed9b8e..63da55718 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -35,9 +35,11 @@ class SpiderLoader: if dupes: dupes_string = "\n\n".join(dupes) - msg = ("There are several spiders with the same name:\n\n" - "{}\n\n This can cause unexpected behavior.".format(dupes_string)) - warnings.warn(msg, UserWarning) + warnings.warn( + "There are several spiders with the same name:\n\n" + "{}\n\n This can cause unexpected behavior.".format(dupes_string), + category=UserWarning, + ) def _load_spiders(self, module): for spcls in iter_spider_classes(module): @@ -51,10 +53,13 @@ class SpiderLoader: self._load_spiders(module) except ImportError: if self.warn_only: - msg = ("\n{tb}Could not load spiders from module '{modname}'. " - "See above traceback for details.".format( - modname=name, tb=traceback.format_exc())) - warnings.warn(msg, RuntimeWarning) + warnings.warn( + "\n{tb}Could not load spiders from module '{modname}'. " + "See above traceback for details.".format( + modname=name, tb=traceback.format_exc() + ), + category=RuntimeWarning, + ) else: raise self._check_name_duplicates()