diff --git a/.travis.yml b/.travis.yml index 6bde973f4..cb54c8008 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,19 +15,25 @@ matrix: python: 3.7 # Keep in sync with .readthedocs.yml - env: TOXENV=pypy3 - - python: 3.5 + - env: TOXENV=py + python: 3.5 - env: TOXENV=pinned python: 3.5 - env: TOXENV=asyncio python: 3.5.2 - - python: 3.6 - - python: 3.7 - - env: PYPI_RELEASE_JOB=true + - env: TOXENV=py + python: 3.6 + - env: TOXENV=py + python: 3.7 + - env: TOXENV=py PYPI_RELEASE_JOB=true python: 3.8 + dist: bionic - env: TOXENV=extra-deps python: 3.8 + dist: bionic - env: TOXENV=asyncio python: 3.8 + dist: bionic install: - | if [ "$TOXENV" = "pypy3" ]; then diff --git a/pytest.ini b/pytest.ini index 7c3c9cc5b..44b0e070c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -109,7 +109,7 @@ flake8-ignore = # scrapy/spidermiddlewares scrapy/spidermiddlewares/httperror.py E501 scrapy/spidermiddlewares/offsite.py E501 - scrapy/spidermiddlewares/referer.py E501 E129 + scrapy/spidermiddlewares/referer.py E501 scrapy/spidermiddlewares/urllength.py E501 # scrapy/spiders scrapy/spiders/__init__.py E501 E402 @@ -233,7 +233,7 @@ flake8-ignore = tests/test_utils_defer.py E501 F841 tests/test_utils_deprecate.py F841 E501 tests/test_utils_http.py E501 - tests/test_utils_iterators.py E501 E129 + tests/test_utils_iterators.py E501 tests/test_utils_log.py E741 tests/test_utils_python.py E501 tests/test_utils_reqser.py E501 diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 8dc89c2e9..63da55718 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from collections import defaultdict import traceback import warnings +from collections import defaultdict from zope.interface import implementer @@ -16,6 +16,7 @@ class SpiderLoader: SpiderLoader is a class which locates and loads spiders in a Scrapy project. """ + def __init__(self, settings): self.spider_modules = settings.getlist('SPIDER_MODULES') self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY') @@ -29,13 +30,16 @@ class SpiderLoader: dupes.extend([ " {cls} named {name!r} (in {module})".format(module=mod, cls=cls, name=name) for mod, cls in locations + if len(locations) > 1 ]) if dupes: dupes_string = "\n\n".join(dupes) - msg = ("There are several spiders with the same name:\n\n" - "{}\n\n This can cause unexpected behavior.".format(dupes_string)) - warnings.warn(msg, UserWarning) + warnings.warn( + "There are several spiders with the same name:\n\n" + "{}\n\n This can cause unexpected behavior.".format(dupes_string), + category=UserWarning, + ) def _load_spiders(self, module): for spcls in iter_spider_classes(module): @@ -49,11 +53,13 @@ class SpiderLoader: self._load_spiders(module) except ImportError: if self.warn_only: - msg = ( + warnings.warn( "\n{tb}Could not load spiders from module '{modname}'. " - "See above traceback for details.".format(modname=name, tb=traceback.format_exc()) + "See above traceback for details.".format( + modname=name, tb=traceback.format_exc() + ), + category=RuntimeWarning, ) - warnings.warn(msg, RuntimeWarning) else: raise self._check_name_duplicates() @@ -76,8 +82,10 @@ class SpiderLoader: """ Return the list of spider names that can handle the given request. """ - return [name for name, cls in self._spiders.items() - if cls.handles_request(request)] + return [ + name for name, cls in self._spiders.items() + if cls.handles_request(request) + ] def list(self): """ diff --git a/tests/test_crawler.py b/tests/test_crawler.py index b4144ea1d..9151278a5 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -311,14 +311,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_ipv6_alternative_name_resolver(self): log = self.run_script('alternative_name_resolver.py') self.assertIn('Spider closed (finished)', log) - self.assertTrue(any([ - "twisted.internet.error.ConnectionRefusedError" in log, - "twisted.internet.error.ConnectError" in log, - ])) - self.assertTrue(any([ - "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log, - "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log, - ])) + self.assertNotIn("twisted.internet.error.DNSLookupError", log) def test_reactor_select(self): log = self.run_script("twisted_reactor_select.py") diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index c677dabe5..3a58cc36b 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -92,8 +92,8 @@ class XmliterTestCase(unittest.TestCase): # with bytes XmlResponse(url="http://example.com", body=body.encode('utf-8')), # Unicode body needs encoding information - XmlResponse(url="http://example.com", body=body, encoding='utf-8')): - + XmlResponse(url="http://example.com", body=body, encoding='utf-8'), + ): attrs = [] for x in self.xmliter(r, u'þingflokkur'): attrs.append((x.attrib['id'], diff --git a/tox.ini b/tox.ini index 697328ebd..2102fc602 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = security,flake8,py3 +envlist = security,flake8,py minversion = 1.7.0 [testenv]