From e3c3ec2ba988f654be1676586714fd96dba32c23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 28 Apr 2020 13:48:50 +0200 Subject: [PATCH 1/6] Run quick tests first in Travis CI --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 66e1a9617..dc91dfe4c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,9 @@ matrix: python: 3.8 - env: TOXENV=flake8 python: 3.8 + - env: TOXENV=docs + python: 3.7 # Keep in sync with .readthedocs.yml + - env: TOXENV=pypy3 - env: TOXENV=py35 python: 3.5 @@ -28,8 +31,6 @@ matrix: python: 3.8 - env: TOXENV=py38-asyncio python: 3.8 - - env: TOXENV=docs - python: 3.7 # Keep in sync with .readthedocs.yml install: - | if [ "$TOXENV" = "pypy3" ]; then From 5c0f11b4ef1d58de4245d9f4ac9a26f21faf082c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 28 Apr 2020 17:32:53 +0200 Subject: [PATCH 2/6] Simplify the asyncio Tox environment --- .travis.yml | 4 ++-- tox.ini | 12 +----------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 66e1a9617..a924eb68c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ matrix: python: 3.5 - env: TOXENV=pinned python: 3.5 - - env: TOXENV=py35-asyncio + - env: TOXENV=asyncio python: 3.5.2 - env: TOXENV=py36 python: 3.6 @@ -26,7 +26,7 @@ matrix: python: 3.8 - env: TOXENV=extra-deps python: 3.8 - - env: TOXENV=py38-asyncio + - env: TOXENV=asyncio python: 3.8 - env: TOXENV=docs python: 3.7 # Keep in sync with .readthedocs.yml diff --git a/tox.ini b/tox.ini index cd118c921..697328ebd 100644 --- a/tox.ini +++ b/tox.ini @@ -102,16 +102,6 @@ setenv = {[docs]setenv} commands = sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck -[asyncio] +[testenv:asyncio] commands = {[testenv]commands} --reactor=asyncio - -[testenv:py35-asyncio] -basepython = python3.5 -deps = {[testenv]deps} -commands = {[asyncio]commands} - -[testenv:py38-asyncio] -basepython = python3.8 -deps = {[testenv]deps} -commands = {[asyncio]commands} From 3a64f3eb2902ed8168b78c43f3516cf657873cef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 28 Apr 2020 17:44:19 +0200 Subject: [PATCH 3/6] Remove TOXENV from .travis.yml unless needed --- .travis.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 66e1a9617..b029d8bda 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,17 +12,14 @@ matrix: - env: TOXENV=flake8 python: 3.8 - env: TOXENV=pypy3 - - env: TOXENV=py35 - python: 3.5 + - python: 3.5 - env: TOXENV=pinned python: 3.5 - env: TOXENV=py35-asyncio python: 3.5.2 - - env: TOXENV=py36 - python: 3.6 - - env: TOXENV=py37 - python: 3.7 - - env: TOXENV=py38 + - python: 3.6 + - python: 3.7 + - env: PYPI_RELEASE_JOB=true python: 3.8 - env: TOXENV=extra-deps python: 3.8 @@ -62,4 +59,4 @@ deploy: on: tags: true repo: scrapy/scrapy - condition: "$TOXENV == py37 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$" + condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$" From 83d7360bb709cf2c73680260c58b767006f42b12 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 4 May 2020 02:00:11 +0500 Subject: [PATCH 4/6] Don't mention unsupported package versions in docs --- docs/topics/settings.rst | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 18f81838f..e3da1bd12 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -420,10 +420,9 @@ connections (for ``HTTP10DownloadHandler``). .. note:: HTTP/1.0 is rarely used nowadays so you can safely ignore this setting, - unless you use Twisted<11.1, or if you really want to use HTTP/1.0 - and override :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme - accordingly, i.e. to - ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``. + unless you really want to use HTTP/1.0 and override + :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme accordingly, + i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``. .. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY @@ -447,7 +446,6 @@ or even enable client-side authentication (and various other things). Scrapy also has another context factory class that you can set, ``'scrapy.core.downloader.contextfactory.BrowserLikeContextFactory'``, which uses the platform's certificates to validate remote endpoints. - **This is only available if you use Twisted>=14.0.** If you do use a custom ContextFactory, make sure its ``__init__`` method accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping @@ -494,10 +492,6 @@ This setting must be one of these string values: - ``'TLSv1.2'``: forces TLS version 1.2 - ``'SSLv3'``: forces SSL version 3 (**not recommended**) -.. note:: - - We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13 - or above (Twisted>=14.0 if you can). .. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING @@ -660,8 +654,6 @@ If you want to disable it set to 0. spider attribute and per-request using :reqmeta:`download_maxsize` Request.meta key. - This feature needs Twisted >= 11.1. - .. setting:: DOWNLOAD_WARNSIZE DOWNLOAD_WARNSIZE @@ -679,8 +671,6 @@ If you want to disable it set to 0. spider attribute and per-request using :reqmeta:`download_warnsize` Request.meta key. - This feature needs Twisted >= 11.1. - .. setting:: DOWNLOAD_FAIL_ON_DATALOSS DOWNLOAD_FAIL_ON_DATALOSS From fe6154e4faee375e7f47d61ceafabde7a3289bf3 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 4 May 2020 18:18:38 +0500 Subject: [PATCH 5/6] clarify DOWNLOADER_HTTPCLIENTFACTORY docs --- docs/topics/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index e3da1bd12..f06d9db3c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -421,7 +421,7 @@ connections (for ``HTTP10DownloadHandler``). HTTP/1.0 is rarely used nowadays so you can safely ignore this setting, unless you really want to use HTTP/1.0 and override - :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme accordingly, + :setting:`DOWNLOAD_HANDLERS` for ``http(s)`` scheme accordingly, i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``. .. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY From 17c0cf64aee1641e1ad33c5b46a61435c5969f2f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 5 May 2020 19:14:48 -0300 Subject: [PATCH 6/6] Flake8: remove W504 code (#4525) Co-authored-by: Mikhail Korobov --- pytest.ini | 16 ++++++++-------- scrapy/contracts/__init__.py | 4 ++-- scrapy/downloadermiddlewares/redirect.py | 11 +++++++---- scrapy/extensions/telnet.py | 6 ++++-- scrapy/linkextractors/__init__.py | 3 +-- scrapy/spidermiddlewares/referer.py | 14 ++++++++------ scrapy/utils/gz.py | 3 +-- tests/test_utils_http.py | 8 ++++---- 8 files changed, 35 insertions(+), 30 deletions(-) diff --git a/pytest.ini b/pytest.ini index e8911ee3f..4f3494e0e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -44,12 +44,12 @@ flake8-ignore = scrapy/commands/startproject.py E127 E501 E128 scrapy/commands/version.py E501 E128 # scrapy/contracts - scrapy/contracts/__init__.py E501 W504 + scrapy/contracts/__init__.py E501 scrapy/contracts/default.py E128 # scrapy/core scrapy/core/engine.py E501 E128 E127 scrapy/core/scheduler.py E501 - scrapy/core/scraper.py E501 E128 W504 + scrapy/core/scraper.py E501 E128 scrapy/core/spidermw.py E501 E126 scrapy/core/downloader/__init__.py E501 scrapy/core/downloader/contextfactory.py E501 E128 E126 @@ -68,7 +68,7 @@ flake8-ignore = scrapy/downloadermiddlewares/httpcache.py E501 E126 scrapy/downloadermiddlewares/httpcompression.py E501 E128 scrapy/downloadermiddlewares/httpproxy.py E501 - scrapy/downloadermiddlewares/redirect.py E501 W504 + scrapy/downloadermiddlewares/redirect.py E501 scrapy/downloadermiddlewares/retry.py E501 E126 scrapy/downloadermiddlewares/robotstxt.py E501 scrapy/downloadermiddlewares/stats.py E501 @@ -79,7 +79,7 @@ flake8-ignore = scrapy/extensions/httpcache.py E128 E501 scrapy/extensions/memdebug.py E501 scrapy/extensions/spiderstate.py E501 - scrapy/extensions/telnet.py E501 W504 + scrapy/extensions/telnet.py E501 scrapy/extensions/throttle.py E501 # scrapy/http scrapy/http/common.py E501 @@ -90,7 +90,7 @@ flake8-ignore = scrapy/http/response/__init__.py E501 E128 scrapy/http/response/text.py E501 E128 E124 # scrapy/linkextractors - scrapy/linkextractors/__init__.py E501 E402 W504 + scrapy/linkextractors/__init__.py E501 E402 scrapy/linkextractors/lxmlhtml.py E501 # scrapy/loader scrapy/loader/__init__.py E501 E128 @@ -110,7 +110,7 @@ flake8-ignore = # scrapy/spidermiddlewares scrapy/spidermiddlewares/httperror.py E501 scrapy/spidermiddlewares/offsite.py E501 - scrapy/spidermiddlewares/referer.py E501 E129 W504 + scrapy/spidermiddlewares/referer.py E501 E129 scrapy/spidermiddlewares/urllength.py E501 # scrapy/spiders scrapy/spiders/__init__.py E501 E402 @@ -125,7 +125,7 @@ flake8-ignore = scrapy/utils/decorators.py E501 scrapy/utils/defer.py E501 E128 scrapy/utils/deprecate.py E128 E501 E127 - scrapy/utils/gz.py E501 W504 + scrapy/utils/gz.py E501 scrapy/utils/http.py F403 scrapy/utils/httpobj.py E501 scrapy/utils/iterators.py E501 @@ -234,7 +234,7 @@ flake8-ignore = tests/test_utils_datatypes.py E402 E501 tests/test_utils_defer.py E501 F841 tests/test_utils_deprecate.py F841 E501 - tests/test_utils_http.py E501 E128 W504 + tests/test_utils_http.py E501 E128 tests/test_utils_iterators.py E501 E128 E129 tests/test_utils_log.py E741 tests/test_utils_python.py E501 diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 41d4f25b2..5af3831a2 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -17,10 +17,10 @@ class ContractsManager: self.contracts[contract.name] = contract def tested_methods_from_spidercls(self, spidercls): + is_method = re.compile(r"^\s*@", re.MULTILINE).search methods = [] for key, value in getmembers(spidercls): - if (callable(value) and value.__doc__ and - re.search(r'^\s*@', value.__doc__, re.MULTILINE)): + if callable(value) and value.__doc__ and is_method(value.__doc__): methods.append(key) return methods diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 09ee8377e..b32afb8e4 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -60,11 +60,14 @@ class RedirectMiddleware(BaseRedirectMiddleware): Handle redirection of requests based on response status and meta-refresh html tag. """ + def process_response(self, request, response, spider): - if (request.meta.get('dont_redirect', False) or - response.status in getattr(spider, 'handle_httpstatus_list', []) or - response.status in request.meta.get('handle_httpstatus_list', []) or - request.meta.get('handle_httpstatus_all', False)): + if ( + request.meta.get('dont_redirect', False) + or response.status in getattr(spider, 'handle_httpstatus_list', []) + or response.status in request.meta.get('handle_httpstatus_list', []) + or request.meta.get('handle_httpstatus_all', False) + ): return response allowed_status = (301, 302, 303, 307, 308) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 04ffd7235..1663604e7 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -76,8 +76,10 @@ class TelnetConsole(protocol.ServerFactory): """An implementation of IPortal""" @defers def login(self_, credentials, mind, *interfaces): - if not (credentials.username == self.username.encode('utf8') and - credentials.checkPassword(self.password.encode('utf8'))): + if not ( + credentials.username == self.username.encode('utf8') + and credentials.checkPassword(self.password.encode('utf8')) + ): raise ValueError("Invalid credentials") protocol = telnet.TelnetBootstrapProtocol( diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index d0b5066b6..ae019c70f 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -61,8 +61,7 @@ class FilteringLinkExtractor: def __new__(cls, *args, **kwargs): from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor - if (issubclass(cls, FilteringLinkExtractor) and - not issubclass(cls, LxmlLinkExtractor)): + if issubclass(cls, FilteringLinkExtractor) and not issubclass(cls, LxmlLinkExtractor): warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, ' 'please use scrapy.linkextractors.LinkExtractor instead', ScrapyDeprecationWarning, stacklevel=2) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 3784de885..434067b00 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -163,9 +163,10 @@ class StrictOriginPolicy(ReferrerPolicy): name = POLICY_STRICT_ORIGIN def referrer(self, response_url, request_url): - if ((self.tls_protected(response_url) and - self.potentially_trustworthy(request_url)) - or not self.tls_protected(response_url)): + if ( + self.tls_protected(response_url) and self.potentially_trustworthy(request_url) + or not self.tls_protected(response_url) + ): return self.origin_referrer(response_url) @@ -213,9 +214,10 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): origin = self.origin(response_url) if origin == self.origin(request_url): return self.stripped_referrer(response_url) - elif ((self.tls_protected(response_url) and - self.potentially_trustworthy(request_url)) - or not self.tls_protected(response_url)): + elif ( + self.tls_protected(response_url) and self.potentially_trustworthy(request_url) + or not self.tls_protected(response_url) + ): return self.origin_referrer(response_url) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index c291ae237..fbd7bd18f 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -52,8 +52,7 @@ def is_gzipped(response): """Return True if the response is gzipped, or False otherwise""" ctype = response.headers.get('Content-Type', b'') cenc = response.headers.get('Content-Encoding', b'').lower() - return (_is_gzipped(ctype) or - (_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip'))) + return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip') def gzip_magic_number(response): diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py index 2fac3da1f..363b015a8 100644 --- a/tests/test_utils_http.py +++ b/tests/test_utils_http.py @@ -13,7 +13,7 @@ class ChunkedTest(unittest.TestCase): chunked_body += "8\r\n" + "sequence\r\n" chunked_body += "0\r\n\r\n" body = decode_chunked_transfer(chunked_body) - self.assertEqual(body, - "This is the data in the first chunk\r\n" + - "and this is the second one\r\n" + - "consequence") + self.assertEqual( + body, + "This is the data in the first chunk\r\nand this is the second one\r\nconsequence" + )