Merge branch 'master' into test-ubuntu-bionic

.travis.yml

@@ -11,28 +11,26 @@ matrix:
       python: 3.8
     - env: TOXENV=flake8
       python: 3.8
+    - env: TOXENV=docs
+      python: 3.7  # Keep in sync with .readthedocs.yml
+
     - env: TOXENV=pypy3
-    - env: TOXENV=py35
-      python: 3.5
+    - python: 3.5
     - env: TOXENV=pinned
       python: 3.5
-    - env: TOXENV=py35-asyncio
+    - env: TOXENV=asyncio
       python: 3.5.2
-    - env: TOXENV=py36
-      python: 3.6
-    - env: TOXENV=py37
-      python: 3.7
-    - env: TOXENV=py38
+    - python: 3.6
+    - python: 3.7
+    - env: PYPI_RELEASE_JOB=true
       python: 3.8
       dist: bionic
     - env: TOXENV=extra-deps
       python: 3.8
       dist: bionic
-    - env: TOXENV=py38-asyncio
+    - env: TOXENV=asyncio
       python: 3.8
       dist: bionic
-    - env: TOXENV=docs
-      python: 3.7  # Keep in sync with .readthedocs.yml
 install:
   - |
       if [ "$TOXENV" = "pypy3" ]; then
@@ -65,4 +63,4 @@ deploy:
   on:
     tags: true
     repo: scrapy/scrapy
-    condition: "$TOXENV == py37 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
+    condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"

docs/topics/settings.rst

@@ -420,10 +420,9 @@ connections (for ``HTTP10DownloadHandler``).
 .. note::
 
     HTTP/1.0 is rarely used nowadays so you can safely ignore this setting,
-    unless you use Twisted<11.1, or if you really want to use HTTP/1.0
-    and override :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme
-    accordingly, i.e. to
-    ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
+    unless you really want to use HTTP/1.0 and override
+    :setting:`DOWNLOAD_HANDLERS` for ``http(s)`` scheme accordingly,
+    i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
 
 .. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY
 
@@ -447,7 +446,6 @@ or even enable client-side authentication (and various other things).
     Scrapy also has another context factory class that you can set,
     ``'scrapy.core.downloader.contextfactory.BrowserLikeContextFactory'``,
     which uses the platform's certificates to validate remote endpoints.
-    **This is only available if you use Twisted>=14.0.**
 
 If you do use a custom ContextFactory, make sure its ``__init__`` method
 accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping
@@ -494,10 +492,6 @@ This setting must be one of these string values:
 - ``'TLSv1.2'``: forces TLS version 1.2
 - ``'SSLv3'``: forces SSL version 3 (**not recommended**)
 
-.. note::
-
-    We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13
-    or above (Twisted>=14.0 if you can).
 
 .. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
 
@@ -660,8 +654,6 @@ If you want to disable it set to 0.
     spider attribute and per-request using :reqmeta:`download_maxsize`
     Request.meta key.
 
-    This feature needs Twisted >= 11.1.
-
 .. setting:: DOWNLOAD_WARNSIZE
 
 DOWNLOAD_WARNSIZE
@@ -679,8 +671,6 @@ If you want to disable it set to 0.
     spider attribute and per-request using :reqmeta:`download_warnsize`
     Request.meta key.
 
-    This feature needs Twisted >= 11.1.
-
 .. setting:: DOWNLOAD_FAIL_ON_DATALOSS
 
 DOWNLOAD_FAIL_ON_DATALOSS

pytest.ini

@@ -44,12 +44,12 @@ flake8-ignore =
     scrapy/commands/startproject.py E127 E501 E128
     scrapy/commands/version.py E501 E128
     # scrapy/contracts
-    scrapy/contracts/__init__.py E501 W504
+    scrapy/contracts/__init__.py E501
     scrapy/contracts/default.py E128
     # scrapy/core
     scrapy/core/engine.py E501 E128 E127
     scrapy/core/scheduler.py E501
-    scrapy/core/scraper.py E501 E128 W504
+    scrapy/core/scraper.py E501 E128
     scrapy/core/spidermw.py E501 E126
     scrapy/core/downloader/__init__.py E501
     scrapy/core/downloader/contextfactory.py E501 E128 E126
@@ -68,7 +68,7 @@ flake8-ignore =
     scrapy/downloadermiddlewares/httpcache.py E501 E126
     scrapy/downloadermiddlewares/httpcompression.py E501 E128
     scrapy/downloadermiddlewares/httpproxy.py E501
-    scrapy/downloadermiddlewares/redirect.py E501 W504
+    scrapy/downloadermiddlewares/redirect.py E501
     scrapy/downloadermiddlewares/retry.py E501 E126
     scrapy/downloadermiddlewares/robotstxt.py E501
     scrapy/downloadermiddlewares/stats.py E501
@@ -79,7 +79,7 @@ flake8-ignore =
     scrapy/extensions/httpcache.py E128 E501
     scrapy/extensions/memdebug.py E501
     scrapy/extensions/spiderstate.py E501
-    scrapy/extensions/telnet.py E501 W504
+    scrapy/extensions/telnet.py E501
     scrapy/extensions/throttle.py E501
     # scrapy/http
     scrapy/http/common.py E501
@@ -90,7 +90,7 @@ flake8-ignore =
     scrapy/http/response/__init__.py E501 E128
     scrapy/http/response/text.py E501 E128 E124
     # scrapy/linkextractors
-    scrapy/linkextractors/__init__.py E501 E402 W504
+    scrapy/linkextractors/__init__.py E501 E402
     scrapy/linkextractors/lxmlhtml.py E501
     # scrapy/loader
     scrapy/loader/__init__.py E501 E128
@@ -110,7 +110,7 @@ flake8-ignore =
     # scrapy/spidermiddlewares
     scrapy/spidermiddlewares/httperror.py E501
     scrapy/spidermiddlewares/offsite.py E501
-    scrapy/spidermiddlewares/referer.py E501 E129 W504
+    scrapy/spidermiddlewares/referer.py E501 E129
     scrapy/spidermiddlewares/urllength.py E501
     # scrapy/spiders
     scrapy/spiders/__init__.py E501 E402
@@ -125,7 +125,7 @@ flake8-ignore =
     scrapy/utils/decorators.py E501
     scrapy/utils/defer.py E501 E128
     scrapy/utils/deprecate.py E128 E501 E127
-    scrapy/utils/gz.py E501 W504
+    scrapy/utils/gz.py E501
     scrapy/utils/http.py F403
     scrapy/utils/httpobj.py E501
     scrapy/utils/iterators.py E501
@@ -234,7 +234,7 @@ flake8-ignore =
     tests/test_utils_datatypes.py E402 E501
     tests/test_utils_defer.py E501 F841
     tests/test_utils_deprecate.py F841 E501
-    tests/test_utils_http.py E501 E128 W504
+    tests/test_utils_http.py E501 E128
     tests/test_utils_iterators.py E501 E128 E129
     tests/test_utils_log.py E741
     tests/test_utils_python.py E501

scrapy/contracts/__init__.py

@@ -17,10 +17,10 @@ class ContractsManager:
             self.contracts[contract.name] = contract
 
     def tested_methods_from_spidercls(self, spidercls):
+        is_method = re.compile(r"^\s*@", re.MULTILINE).search
         methods = []
         for key, value in getmembers(spidercls):
-            if (callable(value) and value.__doc__ and
-                    re.search(r'^\s*@', value.__doc__, re.MULTILINE)):
+            if callable(value) and value.__doc__ and is_method(value.__doc__):
                 methods.append(key)
 
         return methods

scrapy/downloadermiddlewares/redirect.py

@@ -60,11 +60,14 @@ class RedirectMiddleware(BaseRedirectMiddleware):
     Handle redirection of requests based on response status
     and meta-refresh html tag.
     """
+
     def process_response(self, request, response, spider):
-        if (request.meta.get('dont_redirect', False) or
-                response.status in getattr(spider, 'handle_httpstatus_list', []) or
-                response.status in request.meta.get('handle_httpstatus_list', []) or
-                request.meta.get('handle_httpstatus_all', False)):
+        if (
+            request.meta.get('dont_redirect', False)
+            or response.status in getattr(spider, 'handle_httpstatus_list', [])
+            or response.status in request.meta.get('handle_httpstatus_list', [])
+            or request.meta.get('handle_httpstatus_all', False)
+        ):
             return response
 
         allowed_status = (301, 302, 303, 307, 308)

scrapy/extensions/telnet.py

@@ -76,8 +76,10 @@ class TelnetConsole(protocol.ServerFactory):
             """An implementation of IPortal"""
             @defers
             def login(self_, credentials, mind, *interfaces):
-                if not (credentials.username == self.username.encode('utf8') and
-                        credentials.checkPassword(self.password.encode('utf8'))):
+                if not (
+                    credentials.username == self.username.encode('utf8')
+                    and credentials.checkPassword(self.password.encode('utf8'))
+                ):
                     raise ValueError("Invalid credentials")
 
                 protocol = telnet.TelnetBootstrapProtocol(

scrapy/linkextractors/__init__.py

@@ -61,8 +61,7 @@ class FilteringLinkExtractor:
 
     def __new__(cls, *args, **kwargs):
         from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
-        if (issubclass(cls, FilteringLinkExtractor) and
-                not issubclass(cls, LxmlLinkExtractor)):
+        if issubclass(cls, FilteringLinkExtractor) and not issubclass(cls, LxmlLinkExtractor):
             warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
                  'please use scrapy.linkextractors.LinkExtractor instead',
                  ScrapyDeprecationWarning, stacklevel=2)

scrapy/spidermiddlewares/referer.py

@@ -163,9 +163,10 @@ class StrictOriginPolicy(ReferrerPolicy):
     name = POLICY_STRICT_ORIGIN
 
     def referrer(self, response_url, request_url):
-        if ((self.tls_protected(response_url) and
-             self.potentially_trustworthy(request_url))
-            or not self.tls_protected(response_url)):
+        if (
+            self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
+            or not self.tls_protected(response_url)
+        ):
             return self.origin_referrer(response_url)
 
 
@@ -213,9 +214,10 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
         origin = self.origin(response_url)
         if origin == self.origin(request_url):
             return self.stripped_referrer(response_url)
-        elif ((self.tls_protected(response_url) and
-               self.potentially_trustworthy(request_url))
-              or not self.tls_protected(response_url)):
+        elif (
+            self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
+            or not self.tls_protected(response_url)
+        ):
             return self.origin_referrer(response_url)
 
 

scrapy/utils/gz.py

@@ -52,8 +52,7 @@ def is_gzipped(response):
     """Return True if the response is gzipped, or False otherwise"""
     ctype = response.headers.get('Content-Type', b'')
     cenc = response.headers.get('Content-Encoding', b'').lower()
-    return (_is_gzipped(ctype) or
-            (_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')))
+    return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')
 
 
 def gzip_magic_number(response):

tests/test_utils_http.py

@@ -13,7 +13,7 @@ class ChunkedTest(unittest.TestCase):
         chunked_body += "8\r\n" + "sequence\r\n"
         chunked_body += "0\r\n\r\n"
         body = decode_chunked_transfer(chunked_body)
-        self.assertEqual(body,
-                         "This is the data in the first chunk\r\n" +
-                         "and this is the second one\r\n" +
-                         "consequence")
+        self.assertEqual(
+            body,
+            "This is the data in the first chunk\r\nand this is the second one\r\nconsequence"
+        )

tox.ini

@@ -102,16 +102,6 @@ setenv = {[docs]setenv}
 commands =
     sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
 
-[asyncio]
+[testenv:asyncio]
 commands =
     {[testenv]commands} --reactor=asyncio
-
-[testenv:py35-asyncio]
-basepython = python3.5
-deps = {[testenv]deps}
-commands = {[asyncio]commands}
-
-[testenv:py38-asyncio]
-basepython = python3.8
-deps = {[testenv]deps}
-commands = {[asyncio]commands}