Merge remote-tracking branch 'upstream/master' into fix_iternodes

.travis.yml

@@ -19,16 +19,10 @@ matrix:
       python: 3.8
 
     - env: TOXENV=pinned
-      python: 3.5.2
+      python: 3.6.1
     - env: TOXENV=asyncio-pinned
-      python: 3.5.2  # We use additional code to support 3.5.3 and earlier
-    - env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0
-
-    - env: TOXENV=py
-      python: 3.5
-    - env: TOXENV=asyncio
-      python: 3.5  # We use specific code to support >= 3.5.4, < 3.6
-    - env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0
+      python: 3.6.1
+    - env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0
 
     - env: TOXENV=py
       python: 3.6

azure-pipelines.yml

@@ -4,11 +4,9 @@ pool:
   vmImage: 'windows-latest'
 strategy:
   matrix:
-    Python35:
-      python.version: '3.5'
-      TOXENV: windows-pinned
     Python36:
       python.version: '3.6'
+      TOXENV: windows-pinned
     Python37:
       python.version: '3.7'
     Python38:

pylintrc

@@ -68,6 +68,7 @@ disable=abstract-method,
         pointless-statement,
         pointless-string-statement,
         protected-access,
+        raise-missing-from,
         redefined-argument-from-local,
         redefined-builtin,
         redefined-outer-name,
@@ -75,6 +76,7 @@ disable=abstract-method,
         signature-differs,
         singleton-comparison,
         super-init-not-called,
+        super-with-arguments,
         superfluous-parens,
         too-few-public-methods,
         too-many-ancestors,

scrapy/core/downloader/handlers/http11.py

@@ -394,13 +394,14 @@ class ScrapyAgent:
         fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss)
 
         if maxsize and expected_size > maxsize:
-            error_msg = ("Cancelling download of %(url)s: expected response "
-                         "size (%(size)s) larger than download max size (%(maxsize)s).")
-            error_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}
+            warning_msg = ("Cancelling download of %(url)s: expected response "
+                           "size (%(size)s) larger than download max size (%(maxsize)s).")
+            warning_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}
+
+            logger.warning(warning_msg, warning_args)
 
-            logger.error(error_msg, error_args)
             txresponse._transport._producer.loseConnection()
-            raise defer.CancelledError(error_msg % error_args)
+            raise defer.CancelledError(warning_msg % warning_args)
 
         if warnsize and expected_size > warnsize:
             logger.warning("Expected response size (%(size)s) larger than "
@@ -523,11 +524,11 @@ class _ResponseReader(protocol.Protocol):
                 self._finish_response(flags=["download_stopped"], failure=failure)
 
         if self._maxsize and self._bytes_received > self._maxsize:
-            logger.error("Received (%(bytes)s) bytes larger than download "
-                         "max size (%(maxsize)s) in request %(request)s.",
-                         {'bytes': self._bytes_received,
-                          'maxsize': self._maxsize,
-                          'request': self._request})
+            logger.warning("Received (%(bytes)s) bytes larger than download "
+                           "max size (%(maxsize)s) in request %(request)s.",
+                           {'bytes': self._bytes_received,
+                            'maxsize': self._maxsize,
+                            'request': self._request})
             # Clear buffer earlier to avoid keeping data in memory for a long time.
             self._bodybuf.truncate(0)
             self._finished.cancel()

tests/mockserver.py

@@ -289,8 +289,8 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c
     )
     if cipher_string:
         ctx = factory.getContext()
-        # disabling TLS1.2+ because it unconditionally enables some strong ciphers
-        ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_2 | SSL_OP_NO_TLSv1_3)
+        # disabling TLS1.3 because it unconditionally enables some strong ciphers
+        ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL_OP_NO_TLSv1_3)
         ctx.set_cipher_list(to_bytes(cipher_string))
     return factory
 

tests/test_downloader_handlers.py

@@ -410,7 +410,7 @@ class Http11TestCase(HttpTestCase):
             request = Request(self.getURL('largechunkedfile'))
 
             def check(logger):
-                logger.error.assert_called_once_with(mock.ANY, mock.ANY)
+                logger.warning.assert_called_once_with(mock.ANY, mock.ANY)
 
             d = self.download_request(request, Spider('foo', download_maxsize=1500))
             yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)

tests/test_utils_python.py

@@ -3,8 +3,8 @@ import gc
 import operator
 import platform
 import unittest
+from datetime import datetime
 from itertools import count
-from sys import version_info
 from warnings import catch_warnings
 
 from scrapy.utils.python import (
@@ -216,15 +216,15 @@ class UtilsPythonTestCase(unittest.TestCase):
             self.assertEqual(get_func_args(str.split), [])
             self.assertEqual(get_func_args(" ".join), [])
             self.assertEqual(get_func_args(operator.itemgetter(2)), [])
-        else:
-            self.assertEqual(
-                get_func_args(str.split, stripself=True), ['sep', 'maxsplit'])
-            self.assertEqual(
-                get_func_args(operator.itemgetter(2), stripself=True), ['obj'])
-            if version_info < (3, 6):
-                self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])
-            else:
+        elif platform.python_implementation() == 'PyPy':
+            self.assertEqual(get_func_args(str.split, stripself=True), ['sep', 'maxsplit'])
+            self.assertEqual(get_func_args(operator.itemgetter(2), stripself=True), ['obj'])
+
+            build_date = datetime.strptime(platform.python_build()[1], '%b %d %Y')
+            if build_date >= datetime(2020, 4, 7):  # PyPy 3.6-v7.3.1
                 self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable'])
+            else:
+                self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])
 
     def test_without_none_values(self):
         self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])

tox.ini

@@ -14,7 +14,7 @@ deps =
     # Extras
     boto3>=1.13.0
     botocore>=1.4.87
-    Pillow>=3.4.2
+    Pillow>=4.0.0
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
@@ -78,7 +78,7 @@ deps =
     # Extras
     botocore==1.4.87
     google-cloud-storage==1.29.0
-    Pillow==3.4.2
+    Pillow==4.0.0
 
 [testenv:pinned]
 deps =