From 5a55c4269d3389df4e486657caa58985f8e37e5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Thu, 31 Jan 2019 17:20:29 -0200
Subject: [PATCH 001/243] Adding GCSFeedStorage

---
 scrapy/extensions/feedexport.py | 26 ++++++++++++++++++
 tests/test_feedexport.py        | 48 ++++++++++++++++++++++++++++++++-
 2 files changed, 73 insertions(+), 1 deletion(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 22ebf3b3f..00d5d8025 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -146,6 +146,32 @@ class S3FeedStorage(BlockingFeedStorage):
             key.close()
 
 
+
+class GCSFeedStorage(BlockingFeedStorage):
+
+    project_id = None
+    bucket_name = None
+    blob_name = None
+
+    def __init__(self, uri, project_id):
+        self.project_id = project_id
+        u = urlparse(uri)
+        self.bucket_name = u.hostname
+        self.blob_name = u.path[1:]  # remove first "/"
+
+    @classmethod
+    def from_crawler(cls, crawler, uri):
+        return cls(uri, crawler.settings['GCS_PROJECT_ID'])
+
+    def _store_in_thread(self, file):
+        file.seek(0)
+        from google.cloud.storage import Client
+        client = Client(project=self.project_id)
+        bucket = client.get_bucket(self.bucket_name)
+        blob = bucket.blob(self.blob_name)
+        blob.upload_from_file(file)
+
+
 class FTPFeedStorage(BlockingFeedStorage):
 
     def __init__(self, uri):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index e46c8c14e..f3c499b3f 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -21,7 +21,7 @@ from w3lib.url import path_to_file_uri
 import scrapy
 from scrapy.exporters import CsvItemExporter
 from scrapy.extensions.feedexport import (
-    IFeedStorage, FileFeedStorage, FTPFeedStorage,
+    IFeedStorage, FileFeedStorage, FTPFeedStorage, GCSFeedStorage,
     S3FeedStorage, StdoutFeedStorage,
     BlockingFeedStorage)
 from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
@@ -187,6 +187,52 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(content, expected_content)
 
 
+class GCSFeedStorageTest(unittest.TestCase):
+
+    @mock.patch('scrapy.conf.settings',
+                new={'GCS_PROJECT_ID': 'conf_id' }, create=True)
+    def test_parse_settings(self):
+        try:
+            from google.cloud.storage import Client
+        except ImportError:
+            raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
+
+        settings = {'GCS_PROJECT_ID': '123' }
+        crawler = get_crawler(settings_dict=settings)
+        storage = GCSFeedStorage.from_crawler(crawler, 'gcs://mybucket/export.csv')
+        assert storage.project_id == '123'
+        assert storage.bucket_name == 'mybucket'
+        assert storage.blob_name == 'export.csv'
+
+    @defer.inlineCallbacks
+    def test_store(self):
+        try:
+            from google.cloud.storage import Client, Bucket, Blob
+        except ImportError:
+            raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
+
+        uri = 'gcs://mybucket/export.csv'
+        project_id = 'myproject-123'
+        with mock.patch('google.cloud.storage.Client') as m:
+            client_mock = mock.create_autospec(Client)
+            m.return_value = client_mock
+
+            bucket_mock = mock.create_autospec(Bucket)
+            client_mock.get_bucket.return_value = bucket_mock
+
+            blob_mock = mock.create_autospec(Blob)
+            bucket_mock.blob.return_value = blob_mock
+
+            f = mock.Mock()
+            storage = GCSFeedStorage(uri, project_id)
+            yield storage.store(f)
+
+            f.seek.assert_called_once_with(0)
+            m.assert_called_once_with(project=project_id)
+            client_mock.get_bucket.assert_called_once_with('mybucket')
+            bucket_mock.blob.assert_called_once_with('export.csv')
+            blob_mock.upload_from_file.assert_called_once_with(f)
+
 class StdoutFeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks

From a4059851e7b6c8d712b2bc73dbff99be2d569d21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Thu, 31 Jan 2019 18:29:15 -0200
Subject: [PATCH 002/243] Refactoring tests

---
 scrapy/utils/test.py     | 16 ++++++++++++++++
 tests/test_feedexport.py | 15 +++++----------
 2 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 4b935c51b..84eae97a7 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -7,6 +7,7 @@ import os
 
 from importlib import import_module
 from twisted.trial.unittest import SkipTest
+from tests import mock
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.boto import is_botocore
@@ -91,3 +92,18 @@ def assert_samelines(testcase, text1, text2, msg=None):
     line endings between platforms
     """
     testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)
+
+def mock_google_cloud_storage():
+    """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
+    classes and set their proper return values.
+    """
+    from google.cloud.storage import Client, Bucket, Blob
+    client_mock = mock.create_autospec(Client)
+
+    bucket_mock = mock.create_autospec(Bucket)
+    client_mock.get_bucket.return_value = bucket_mock
+
+    blob_mock = mock.create_autospec(Blob)
+    bucket_mock.blob.return_value = blob_mock
+
+    return (client_mock, bucket_mock, blob_mock)
\ No newline at end of file
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index f3c499b3f..6d23c68c3 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -24,7 +24,8 @@ from scrapy.extensions.feedexport import (
     IFeedStorage, FileFeedStorage, FTPFeedStorage, GCSFeedStorage,
     S3FeedStorage, StdoutFeedStorage,
     BlockingFeedStorage)
-from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
+from scrapy.utils.test import (assert_aws_environ, get_s3_content_and_delete,
+    get_crawler, mock_google_cloud_storage)
 from scrapy.utils.python import to_native_str
 
 
@@ -207,22 +208,16 @@ class GCSFeedStorageTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_store(self):
         try:
-            from google.cloud.storage import Client, Bucket, Blob
+            from google.cloud.storage import Client
         except ImportError:
             raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
 
         uri = 'gcs://mybucket/export.csv'
         project_id = 'myproject-123'
-        with mock.patch('google.cloud.storage.Client') as m:
-            client_mock = mock.create_autospec(Client)
+        (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()
+        with mock.patch('google.cloud.storage.Client') as m:    
             m.return_value = client_mock
 
-            bucket_mock = mock.create_autospec(Bucket)
-            client_mock.get_bucket.return_value = bucket_mock
-
-            blob_mock = mock.create_autospec(Blob)
-            bucket_mock.blob.return_value = blob_mock
-
             f = mock.Mock()
             storage = GCSFeedStorage(uri, project_id)
             yield storage.store(f)

From 1bb6c4154c43d03d00147d810147c4f3c807505e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Fri, 8 Feb 2019 09:04:01 -0200
Subject: [PATCH 003/243] Turning into instance attributes

---
 scrapy/extensions/feedexport.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 00d5d8025..a81f44045 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -149,10 +149,6 @@ class S3FeedStorage(BlockingFeedStorage):
 
 class GCSFeedStorage(BlockingFeedStorage):
 
-    project_id = None
-    bucket_name = None
-    blob_name = None
-
     def __init__(self, uri, project_id):
         self.project_id = project_id
         u = urlparse(uri)

From fc6809b024dc25663432b6c0c5780021a827ea20 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Fri, 8 Feb 2019 09:08:54 -0200
Subject: [PATCH 004/243] Add gcs schema to FEED_STORAGES_BASE

---
 scrapy/settings/default_settings.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 3734a0a58..1a12f35a3 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -145,6 +145,7 @@ FEED_STORAGES_BASE = {
     'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
+    'gcs': 'scrapy.extensions.feedexport.GCSFeedStorage',
 }
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {

From 4a53de165a53433a9a2a8cc4db34e5507c47fbd5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Fri, 8 Feb 2019 09:09:56 -0200
Subject: [PATCH 005/243] Sorted schemas alphabetically

---
 scrapy/extensions/feedexport.py     | 1 -
 scrapy/settings/default_settings.py | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index a81f44045..8347b42ca 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -146,7 +146,6 @@ class S3FeedStorage(BlockingFeedStorage):
             key.close()
 
 
-
 class GCSFeedStorage(BlockingFeedStorage):
 
     def __init__(self, uri, project_id):
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 1a12f35a3..8769c01ba 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -142,10 +142,10 @@ FEED_STORAGES = {}
 FEED_STORAGES_BASE = {
     '': 'scrapy.extensions.feedexport.FileFeedStorage',
     'file': 'scrapy.extensions.feedexport.FileFeedStorage',
-    'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
-    's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
     'gcs': 'scrapy.extensions.feedexport.GCSFeedStorage',
+    's3': 'scrapy.extensions.feedexport.S3FeedStorage',
+    'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
 }
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {

From 2bbbd02bda368c2f052f3aa38f99498a632328bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Fri, 8 Feb 2019 09:45:10 -0200
Subject: [PATCH 006/243] Adding an option to set ACL while uploading the blob
 to GCS

---
 scrapy/extensions/feedexport.py     | 11 ++++++++---
 scrapy/settings/default_settings.py |  2 ++
 tests/test_feedexport.py            | 10 ++++++----
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 8347b42ca..fbbf9bb97 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -148,15 +148,20 @@ class S3FeedStorage(BlockingFeedStorage):
 
 class GCSFeedStorage(BlockingFeedStorage):
 
-    def __init__(self, uri, project_id):
+    def __init__(self, uri, project_id, acl):
         self.project_id = project_id
+        self.acl = acl
         u = urlparse(uri)
         self.bucket_name = u.hostname
         self.blob_name = u.path[1:]  # remove first "/"
 
     @classmethod
     def from_crawler(cls, crawler, uri):
-        return cls(uri, crawler.settings['GCS_PROJECT_ID'])
+        return cls(
+            uri,
+            crawler.settings['GCS_PROJECT_ID'],
+            crawler.settings['FEED_STORAGE_GCS_ACL']
+        )
 
     def _store_in_thread(self, file):
         file.seek(0)
@@ -164,7 +169,7 @@ class GCSFeedStorage(BlockingFeedStorage):
         client = Client(project=self.project_id)
         bucket = client.get_bucket(self.bucket_name)
         blob = bucket.blob(self.blob_name)
-        blob.upload_from_file(file)
+        blob.upload_from_file(file, predefined_acl=self.acl)
 
 
 class FTPFeedStorage(BlockingFeedStorage):
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 8769c01ba..5d2862980 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -159,6 +159,8 @@ FEED_EXPORTERS_BASE = {
 }
 FEED_EXPORT_INDENT = 0
 
+FEED_STORAGE_GCS_ACL = None
+
 FILES_STORE_S3_ACL = 'private'
 FILES_STORE_GCS_ACL = ''
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 6d23c68c3..5cbca6d28 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -191,17 +191,18 @@ class S3FeedStorageTest(unittest.TestCase):
 class GCSFeedStorageTest(unittest.TestCase):
 
     @mock.patch('scrapy.conf.settings',
-                new={'GCS_PROJECT_ID': 'conf_id' }, create=True)
+                new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': None }, create=True)
     def test_parse_settings(self):
         try:
             from google.cloud.storage import Client
         except ImportError:
             raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
 
-        settings = {'GCS_PROJECT_ID': '123' }
+        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead' }
         crawler = get_crawler(settings_dict=settings)
         storage = GCSFeedStorage.from_crawler(crawler, 'gcs://mybucket/export.csv')
         assert storage.project_id == '123'
+        assert storage.acl == 'publicRead'
         assert storage.bucket_name == 'mybucket'
         assert storage.blob_name == 'export.csv'
 
@@ -214,19 +215,20 @@ class GCSFeedStorageTest(unittest.TestCase):
 
         uri = 'gcs://mybucket/export.csv'
         project_id = 'myproject-123'
+        acl = 'publicRead'
         (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()
         with mock.patch('google.cloud.storage.Client') as m:    
             m.return_value = client_mock
 
             f = mock.Mock()
-            storage = GCSFeedStorage(uri, project_id)
+            storage = GCSFeedStorage(uri, project_id, acl)
             yield storage.store(f)
 
             f.seek.assert_called_once_with(0)
             m.assert_called_once_with(project=project_id)
             client_mock.get_bucket.assert_called_once_with('mybucket')
             bucket_mock.blob.assert_called_once_with('export.csv')
-            blob_mock.upload_from_file.assert_called_once_with(f)
+            blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
 
 class StdoutFeedStorageTest(unittest.TestCase):
 

From cb5f800b0f7029d2bf2e09e9bc1391078c133a92 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Fri, 8 Feb 2019 11:26:33 -0200
Subject: [PATCH 007/243] Adding documentation about Google Cloud Storage Feed
 Export

---
 docs/topics/feed-exports.rst        | 22 ++++++++++++++++++++++
 docs/topics/settings.rst            | 18 ++++++++++++++++++
 scrapy/settings/default_settings.py |  2 ++
 3 files changed, 42 insertions(+)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index b64dbfbfd..efb63b0ba 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -185,6 +185,27 @@ passed through the following settings:
  * :setting:`AWS_ACCESS_KEY_ID`
  * :setting:`AWS_SECRET_ACCESS_KEY`
 
+.. _topics-feed-storage-gcs:
+
+Google Cloud Storage (GCS)
+--------------------------
+
+The feeds are stored on `Google Cloud Storage`_.
+
+ * URI scheme: ``gcs``
+ * Example URIs:
+
+   * ``gcs://mybucket/path/to/export.csv``
+
+ * Required external libraries: `google-cloud-storage <https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python>`_.
+
+For more information about authentication, please refer to `Google Cloud documentation <https://cloud.google.com/docs/authentication/production>`_.
+
+You can set a *Project ID* and *Access Control List (ACL)* through the following settings:
+
+ * :setting:`FEED_STORAGE_GCS_ACL`
+ * :setting:`GCS_PROJECT_ID`
+
 .. _topics-feed-storage-stdout:
 
 Standard output
@@ -366,3 +387,4 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 .. _Amazon S3: https://aws.amazon.com/s3/
 .. _boto: https://github.com/boto/boto
 .. _botocore: https://github.com/boto/botocore
+.. _Google Cloud Storage: https://cloud.google.com/storage/
\ No newline at end of file
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 0ac26a9bd..90ae8fd93 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -749,6 +749,14 @@ The Feed Temp dir allows you to set a custom folder to save crawler
 temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
 :ref:`Amazon S3 <topics-feed-storage-s3>`.
 
+.. setting:: FEED_STORAGE_GCS_ACL
+
+FEED_STORAGE_GCS_ACL
+--------------------
+
+The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage <topics-feed-storage-gcs>`.
+For more information on how to set this value, please refer to `Google Cloud documentation <https://cloud.google.com/storage/docs/access-control/lists>`_.
+
 .. setting:: FTP_PASSIVE_MODE
 
 FTP_PASSIVE_MODE
@@ -786,6 +794,15 @@ Default: ``"anonymous"``
 The username to use for FTP connections when there is no ``"ftp_user"``
 in ``Request`` meta.
 
+.. setting:: GCS_PROJECT_ID
+
+GCS_PROJECT_ID
+-----------------
+
+Default: ``None``
+
+The Project ID that will be used when storing data on `Google Cloud Storage`_.
+
 .. setting:: ITEM_PIPELINES
 
 ITEM_PIPELINES
@@ -1371,3 +1388,4 @@ case to see how to enable and use them.
 .. _Amazon web services: https://aws.amazon.com/
 .. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search
 .. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search
+.. _Google Cloud Storage: https://cloud.google.com/storage/
\ No newline at end of file
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 5d2862980..c17e94a64 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -168,6 +168,8 @@ FTP_USER = 'anonymous'
 FTP_PASSWORD = 'guest'
 FTP_PASSIVE_MODE = True
 
+GCS_PROJECT_ID = None
+
 HTTPCACHE_ENABLED = False
 HTTPCACHE_DIR = 'httpcache'
 HTTPCACHE_IGNORE_MISSING = False

From 0bb3d8ca93cf68e0ef231ee734f8a1a3c4075a72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Wed, 27 Feb 2019 18:41:01 -0300
Subject: [PATCH 008/243] Updating Google Cloud Storage scheme to gs instead of
 gcs

---
 docs/topics/feed-exports.rst        | 4 ++--
 scrapy/settings/default_settings.py | 2 +-
 tests/test_feedexport.py            | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index efb63b0ba..0957a5997 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -192,10 +192,10 @@ Google Cloud Storage (GCS)
 
 The feeds are stored on `Google Cloud Storage`_.
 
- * URI scheme: ``gcs``
+ * URI scheme: ``gs``
  * Example URIs:
 
-   * ``gcs://mybucket/path/to/export.csv``
+   * ``gs://mybucket/path/to/export.csv``
 
  * Required external libraries: `google-cloud-storage <https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python>`_.
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index c17e94a64..50fcd1d0a 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -143,7 +143,7 @@ FEED_STORAGES_BASE = {
     '': 'scrapy.extensions.feedexport.FileFeedStorage',
     'file': 'scrapy.extensions.feedexport.FileFeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
-    'gcs': 'scrapy.extensions.feedexport.GCSFeedStorage',
+    'gs': 'scrapy.extensions.feedexport.GCSFeedStorage',
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
 }
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 5cbca6d28..41df7d7af 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -200,7 +200,7 @@ class GCSFeedStorageTest(unittest.TestCase):
 
         settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead' }
         crawler = get_crawler(settings_dict=settings)
-        storage = GCSFeedStorage.from_crawler(crawler, 'gcs://mybucket/export.csv')
+        storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
         assert storage.project_id == '123'
         assert storage.acl == 'publicRead'
         assert storage.bucket_name == 'mybucket'

From 2cb4dc32052c306568206f0997de4b4e53069efd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?=
 <julio.batista@outlook.com>
Date: Fri, 22 Mar 2019 09:50:11 -0300
Subject: [PATCH 009/243] Mentioning to use JSON API for ACLs

---
 docs/topics/settings.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 90ae8fd93..fcdf31cac 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -755,7 +755,7 @@ FEED_STORAGE_GCS_ACL
 --------------------
 
 The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage <topics-feed-storage-gcs>`.
-For more information on how to set this value, please refer to `Google Cloud documentation <https://cloud.google.com/storage/docs/access-control/lists>`_.
+For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation <https://cloud.google.com/storage/docs/access-control/lists>`_.
 
 .. setting:: FTP_PASSIVE_MODE
 
@@ -1388,4 +1388,4 @@ case to see how to enable and use them.
 .. _Amazon web services: https://aws.amazon.com/
 .. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search
 .. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search
-.. _Google Cloud Storage: https://cloud.google.com/storage/
\ No newline at end of file
+.. _Google Cloud Storage: https://cloud.google.com/storage/

From 110bc92e6b9c9c3ce1775a9ca1487df42a80d219 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Thu, 29 Aug 2019 11:10:00 -0300
Subject: [PATCH 010/243] Fix default value of FEED_STORAGE_GCS_ACL

---
 scrapy/extensions/feedexport.py     |  2 +-
 scrapy/settings/default_settings.py |  2 +-
 tests/test_feedexport.py            | 18 ++++++++++++++++++
 3 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index fbbf9bb97..1e982c684 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -160,7 +160,7 @@ class GCSFeedStorage(BlockingFeedStorage):
         return cls(
             uri,
             crawler.settings['GCS_PROJECT_ID'],
-            crawler.settings['FEED_STORAGE_GCS_ACL']
+            crawler.settings['FEED_STORAGE_GCS_ACL'] or None
         )
 
     def _store_in_thread(self, file):
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 50fcd1d0a..45257a61c 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -159,7 +159,7 @@ FEED_EXPORTERS_BASE = {
 }
 FEED_EXPORT_INDENT = 0
 
-FEED_STORAGE_GCS_ACL = None
+FEED_STORAGE_GCS_ACL = ''
 
 FILES_STORE_S3_ACL = 'private'
 FILES_STORE_GCS_ACL = ''
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 41df7d7af..69f144d07 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -206,6 +206,24 @@ class GCSFeedStorageTest(unittest.TestCase):
         assert storage.bucket_name == 'mybucket'
         assert storage.blob_name == 'export.csv'
 
+    @mock.patch('scrapy.conf.settings',
+                new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': '' }, create=True)
+    def test_parse_empty_acl(self):
+        try:
+            from google.cloud.storage import Client
+        except ImportError:
+            raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
+
+        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': '' }
+        crawler = get_crawler(settings_dict=settings)
+        storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
+        assert storage.acl is None
+
+        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': None }
+        crawler = get_crawler(settings_dict=settings)
+        storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
+        assert storage.acl is None
+
     @defer.inlineCallbacks
     def test_store(self):
         try:

From 3b6f7ac9f2f5b48b9f2f3ce106d1205599d2164f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 22 Oct 2019 19:43:02 +0200
Subject: [PATCH 011/243] Use pylint

---
 .travis.yml           |  2 +
 docs/utils/linkfix.py | 85 ++++++++++++++++++++++-------------------
 pylintrc              | 88 +++++++++++++++++++++++++++++++++++++++++++
 tox.ini               | 14 +++++++
 4 files changed, 150 insertions(+), 39 deletions(-)
 create mode 100644 pylintrc

diff --git a/.travis.yml b/.travis.yml
index 0190a7f4d..28a19f4f0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,6 +7,8 @@ branches:
     - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
 matrix:
   include:
+    - env: TOXENV=pylint
+      python: 3.7
     - env: TOXENV=py27
       python: 2.7
     - env: TOXENV=py27-pinned
diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py
index 6290adbe2..9acfc3b23 100755
--- a/docs/utils/linkfix.py
+++ b/docs/utils/linkfix.py
@@ -14,50 +14,57 @@ Author: dufferzafar
 
 import re
 
-# Used for remembering the file (and its contents)
-# so we don't have to open the same file again.
-_filename = None
-_contents = None
 
-# A regex that matches standard linkcheck output lines
-line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
+def main():
 
-# Read lines from the linkcheck output file
-try:
-    with open("build/linkcheck/output.txt") as out:
-        output_lines = out.readlines()
-except IOError:
-    print("linkcheck output not found; please run linkcheck first.")
-    exit(1)
+    # Used for remembering the file (and its contents)
+    # so we don't have to open the same file again.
+    _filename = None
+    _contents = None
 
-# For every line, fix the respective file
-for line in output_lines:
-    match = re.match(line_re, line)
+    # A regex that matches standard linkcheck output lines
+    line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
 
-    if match:
-        newfilename = match.group(1)
-        errortype = match.group(2)
+    # Read lines from the linkcheck output file
+    try:
+        with open("build/linkcheck/output.txt") as out:
+            output_lines = out.readlines()
+    except IOError:
+        print("linkcheck output not found; please run linkcheck first.")
+        exit(1)
 
-        # Broken links can't be fixed and
-        # I am not sure what do with the local ones.
-        if errortype.lower() in ["broken", "local"]:
-            print("Not Fixed: " + line)
+    # For every line, fix the respective file
+    for line in output_lines:
+        match = re.match(line_re, line)
+
+        if match:
+            newfilename = match.group(1)
+            errortype = match.group(2)
+
+            # Broken links can't be fixed and
+            # I am not sure what do with the local ones.
+            if errortype.lower() in ["broken", "local"]:
+                print("Not Fixed: " + line)
+            else:
+                # If this is a new file
+                if newfilename != _filename:
+
+                    # Update the previous file
+                    if _filename:
+                        with open(_filename, "w") as _file:
+                            _file.write(_contents)
+
+                    _filename = newfilename
+
+                    # Read the new file to memory
+                    with open(_filename) as _file:
+                        _contents = _file.read()
+
+                _contents = _contents.replace(match.group(3), match.group(4))
         else:
-            # If this is a new file
-            if newfilename != _filename:
+            # We don't understand what the current line means!
+            print("Not Understood: " + line)
 
-                # Update the previous file
-                if _filename:
-                    with open(_filename, "w") as _file:
-                        _file.write(_contents)
 
-                _filename = newfilename
-
-                # Read the new file to memory
-                with open(_filename) as _file:
-                    _contents = _file.read()
-
-            _contents = _contents.replace(match.group(3), match.group(4))
-    else:
-        # We don't understand what the current line means!
-        print("Not Understood: " + line)
+if __name__ == '__main__':
+    main()
diff --git a/pylintrc b/pylintrc
new file mode 100644
index 000000000..b83bc9f82
--- /dev/null
+++ b/pylintrc
@@ -0,0 +1,88 @@
+[MASTER]
+persistent=no
+jobs=1  # >1 hides results
+
+[MESSAGES CONTROL]
+disable=abstract-method,
+        anomalous-backslash-in-string,
+        arguments-differ,
+        attribute-defined-outside-init,
+        bad-classmethod-argument,
+        bad-continuation,
+        bad-indentation,
+        bad-mcs-classmethod-argument,
+        bad-whitespace,
+        broad-except,
+        c-extension-no-member,
+        catching-non-exception,
+        cell-var-from-loop,
+        comparison-with-callable,
+        consider-using-in,
+        cyclic-import,
+        dangerous-default-value,
+        deprecated-method,
+        deprecated-module,
+        duplicate-code,  # https://github.com/PyCQA/pylint/issues/214
+        eval-used,
+        expression-not-assigned,
+        fixme,
+        function-redefined,
+        global-statement,
+        import-error,
+        import-outside-toplevel,
+        inconsistent-return-statements,
+        inherit-non-class,
+        invalid-name,
+        keyword-arg-before-vararg,
+        line-too-long,
+        logging-format-interpolation,
+        logging-not-lazy,
+        lost-exception,
+        method-hidden,
+        missing-docstring,
+        missing-final-newline,
+        multiple-imports,
+        multiple-statements,
+        no-else-continue,
+        no-else-raise,
+        no-else-return,
+        no-init,
+        no-member,
+        no-method-argument,
+        no-name-in-module,
+        no-self-argument,
+        no-self-use,
+        pointless-string-statement,
+        protected-access,
+        redefined-argument-from-local,
+        redefined-builtin,
+        redefined-outer-name,
+        reimported,
+        signature-differs,
+        super-init-not-called,
+        superfluous-parens,
+        too-few-public-methods,
+        too-many-ancestors,
+        too-many-arguments,
+        too-many-branches,
+        too-many-function-args,
+        too-many-instance-attributes,
+        too-many-locals,
+        too-many-return-statements,
+        trailing-newlines,
+        trailing-whitespace,
+        unexpected-special-method-signature,
+        ungrouped-imports,
+        unidiomatic-typecheck,
+        unnecessary-comprehension,
+        unnecessary-pass,
+        unsubscriptable-object,
+        unused-argument,
+        unused-import,
+        unused-variable,
+        unused-wildcard-import,
+        used-before-assignment,
+        useless-object-inheritance,  # Required for Python 2 support
+        wildcard-import,
+        wrong-import-order,
+        wrong-import-position
diff --git a/tox.ini b/tox.ini
index ffe7360d3..e7d366fe9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -98,6 +98,20 @@ deps = {[testenv:py35]deps}
 commands =
     py.test {posargs:scrapy tests}
 
+[testenv:pylint]
+basepython = python3.7
+deps =
+    {[testenv:py35]deps}
+    # Optional dependencies
+    boto
+    reppy
+    robotexclusionrulesparser
+    # Test dependencies
+    pylint
+
+commands =
+    pylint scrapy
+
 [docs]
 changedir = docs
 deps =

From 02577f55a0586bc3e6c13a4a3ea572c7eefc82b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 24 Oct 2019 13:25:11 +0200
Subject: [PATCH 012/243] Have PyLint cover all Python files in the repository

---
 pylintrc | 19 +++++++++++++++++++
 tox.ini  |  2 +-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/pylintrc b/pylintrc
index b83bc9f82..ca3ea1c57 100644
--- a/pylintrc
+++ b/pylintrc
@@ -11,13 +11,18 @@ disable=abstract-method,
         bad-continuation,
         bad-indentation,
         bad-mcs-classmethod-argument,
+        bad-super-call,
         bad-whitespace,
+        blacklisted-name,
         broad-except,
         c-extension-no-member,
         catching-non-exception,
         cell-var-from-loop,
         comparison-with-callable,
+        consider-iterating-dictionary,
         consider-using-in,
+        consider-using-set-comprehension,
+        consider-using-sys-exit,
         cyclic-import,
         dangerous-default-value,
         deprecated-method,
@@ -30,6 +35,7 @@ disable=abstract-method,
         global-statement,
         import-error,
         import-outside-toplevel,
+        import-self,
         inconsistent-return-statements,
         inherit-non-class,
         invalid-name,
@@ -39,6 +45,7 @@ disable=abstract-method,
         logging-not-lazy,
         lost-exception,
         method-hidden,
+        misplaced-comparison-constant,
         missing-docstring,
         missing-final-newline,
         multiple-imports,
@@ -52,6 +59,9 @@ disable=abstract-method,
         no-name-in-module,
         no-self-argument,
         no-self-use,
+        no-value-for-parameter,
+        not-callable,
+        pointless-statement,
         pointless-string-statement,
         protected-access,
         redefined-argument-from-local,
@@ -59,6 +69,7 @@ disable=abstract-method,
         redefined-outer-name,
         reimported,
         signature-differs,
+        singleton-comparison,
         super-init-not-called,
         superfluous-parens,
         too-few-public-methods,
@@ -67,15 +78,21 @@ disable=abstract-method,
         too-many-branches,
         too-many-function-args,
         too-many-instance-attributes,
+        too-many-lines,
         too-many-locals,
+        too-many-public-methods,
         too-many-return-statements,
         trailing-newlines,
         trailing-whitespace,
+        unbalanced-tuple-unpacking,
+        undefined-variable,
         unexpected-special-method-signature,
         ungrouped-imports,
         unidiomatic-typecheck,
         unnecessary-comprehension,
+        unnecessary-lambda,
         unnecessary-pass,
+        unreachable,
         unsubscriptable-object,
         unused-argument,
         unused-import,
@@ -83,6 +100,8 @@ disable=abstract-method,
         unused-wildcard-import,
         used-before-assignment,
         useless-object-inheritance,  # Required for Python 2 support
+        useless-return,
+        useless-super-delegation,
         wildcard-import,
         wrong-import-order,
         wrong-import-position
diff --git a/tox.ini b/tox.ini
index e7d366fe9..428571ef2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -110,7 +110,7 @@ deps =
     pylint
 
 commands =
-    pylint scrapy
+    pylint conftest.py docs extras scrapy setup.py tests
 
 [docs]
 changedir = docs

From c7f9b955bdf2405fce58907b0395abce2400a66d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 19 Dec 2019 12:44:52 +0100
Subject: [PATCH 013/243] Pylint: ignore not-an-iterable

---
 pylintrc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pylintrc b/pylintrc
index ca3ea1c57..c52a4c2d0 100644
--- a/pylintrc
+++ b/pylintrc
@@ -60,6 +60,7 @@ disable=abstract-method,
         no-self-argument,
         no-self-use,
         no-value-for-parameter,
+        not-an-iterable,
         not-callable,
         pointless-statement,
         pointless-string-statement,

From b9a58798eed39bf543b9682d9ce43b13378a5074 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Sat, 24 May 2014 20:24:01 +0000
Subject: [PATCH 014/243] change Scraper API to call internal `_parse` method

A Spider class using internal pre-processing can have first dibs
at this and then call a public `parse` method for subclass hooking.
---
 scrapy/core/scraper.py     | 2 +-
 scrapy/spiders/__init__.py | 3 +++
 scrapy/spiders/crawl.py    | 2 +-
 scrapy/spiders/feed.py     | 4 ++--
 tests/test_spider.py       | 2 +-
 5 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 99114d3bb..ad6649134 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -145,7 +145,7 @@ class Scraper(object):
     def call_spider(self, result, request, spider):
         result.request = request
         dfd = defer_result(result)
-        callback = request.callback or spider.parse
+        callback = request.callback or spider._parse
         warn_on_generator_with_return_value(spider, callback)
         warn_on_generator_with_return_value(spider, request.errback)
         dfd.addCallbacks(callback=callback,
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 9429f6cb2..1011eb870 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -80,6 +80,9 @@ class Spider(object_ref):
         """ This method is deprecated. """
         return Request(url, dont_filter=True)
 
+    def _parse(self, response):
+        return self.parse(response)
+
     def parse(self, response):
         raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__))
 
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index a2c364c0e..e28d17dcd 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -74,7 +74,7 @@ class CrawlSpider(Spider):
         super(CrawlSpider, self).__init__(*a, **kw)
         self._compile_rules()
 
-    def parse(self, response):
+    def _parse(self, response):
         return self._parse_response(response, self.parse_start_url, cb_kwargs={}, follow=True)
 
     def parse_start_url(self, response):
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index c566f0236..11bd17db4 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -61,7 +61,7 @@ class XMLFeedSpider(Spider):
             for result_item in self.process_results(response, ret):
                 yield result_item
 
-    def parse(self, response):
+    def _parse(self, response):
         if not hasattr(self, 'parse_node'):
             raise NotConfigured('You must define parse_node method in order to scrape this XML feed')
 
@@ -128,7 +128,7 @@ class CSVFeedSpider(Spider):
             for result_item in self.process_results(response, ret):
                 yield result_item
 
-    def parse(self, response):
+    def _parse(self, response):
         if not hasattr(self, 'parse_row'):
             raise NotConfigured('You must define parse_row method in order to scrape this CSV feed')
         response = self.adapt_response(response)
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 317a27076..6fbec7e58 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -142,7 +142,7 @@ class XMLFeedSpiderTest(SpiderTest):
 
         for iterator in ('iternodes', 'xml'):
             spider = _XMLSpider('example', iterator=iterator)
-            output = list(spider.parse(response))
+            output = list(spider._parse(response))
             self.assertEqual(len(output), 2, iterator)
             self.assertEqual(output, [
                 {'loc': [u'http://www.example.com/Special-Offers.html'],

From 5982e3477c732f4dff5accea6eab486e4ab52c3e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Dec 2019 14:12:21 -0300
Subject: [PATCH 015/243] Take keyword arguments in base parsing methods

---
 docs/topics/spiders.rst    | 12 +++++++-----
 scrapy/spiders/__init__.py |  6 +++---
 scrapy/spiders/crawl.py    | 11 ++++++++---
 scrapy/spiders/feed.py     |  4 ++--
 4 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index b0fb14e24..dd763b607 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -362,12 +362,14 @@ CrawlSpider
 
    This spider also exposes an overrideable method:
 
-   .. method:: parse_start_url(response)
+   .. method:: parse_start_url(response, **kwargs)
 
-      This method is called for the start_urls responses. It allows to parse
-      the initial responses and must return either an
-      :class:`~scrapy.item.Item` object, a :class:`~scrapy.http.Request`
-      object, or an iterable containing any of them.
+      This method is called for each response produced for the URLs in
+      the spider's ``start_urls`` attribute. It allows to parse
+      the initial responses and must return either an item
+      (:class:`scrapy.item.Item` or :class:`dict`),
+      a :class:`~scrapy.http.Request`,
+      or an iterable containing any of them.
 
 Crawling rules
 ~~~~~~~~~~~~~~
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 1011eb870..3e19f1e23 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -80,10 +80,10 @@ class Spider(object_ref):
         """ This method is deprecated. """
         return Request(url, dont_filter=True)
 
-    def _parse(self, response):
-        return self.parse(response)
+    def _parse(self, response, **kwargs):
+        return self.parse(response, **kwargs)
 
-    def parse(self, response):
+    def parse(self, response, **kwargs):
         raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__))
 
     @classmethod
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index e28d17dcd..4ec0de78c 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -74,10 +74,15 @@ class CrawlSpider(Spider):
         super(CrawlSpider, self).__init__(*a, **kw)
         self._compile_rules()
 
-    def _parse(self, response):
-        return self._parse_response(response, self.parse_start_url, cb_kwargs={}, follow=True)
+    def _parse(self, response, **kwargs):
+        return self._parse_response(
+            response=response,
+            callback=self.parse_start_url,
+            cb_kwargs=kwargs,
+            follow=True,
+        )
 
-    def parse_start_url(self, response):
+    def parse_start_url(self, response, **kwargs):
         return []
 
     def process_results(self, response, results):
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index 11bd17db4..4fa6009a5 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -61,7 +61,7 @@ class XMLFeedSpider(Spider):
             for result_item in self.process_results(response, ret):
                 yield result_item
 
-    def _parse(self, response):
+    def _parse(self, response, **kwargs):
         if not hasattr(self, 'parse_node'):
             raise NotConfigured('You must define parse_node method in order to scrape this XML feed')
 
@@ -128,7 +128,7 @@ class CSVFeedSpider(Spider):
             for result_item in self.process_results(response, ret):
                 yield result_item
 
-    def _parse(self, response):
+    def _parse(self, response, **kwargs):
         if not hasattr(self, 'parse_row'):
             raise NotConfigured('You must define parse_row method in order to scrape this CSV feed')
         response = self.adapt_response(response)

From 8d4948f6ca44a76ee7714c8b4b1c46ef73a8845e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 26 Dec 2019 14:38:11 -0300
Subject: [PATCH 016/243] [test] Override CrawlSpider.parse

---
 tests/spiders.py    | 35 +++++++++++++++++++++++++++++------
 tests/test_crawl.py | 31 +++++++++++++++++++++++++++----
 2 files changed, 56 insertions(+), 10 deletions(-)

diff --git a/tests/spiders.py b/tests/spiders.py
index 39c8da0b6..dcc475ca7 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -186,13 +186,39 @@ class DuplicateStartRequestsSpider(MockServerSpider):
         self.visited += 1
 
 
-class CrawlSpiderWithErrback(MockServerSpider, CrawlSpider):
-    name = 'crawl_spider_with_errback'
+class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
+    """
+    A CrawlSpider which overrides the 'parse' method
+    """
+    name = 'crawl_spider_with_parse_method'
     custom_settings = {
         'RETRY_HTTP_CODES': [],  # no need to retry
     }
     rules = (
-        Rule(LinkExtractor(), callback='callback', errback='errback', follow=True),
+        Rule(LinkExtractor(), callback='parse', follow=True),
+    )
+
+    def start_requests(self):
+        test_body = b"""
+        <html>
+            <head><title>Page title<title></head>
+            <body>
+                <p><a href="/status?n=200">Item 200</a></p>  <!-- callback -->
+                <p><a href="/status?n=201">Item 201</a></p>  <!-- callback -->
+            </body>
+        </html>
+        """
+        url = self.mockserver.url("/alpayload")
+        yield Request(url, method="POST", body=test_body)
+
+    def parse(self, response):
+        self.logger.info('[parse] status %i', response.status)
+
+
+class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
+    name = 'crawl_spider_with_errback'
+    rules = (
+        Rule(LinkExtractor(), callback='parse', errback='errback', follow=True),
     )
 
     def start_requests(self):
@@ -211,8 +237,5 @@ class CrawlSpiderWithErrback(MockServerSpider, CrawlSpider):
         url = self.mockserver.url("/alpayload")
         yield Request(url, method="POST", body=test_body)
 
-    def callback(self, response):
-        self.logger.info('[callback] status %i', response.status)
-
     def errback(self, failure):
         self.logger.info('[errback] status %i', failure.value.response.status)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index f433fcea6..4299e4bbb 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -9,8 +9,9 @@ from scrapy.crawler import CrawlerRunner
 from scrapy.http import Request
 from scrapy.utils.python import to_unicode
 from tests.mockserver import MockServer
-from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider,
-                           SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback)
+from tests.spiders import (BrokenStartRequestsSpider, CrawlSpiderWithErrback,
+                           CrawlSpiderWithParseMethod, DelaySpider, SimpleSpider,
+                           DuplicateStartRequestsSpider, FollowAllSpider, SingleRequestSpider)
 
 
 class CrawlTestCase(TestCase):
@@ -297,6 +298,27 @@ with multiples lines
         self._assert_retried(log)
         self.assertIn("Got response 200", str(log))
 
+
+class CrawlSpiderTestCase(TestCase):
+
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+        self.runner = CrawlerRunner()
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+
+    @defer.inlineCallbacks
+    def test_crawlspider_with_parse(self):
+        self.runner.crawl(CrawlSpiderWithParseMethod, mockserver=self.mockserver)
+
+        with LogCapture() as log:
+            yield self.runner.join()
+
+        self.assertIn("[parse] status 200", str(log))
+        self.assertIn("[parse] status 201", str(log))
+
     @defer.inlineCallbacks
     def test_crawlspider_with_errback(self):
         self.runner.crawl(CrawlSpiderWithErrback, mockserver=self.mockserver)
@@ -304,7 +326,8 @@ with multiples lines
         with LogCapture() as log:
             yield self.runner.join()
 
-        self.assertIn("[callback] status 200", str(log))
-        self.assertIn("[callback] status 201", str(log))
+        self.assertIn("[parse] status 200", str(log))
+        self.assertIn("[parse] status 201", str(log))
         self.assertIn("[errback] status 404", str(log))
         self.assertIn("[errback] status 500", str(log))
+        self.assertIn("[errback] status 501", str(log))

From c54df8253a67bb6863a25bf7d667096adc73a040 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 26 Dec 2019 15:12:19 -0300
Subject: [PATCH 017/243] [test] Handle keyword args in CrawlSpider.parse

---
 tests/spiders.py    |  5 +++--
 tests/test_crawl.py | 10 ++++++----
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/tests/spiders.py b/tests/spiders.py
index dcc475ca7..c042eb7fe 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -211,8 +211,9 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
         url = self.mockserver.url("/alpayload")
         yield Request(url, method="POST", body=test_body)
 
-    def parse(self, response):
-        self.logger.info('[parse] status %i', response.status)
+    def parse(self, response, foo=None):
+        self.logger.info('[parse] status %i (foo: %s)', response.status, foo)
+        yield Request(self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"})
 
 
 class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 4299e4bbb..6247ced35 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -316,8 +316,9 @@ class CrawlSpiderTestCase(TestCase):
         with LogCapture() as log:
             yield self.runner.join()
 
-        self.assertIn("[parse] status 200", str(log))
-        self.assertIn("[parse] status 201", str(log))
+        self.assertIn("[parse] status 200 (foo: None)", str(log))
+        self.assertIn("[parse] status 201 (foo: None)", str(log))
+        self.assertIn("[parse] status 202 (foo: bar)", str(log))
 
     @defer.inlineCallbacks
     def test_crawlspider_with_errback(self):
@@ -326,8 +327,9 @@ class CrawlSpiderTestCase(TestCase):
         with LogCapture() as log:
             yield self.runner.join()
 
-        self.assertIn("[parse] status 200", str(log))
-        self.assertIn("[parse] status 201", str(log))
+        self.assertIn("[parse] status 200 (foo: None)", str(log))
+        self.assertIn("[parse] status 201 (foo: None)", str(log))
+        self.assertIn("[parse] status 202 (foo: bar)", str(log))
         self.assertIn("[errback] status 404", str(log))
         self.assertIn("[errback] status 500", str(log))
         self.assertIn("[errback] status 501", str(log))

From 8a1dc26d4662ec0c81d22c13cffa44255a7c29ca Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 26 Dec 2019 15:14:47 -0300
Subject: [PATCH 018/243] [doc] Note about the 'parse' method for
 CrawlSpider/XMLFeedSpider

---
 docs/topics/spiders.rst | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index dd763b607..406f50fb3 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -392,11 +392,6 @@ Crawling rules
    object will contain the text of the link that produced the :class:`~scrapy.http.Request`
    in its ``meta`` dictionary (under the ``link_text`` key)
 
-   .. warning:: When writing crawl spider rules, avoid using ``parse`` as
-       callback, since the :class:`CrawlSpider` uses the ``parse`` method
-       itself to implement its logic. So if you override the ``parse`` method,
-       the crawl spider will no longer work.
-
    ``cb_kwargs`` is a dict containing the keyword arguments to be passed to the
    callback function.
 
@@ -422,6 +417,12 @@ Crawling rules
    It receives a :class:`Twisted Failure <twisted.python.failure.Failure>`
    instance as first parameter.
 
+
+.. warning:: Because of its internal implementation, you must explicitly set
+   callbacks for new requests when writing :class:`CrawlSpider`-based spiders;
+   unexpected behaviour can occur otherwise.
+
+
 CrawlSpider example
 ~~~~~~~~~~~~~~~~~~~
 
@@ -452,6 +453,11 @@ Let's now take a look at an example CrawlSpider with rules::
             item['name'] = response.xpath('//td[@id="item_name"]/text()').get()
             item['description'] = response.xpath('//td[@id="item_description"]/text()').get()
             item['link_text'] = response.meta['link_text']
+            url = response.xpath('//td[@id="additional_data"]/@href').get()
+            return response.follow(url, self.parse_additional_page, cb_kwargs=dict(item=item))
+
+        def parse_additional_page(self, response, item):
+            item['additional_data'] = response.xpath('//p[@id="additional_data"]/text()').get()
             return item
 
 
@@ -545,6 +551,11 @@ XMLFeedSpider
         those results. It must return a list of results (Items or Requests).
 
 
+.. warning:: Because of its internal implementation, you must explicitly set
+   callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders;
+   unexpected behaviour can occur otherwise.
+
+
 XMLFeedSpider example
 ~~~~~~~~~~~~~~~~~~~~~
 

From 72b8613ee9827af031862bd84f1bea9acefcbebe Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 27 Nov 2019 14:46:20 -0300
Subject: [PATCH 019/243] bytes_received signal (no tests)

---
 docs/topics/signals.rst                   | 45 ++++++++++++++++-------
 scrapy/core/downloader/handlers/http11.py | 25 +++++++++++--
 scrapy/signals.py                         |  1 +
 3 files changed, 54 insertions(+), 17 deletions(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 3f29aa323..6efb73abb 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -73,7 +73,7 @@ engine_started
 
     Sent when the Scrapy engine has started crawling.
 
-    This signal supports returning deferreds from their handlers.
+    This signal supports returning deferreds from its handlers.
 
 .. note:: This signal may be fired *after* the :signal:`spider_opened` signal,
     depending on how the spider was started. So **don't** rely on this signal
@@ -88,7 +88,7 @@ engine_stopped
     Sent when the Scrapy engine is stopped (for example, when a crawling
     process has finished).
 
-    This signal supports returning deferreds from their handlers.
+    This signal supports returning deferreds from its handlers.
 
 item_scraped
 ------------
@@ -99,7 +99,7 @@ item_scraped
     Sent when an item has been scraped, after it has passed all the
     :ref:`topics-item-pipeline` stages (without being dropped).
 
-    This signal supports returning deferreds from their handlers.
+    This signal supports returning deferreds from its handlers.
 
     :param item: the item scraped
     :type item: dict or :class:`~scrapy.item.Item` object
@@ -119,7 +119,7 @@ item_dropped
     Sent after an item has been dropped from the :ref:`topics-item-pipeline`
     when some stage raised a :exc:`~scrapy.exceptions.DropItem` exception.
 
-    This signal supports returning deferreds from their handlers.
+    This signal supports returning deferreds from its handlers.
 
     :param item: the item dropped from the :ref:`topics-item-pipeline`
     :type item: dict or :class:`~scrapy.item.Item` object
@@ -144,7 +144,7 @@ item_error
     Sent when a :ref:`topics-item-pipeline` generates an error (ie. raises
     an exception), except :exc:`~scrapy.exceptions.DropItem` exception.
 
-    This signal supports returning deferreds from their handlers.
+    This signal supports returning deferreds from its handlers.
 
     :param item: the item dropped from the :ref:`topics-item-pipeline`
     :type item: dict or :class:`~scrapy.item.Item` object
@@ -158,6 +158,23 @@ item_error
     :param failure: the exception raised
     :type failure: twisted.python.failure.Failure
 
+bytes_received
+--------------
+
+.. signal:: bytes_received
+.. function:: bytes_received(data, request)
+
+    Sent by the HTTP 1.1 download handler when a group of bytes is
+    received for a specific request.
+
+    This signal does not support returning deferreds from its handlers.
+
+    :param data: the data received by the download handler
+    :type spider: :class:`bytes` object
+
+    :param request: the request that generated the response
+    :type request: :class:`~scrapy.http.Request` object
+
 spider_closed
 -------------
 
@@ -167,7 +184,7 @@ spider_closed
     Sent after a spider has been closed. This can be used to release per-spider
     resources reserved on :signal:`spider_opened`.
 
-    This signal supports returning deferreds from their handlers.
+    This signal supports returning deferreds from its handlers.
 
     :param spider: the spider which has been closed
     :type spider: :class:`~scrapy.spiders.Spider` object
@@ -191,7 +208,7 @@ spider_opened
     reserve per-spider resources, but can be used for any task that needs to be
     performed when a spider is opened.
 
-    This signal supports returning deferreds from their handlers.
+    This signal supports returning deferreds from its handlers.
 
     :param spider: the spider which has been opened
     :type spider: :class:`~scrapy.spiders.Spider` object
@@ -215,7 +232,7 @@ spider_idle
     You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to
     prevent the spider from being closed.
 
-    This signal does not support returning deferreds from their handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param spider: the spider which has gone idle
     :type spider: :class:`~scrapy.spiders.Spider` object
@@ -234,7 +251,7 @@ spider_error
 
     Sent when a spider callback generates an error (ie. raises an exception).
 
-    This signal does not support returning deferreds from their handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param failure: the exception raised
     :type failure: twisted.python.failure.Failure
@@ -254,7 +271,7 @@ request_scheduled
     Sent when the engine schedules a :class:`~scrapy.http.Request`, to be
     downloaded later.
 
-    The signal does not support returning deferreds from their handlers.
+    The signal does not support returning deferreds from its handlers.
 
     :param request: the request that reached the scheduler
     :type request: :class:`~scrapy.http.Request` object
@@ -271,7 +288,7 @@ request_dropped
     Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be
     downloaded later, is rejected by the scheduler.
 
-    The signal does not support returning deferreds from their handlers.
+    The signal does not support returning deferreds from its handlers.
 
     :param request: the request that reached the scheduler
     :type request: :class:`~scrapy.http.Request` object
@@ -287,7 +304,7 @@ request_reached_downloader
 
     Sent when a :class:`~scrapy.http.Request` reached downloader.
 
-    The signal does not support returning deferreds from their handlers.
+    The signal does not support returning deferreds from its handlers.
 
     :param request: the request that reached downloader
     :type request: :class:`~scrapy.http.Request` object
@@ -304,7 +321,7 @@ response_received
     Sent when the engine receives a new :class:`~scrapy.http.Response` from the
     downloader.
 
-    This signal does not support returning deferreds from their handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param response: the response received
     :type response: :class:`~scrapy.http.Response` object
@@ -323,7 +340,7 @@ response_downloaded
 
     Sent by the downloader right after a ``HTTPResponse`` is downloaded.
 
-    This signal does not support returning deferreds from their handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param response: the response downloaded
     :type response: :class:`~scrapy.http.Response` object
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 5a5f6cf0a..92c3d5f5c 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -16,6 +16,7 @@ from twisted.web.http_headers import Headers as TxHeaders
 from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
 from zope.interface import implementer
 
+from scrapy import signals
 from scrapy.core.downloader.tls import openssl_methods
 from scrapy.core.downloader.webclient import _parse
 from scrapy.exceptions import ScrapyDeprecationWarning
@@ -32,6 +33,7 @@ class HTTP11DownloadHandler:
     lazy = False
 
     def __init__(self, settings, crawler=None):
+        self.crawler = crawler
         self._pool = HTTPConnectionPool(reactor, persistent=True)
         self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
         self._pool._factory.noisy = False
@@ -76,6 +78,7 @@ class HTTP11DownloadHandler:
             maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
             warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
             fail_on_dataloss=self._fail_on_dataloss,
+            crawler=self.crawler,
         )
         return agent.download_request(request)
 
@@ -272,7 +275,7 @@ class ScrapyAgent(object):
     _TunnelingAgent = TunnelingAgent
 
     def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None,
-                 maxsize=0, warnsize=0, fail_on_dataloss=True):
+                 maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None):
         self._contextFactory = contextFactory
         self._connectTimeout = connectTimeout
         self._bindAddress = bindAddress
@@ -281,6 +284,7 @@ class ScrapyAgent(object):
         self._warnsize = warnsize
         self._fail_on_dataloss = fail_on_dataloss
         self._txresponse = None
+        self._crawler = crawler
 
     def _get_agent(self, request, timeout):
         bindaddress = request.meta.get('bindaddress') or self._bindAddress
@@ -409,7 +413,15 @@ class ScrapyAgent(object):
 
         d = defer.Deferred(_cancel)
         txresponse.deliverBody(
-            _ResponseReader(d, txresponse, request, maxsize, warnsize, fail_on_dataloss)
+            _ResponseReader(
+                d,
+                txresponse,
+                request,
+                maxsize,
+                warnsize,
+                fail_on_dataloss,
+                self._crawler,
+            )
         )
 
         # save response for timeouts
@@ -445,7 +457,7 @@ class _RequestBodyProducer(object):
 
 class _ResponseReader(protocol.Protocol):
 
-    def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss):
+    def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler):
         self._finished = finished
         self._txresponse = txresponse
         self._request = request
@@ -456,6 +468,7 @@ class _ResponseReader(protocol.Protocol):
         self._fail_on_dataloss_warned = False
         self._reached_warnsize = False
         self._bytes_received = 0
+        self._crawler = crawler
 
     def dataReceived(self, bodyBytes):
         # This maybe called several times after cancel was called with buffered data.
@@ -465,6 +478,12 @@ class _ResponseReader(protocol.Protocol):
         self._bodybuf.write(bodyBytes)
         self._bytes_received += len(bodyBytes)
 
+        self._crawler.signals.send_catch_log(
+            signal=signals.bytes_received,
+            data=bodyBytes,
+            request=self._request,
+        )
+
         if self._maxsize and self._bytes_received > self._maxsize:
             logger.error("Received (%(bytes)s) bytes larger than download "
                          "max size (%(maxsize)s) in request %(request)s.",
diff --git a/scrapy/signals.py b/scrapy/signals.py
index 6b9125302..590421893 100644
--- a/scrapy/signals.py
+++ b/scrapy/signals.py
@@ -16,6 +16,7 @@ request_dropped = object()
 request_reached_downloader = object()
 response_received = object()
 response_downloaded = object()
+bytes_received = object()
 item_scraped = object()
 item_dropped = object()
 item_error = object()

From cab449b1952020b86fbe2915a537150fc885c567 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 29 Nov 2019 11:37:40 -0300
Subject: [PATCH 020/243] Typo fix

---
 tests/test_engine.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index 25dee7c1f..9d68836cc 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -107,7 +107,7 @@ class CrawlerRun(object):
         self.reqreached = []
         self.itemerror = []
         self.itemresp = []
-        self.signals_catched = {}
+        self.signals_caught = {}
         self.spider_class = spider_class
 
     def run(self):
@@ -172,7 +172,7 @@ class CrawlerRun(object):
         signalargs = kwargs.copy()
         sig = signalargs.pop('signal')
         signalargs.pop('sender', None)
-        self.signals_catched[sig] = signalargs
+        self.signals_caught[sig] = signalargs
 
 
 class EngineTest(unittest.TestCase):
@@ -186,7 +186,7 @@ class EngineTest(unittest.TestCase):
             self._assert_scheduled_requests(urls_to_visit=8)
             self._assert_downloaded_responses()
             self._assert_scraped_items()
-            self._assert_signals_catched()
+            self._assert_signals_caught()
 
     @defer.inlineCallbacks
     def test_crawler_dupefilter(self):
@@ -263,19 +263,19 @@ class EngineTest(unittest.TestCase):
                 self.assertEqual('Item 2 name', item['name'])
                 self.assertEqual('200', item['price'])
 
-    def _assert_signals_catched(self):
-        assert signals.engine_started in self.run.signals_catched
-        assert signals.engine_stopped in self.run.signals_catched
-        assert signals.spider_opened in self.run.signals_catched
-        assert signals.spider_idle in self.run.signals_catched
-        assert signals.spider_closed in self.run.signals_catched
+    def _assert_signals_caught(self):
+        assert signals.engine_started in self.run.signals_caught
+        assert signals.engine_stopped in self.run.signals_caught
+        assert signals.spider_opened in self.run.signals_caught
+        assert signals.spider_idle in self.run.signals_caught
+        assert signals.spider_closed in self.run.signals_caught
 
         self.assertEqual({'spider': self.run.spider},
-                         self.run.signals_catched[signals.spider_opened])
+                         self.run.signals_caught[signals.spider_opened])
         self.assertEqual({'spider': self.run.spider},
-                         self.run.signals_catched[signals.spider_idle])
+                         self.run.signals_caught[signals.spider_idle])
         self.assertEqual({'spider': self.run.spider, 'reason': 'finished'},
-                         self.run.signals_catched[signals.spider_closed])
+                         self.run.signals_caught[signals.spider_closed])
 
     @defer.inlineCallbacks
     def test_close_downloader(self):

From bda37e38bd53d5aae691b56d4136fbff99f78158 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 29 Nov 2019 12:02:27 -0300
Subject: [PATCH 021/243] [Tests] bytes_received signal

---
 tests/test_engine.py | 53 +++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 47 insertions(+), 6 deletions(-)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index 9d68836cc..b63c7e232 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -13,22 +13,24 @@ module with the ``runserver`` argument::
 import os
 import re
 import sys
+from collections import defaultdict
 from urllib.parse import urlparse
 
 from twisted.internet import reactor, defer
-from twisted.web import server, static, util
 from twisted.trial import unittest
+from twisted.web import server, static, util
+from pydispatch import dispatcher
 
 from scrapy import signals
 from scrapy.core.engine import ExecutionEngine
-from scrapy.utils.test import get_crawler
-from pydispatch import dispatcher
-from tests import tests_datadir
-from scrapy.spiders import Spider
+from scrapy.http import Request
 from scrapy.item import Item, Field
 from scrapy.linkextractors import LinkExtractor
-from scrapy.http import Request
+from scrapy.spiders import Spider
 from scrapy.utils.signal import disconnect_all
+from scrapy.utils.test import get_crawler
+
+from tests import tests_datadir, get_testdata
 
 
 class TestItem(Item):
@@ -107,6 +109,7 @@ class CrawlerRun(object):
         self.reqreached = []
         self.itemerror = []
         self.itemresp = []
+        self.bytes = defaultdict(lambda: b"")
         self.signals_caught = {}
         self.spider_class = spider_class
 
@@ -124,6 +127,7 @@ class CrawlerRun(object):
         self.crawler = get_crawler(self.spider_class)
         self.crawler.signals.connect(self.item_scraped, signals.item_scraped)
         self.crawler.signals.connect(self.item_error, signals.item_error)
+        self.crawler.signals.connect(self.bytes_received, signals.bytes_received)
         self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
         self.crawler.signals.connect(self.request_dropped, signals.request_dropped)
         self.crawler.signals.connect(self.request_reached, signals.request_reached_downloader)
@@ -155,6 +159,9 @@ class CrawlerRun(object):
     def item_scraped(self, item, spider, response):
         self.itemresp.append((item, response))
 
+    def bytes_received(self, data, request):
+        self.bytes[request] += data
+
     def request_scheduled(self, request, spider):
         self.reqplug.append((request, spider))
 
@@ -187,6 +194,7 @@ class EngineTest(unittest.TestCase):
             self._assert_downloaded_responses()
             self._assert_scraped_items()
             self._assert_signals_caught()
+            self._assert_bytes_received()
 
     @defer.inlineCallbacks
     def test_crawler_dupefilter(self):
@@ -263,6 +271,39 @@ class EngineTest(unittest.TestCase):
                 self.assertEqual('Item 2 name', item['name'])
                 self.assertEqual('200', item['price'])
 
+    def _assert_bytes_received(self):
+        self.assertEqual(8, len(self.run.bytes))
+        for request, data in self.run.bytes.items():
+            if self.run.getpath(request.url) == "/":
+                self.assertEqual(data, get_testdata("test_site", "index.html"))
+            elif self.run.getpath(request.url) == "/item1.html":
+                self.assertEqual(data, get_testdata("test_site", "item1.html"))
+            elif self.run.getpath(request.url) == "/item2.html":
+                self.assertEqual(data, get_testdata("test_site", "item2.html"))
+            elif self.run.getpath(request.url) == "/redirected":
+                self.assertEqual(data, b"Redirected here")
+            elif self.run.getpath(request.url) == '/redirect':
+                self.assertEqual(data,
+                    b"\n<html>\n"
+                    b"    <head>\n"
+                    b"        <meta http-equiv=\"refresh\" content=\"0;URL=/redirected\">\n"
+                    b"    </head>\n"
+                    b"    <body bgcolor=\"#FFFFFF\" text=\"#000000\">\n"
+                    b"    <a href=\"/redirected\">click here</a>\n"
+                    b"    </body>\n"
+                    b"</html>\n"
+                )
+            elif self.run.getpath(request.url) == "/tem999.html":
+                self.assertEqual(data,
+                    b"\n<html>\n"
+                    b"  <head><title>404 - No Such Resource</title></head>\n"
+                    b"  <body>\n"
+                    b"    <h1>No Such Resource</h1>\n"
+                    b"    <p>File not found.</p>\n"
+                    b"  </body>\n"
+                    b"</html>\n"
+                )
+
     def _assert_signals_caught(self):
         assert signals.engine_started in self.run.signals_caught
         assert signals.engine_stopped in self.run.signals_caught

From 89483ce9f709e230ee5ff9050d206430d2d17c9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 3 Dec 2019 12:06:08 +0100
Subject: [PATCH 022/243] Fix Flake8 issues

---
 tests/test_engine.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index b63c7e232..c0769c992 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -283,7 +283,8 @@ class EngineTest(unittest.TestCase):
             elif self.run.getpath(request.url) == "/redirected":
                 self.assertEqual(data, b"Redirected here")
             elif self.run.getpath(request.url) == '/redirect':
-                self.assertEqual(data,
+                self.assertEqual(
+                    data,
                     b"\n<html>\n"
                     b"    <head>\n"
                     b"        <meta http-equiv=\"refresh\" content=\"0;URL=/redirected\">\n"
@@ -294,7 +295,8 @@ class EngineTest(unittest.TestCase):
                     b"</html>\n"
                 )
             elif self.run.getpath(request.url) == "/tem999.html":
-                self.assertEqual(data,
+                self.assertEqual(
+                    data,
                     b"\n<html>\n"
                     b"  <head><title>404 - No Such Resource</title></head>\n"
                     b"  <body>\n"

From dbe20a863ff63dce937b2d3b159782d8268e6838 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 27 Jan 2020 12:21:18 -0300
Subject: [PATCH 023/243] bytes_received signal: send spider argument

---
 docs/topics/signals.rst                   | 5 ++++-
 scrapy/core/downloader/handlers/http11.py | 1 +
 tests/test_engine.py                      | 2 +-
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 6efb73abb..3e70ca067 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -162,7 +162,7 @@ bytes_received
 --------------
 
 .. signal:: bytes_received
-.. function:: bytes_received(data, request)
+.. function:: bytes_received(data, request, spider)
 
     Sent by the HTTP 1.1 download handler when a group of bytes is
     received for a specific request.
@@ -175,6 +175,9 @@ bytes_received
     :param request: the request that generated the response
     :type request: :class:`~scrapy.http.Request` object
 
+    :param spider: the spider associated with the response
+    :type spider: :class:`~scrapy.spiders.Spider` object
+
 spider_closed
 -------------
 
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 92c3d5f5c..c53c9bb2d 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -482,6 +482,7 @@ class _ResponseReader(protocol.Protocol):
             signal=signals.bytes_received,
             data=bodyBytes,
             request=self._request,
+            spider=self._crawler.spider,
         )
 
         if self._maxsize and self._bytes_received > self._maxsize:
diff --git a/tests/test_engine.py b/tests/test_engine.py
index c0769c992..57cc89ba3 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -159,7 +159,7 @@ class CrawlerRun(object):
     def item_scraped(self, item, spider, response):
         self.itemresp.append((item, response))
 
-    def bytes_received(self, data, request):
+    def bytes_received(self, data, request, spider):
         self.bytes[request] += data
 
     def request_scheduled(self, request, spider):

From 613fd41f44d1455f9c9369087958674f3fdfcc8d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 27 Jan 2020 12:30:26 -0300
Subject: [PATCH 024/243] bytes_received signal: improve test performance

---
 tests/test_engine.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index 57cc89ba3..bb475958e 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -109,7 +109,7 @@ class CrawlerRun(object):
         self.reqreached = []
         self.itemerror = []
         self.itemresp = []
-        self.bytes = defaultdict(lambda: b"")
+        self.bytes = defaultdict(lambda: list())
         self.signals_caught = {}
         self.spider_class = spider_class
 
@@ -160,7 +160,7 @@ class CrawlerRun(object):
         self.itemresp.append((item, response))
 
     def bytes_received(self, data, request, spider):
-        self.bytes[request] += data
+        self.bytes[request].append(data)
 
     def request_scheduled(self, request, spider):
         self.reqplug.append((request, spider))
@@ -274,17 +274,18 @@ class EngineTest(unittest.TestCase):
     def _assert_bytes_received(self):
         self.assertEqual(8, len(self.run.bytes))
         for request, data in self.run.bytes.items():
+            joined_data = b"".join(data)
             if self.run.getpath(request.url) == "/":
-                self.assertEqual(data, get_testdata("test_site", "index.html"))
+                self.assertEqual(joined_data, get_testdata("test_site", "index.html"))
             elif self.run.getpath(request.url) == "/item1.html":
-                self.assertEqual(data, get_testdata("test_site", "item1.html"))
+                self.assertEqual(joined_data, get_testdata("test_site", "item1.html"))
             elif self.run.getpath(request.url) == "/item2.html":
-                self.assertEqual(data, get_testdata("test_site", "item2.html"))
+                self.assertEqual(joined_data, get_testdata("test_site", "item2.html"))
             elif self.run.getpath(request.url) == "/redirected":
-                self.assertEqual(data, b"Redirected here")
+                self.assertEqual(joined_data, b"Redirected here")
             elif self.run.getpath(request.url) == '/redirect':
                 self.assertEqual(
-                    data,
+                    joined_data,
                     b"\n<html>\n"
                     b"    <head>\n"
                     b"        <meta http-equiv=\"refresh\" content=\"0;URL=/redirected\">\n"
@@ -296,7 +297,7 @@ class EngineTest(unittest.TestCase):
                 )
             elif self.run.getpath(request.url) == "/tem999.html":
                 self.assertEqual(
-                    data,
+                    joined_data,
                     b"\n<html>\n"
                     b"  <head><title>404 - No Such Resource</title></head>\n"
                     b"  <body>\n"

From 4ffd18fb11ff89863569b8b4de44241e3ca2f86e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 27 Jan 2020 13:29:33 -0300
Subject: [PATCH 025/243] [docs] Mention that signals.bytes_received could be
 fired multiple times

---
 docs/topics/signals.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 3e70ca067..f490911f3 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -165,7 +165,8 @@ bytes_received
 .. function:: bytes_received(data, request, spider)
 
     Sent by the HTTP 1.1 download handler when a group of bytes is
-    received for a specific request.
+    received for a specific request. This signal might be fired
+    multiple times for the same request.
 
     This signal does not support returning deferreds from its handlers.
 

From 2c9643d38cc076c4d2032efd994fda4cfcc9f88a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 29 Jan 2020 14:11:56 -0300
Subject: [PATCH 026/243] Test: bytes_received signal fired multiple times

---
 tests/test_engine.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index bb475958e..3c5cc403b 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -12,6 +12,7 @@ module with the ``runserver`` argument::
 
 import os
 import re
+import string
 import sys
 from collections import defaultdict
 from urllib.parse import urlparse
@@ -90,6 +91,7 @@ def start_test_site(debug=False):
     r = static.File(root_dir)
     r.putChild(b"redirect", util.Redirect(b"/redirected"))
     r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
+    r.putChild(b"random", static.Data(string.ascii_letters.encode("utf8") * 2**14, "text/plain"))
 
     port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1")
     if debug:
@@ -117,8 +119,12 @@ class CrawlerRun(object):
         self.port = start_test_site()
         self.portno = self.port.getHost().port
 
-        start_urls = [self.geturl("/"), self.geturl("/redirect"),
-                      self.geturl("/redirect")]  # a duplicate
+        start_urls = [
+            self.geturl("/"),
+            self.geturl("/redirect"),
+            self.geturl("/redirect"),  # duplicate
+            self.geturl("/random"),
+        ]
 
         for name, signal in vars(signals).items():
             if not name.startswith('_'):
@@ -190,7 +196,7 @@ class EngineTest(unittest.TestCase):
             self.run = CrawlerRun(spider)
             yield self.run.run()
             self._assert_visited_urls()
-            self._assert_scheduled_requests(urls_to_visit=8)
+            self._assert_scheduled_requests(urls_to_visit=9)
             self._assert_downloaded_responses()
             self._assert_scraped_items()
             self._assert_signals_caught()
@@ -200,7 +206,7 @@ class EngineTest(unittest.TestCase):
     def test_crawler_dupefilter(self):
         self.run = CrawlerRun(TestDupeFilterSpider)
         yield self.run.run()
-        self._assert_scheduled_requests(urls_to_visit=7)
+        self._assert_scheduled_requests(urls_to_visit=8)
         self._assert_dropped_requests()
 
     @defer.inlineCallbacks
@@ -237,8 +243,8 @@ class EngineTest(unittest.TestCase):
 
     def _assert_downloaded_responses(self):
         # response tests
-        self.assertEqual(8, len(self.run.respplug))
-        self.assertEqual(8, len(self.run.reqreached))
+        self.assertEqual(9, len(self.run.respplug))
+        self.assertEqual(9, len(self.run.reqreached))
 
         for response, _ in self.run.respplug:
             if self.run.getpath(response.url) == '/item999.html':
@@ -272,7 +278,7 @@ class EngineTest(unittest.TestCase):
                 self.assertEqual('200', item['price'])
 
     def _assert_bytes_received(self):
-        self.assertEqual(8, len(self.run.bytes))
+        self.assertEqual(9, len(self.run.bytes))
         for request, data in self.run.bytes.items():
             joined_data = b"".join(data)
             if self.run.getpath(request.url) == "/":
@@ -306,6 +312,8 @@ class EngineTest(unittest.TestCase):
                     b"  </body>\n"
                     b"</html>\n"
                 )
+            elif self.run.getpath(request.url) == "/random":
+                self.assertTrue(len(data) > 1)  # signal was fired multiple times
 
     def _assert_signals_caught(self):
         assert signals.engine_started in self.run.signals_caught

From a499f38b14d16338d20084c0dcb24528a1f1f22f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 29 Jan 2020 14:35:17 -0300
Subject: [PATCH 027/243] Remove object parent class

---
 scrapy/core/downloader/handlers/http11.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index c53c9bb2d..6f1bd9ad6 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -268,7 +268,7 @@ class ScrapyProxyAgent(Agent):
         )
 
 
-class ScrapyAgent(object):
+class ScrapyAgent:
 
     _Agent = Agent
     _ProxyAgent = ScrapyProxyAgent
@@ -438,7 +438,7 @@ class ScrapyAgent(object):
 
 
 @implementer(IBodyProducer)
-class _RequestBodyProducer(object):
+class _RequestBodyProducer:
 
     def __init__(self, body):
         self.body = body

From 6f02a8dccb95373f22bac18c08d9fda8169dcb02 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 29 Jan 2020 14:53:23 -0300
Subject: [PATCH 028/243] Add source parameter to bytes_received signal

---
 docs/topics/signals.rst                   | 12 ++++++++----
 scrapy/core/downloader/handlers/http11.py | 18 +++++++++++++-----
 scrapy/core/downloader/handlers/s3.py     |  1 +
 tests/test_downloader_handlers.py         |  3 +++
 tests/test_engine.py                      |  5 ++++-
 5 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index f490911f3..3a15bf95c 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -162,11 +162,11 @@ bytes_received
 --------------
 
 .. signal:: bytes_received
-.. function:: bytes_received(data, request, spider)
+.. function:: bytes_received(data, request, spider, source)
 
-    Sent by the HTTP 1.1 download handler when a group of bytes is
-    received for a specific request. This signal might be fired
-    multiple times for the same request.
+    Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is
+    received for a specific request. This signal might be fired multiple
+    times for the same request, with partial data each time.
 
     This signal does not support returning deferreds from its handlers.
 
@@ -179,6 +179,10 @@ bytes_received
     :param spider: the spider associated with the response
     :type spider: :class:`~scrapy.spiders.Spider` object
 
+    :param source: a string to identify which handler sent the signal
+        (current values could be "http11" or "s3")
+    :type source: :class:`str` object
+
 spider_closed
 -------------
 
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 6f1bd9ad6..49c9eacac 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -32,8 +32,9 @@ logger = logging.getLogger(__name__)
 class HTTP11DownloadHandler:
     lazy = False
 
-    def __init__(self, settings, crawler=None):
+    def __init__(self, settings, crawler=None, source="http11"):
         self.crawler = crawler
+        self.source = source
         self._pool = HTTPConnectionPool(reactor, persistent=True)
         self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
         self._pool._factory.noisy = False
@@ -67,8 +68,8 @@ class HTTP11DownloadHandler:
         self._disconnect_timeout = 1
 
     @classmethod
-    def from_crawler(cls, crawler):
-        return cls(crawler.settings, crawler)
+    def from_crawler(cls, crawler, **kwargs):
+        return cls(crawler.settings, crawler, **kwargs)
 
     def download_request(self, request, spider):
         """Return a deferred for the HTTP download"""
@@ -79,6 +80,7 @@ class HTTP11DownloadHandler:
             warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
             fail_on_dataloss=self._fail_on_dataloss,
             crawler=self.crawler,
+            source=self.source,
         )
         return agent.download_request(request)
 
@@ -275,7 +277,7 @@ class ScrapyAgent:
     _TunnelingAgent = TunnelingAgent
 
     def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None,
-                 maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None):
+                 maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None, source=None):
         self._contextFactory = contextFactory
         self._connectTimeout = connectTimeout
         self._bindAddress = bindAddress
@@ -285,6 +287,7 @@ class ScrapyAgent:
         self._fail_on_dataloss = fail_on_dataloss
         self._txresponse = None
         self._crawler = crawler
+        self._source = source
 
     def _get_agent(self, request, timeout):
         bindaddress = request.meta.get('bindaddress') or self._bindAddress
@@ -421,6 +424,7 @@ class ScrapyAgent:
                 warnsize,
                 fail_on_dataloss,
                 self._crawler,
+                self._source,
             )
         )
 
@@ -457,7 +461,9 @@ class _RequestBodyProducer:
 
 class _ResponseReader(protocol.Protocol):
 
-    def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler):
+    def __init__(
+        self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler, source
+    ):
         self._finished = finished
         self._txresponse = txresponse
         self._request = request
@@ -469,6 +475,7 @@ class _ResponseReader(protocol.Protocol):
         self._reached_warnsize = False
         self._bytes_received = 0
         self._crawler = crawler
+        self._source = source
 
     def dataReceived(self, bodyBytes):
         # This maybe called several times after cancel was called with buffered data.
@@ -483,6 +490,7 @@ class _ResponseReader(protocol.Protocol):
             data=bodyBytes,
             request=self._request,
             spider=self._crawler.spider,
+            source=self._source,
         )
 
         if self._maxsize and self._bytes_received > self._maxsize:
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 40a1fa48e..2366b6394 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -73,6 +73,7 @@ class S3DownloadHandler:
             objcls=httpdownloadhandler,
             settings=settings,
             crawler=crawler,
+            source="s3",
         )
         self._download_http = _http_handler.download_request
 
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 8d95d7cac..22a813647 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -730,6 +730,9 @@ class Http11ProxyTestCase(HttpProxyTestCase):
 
 class HttpDownloadHandlerMock:
 
+    def __init__(self, *args, **kwargs):
+        pass
+
     def download_request(self, request, spider):
         return request
 
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 3c5cc403b..c83a23b55 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -112,6 +112,7 @@ class CrawlerRun(object):
         self.itemerror = []
         self.itemresp = []
         self.bytes = defaultdict(lambda: list())
+        self.bytes_source = set()
         self.signals_caught = {}
         self.spider_class = spider_class
 
@@ -165,8 +166,9 @@ class CrawlerRun(object):
     def item_scraped(self, item, spider, response):
         self.itemresp.append((item, response))
 
-    def bytes_received(self, data, request, spider):
+    def bytes_received(self, data, request, spider, source):
         self.bytes[request].append(data)
+        self.bytes_source.add(source)
 
     def request_scheduled(self, request, spider):
         self.reqplug.append((request, spider))
@@ -279,6 +281,7 @@ class EngineTest(unittest.TestCase):
 
     def _assert_bytes_received(self):
         self.assertEqual(9, len(self.run.bytes))
+        self.assertEqual(self.run.bytes_source, set(["http11"]))
         for request, data in self.run.bytes.items():
             joined_data = b"".join(data)
             if self.run.getpath(request.url) == "/":

From 0641ba0faa97498ca5bee39c4e8faec58d5f0522 Mon Sep 17 00:00:00 2001
From: faizan2700 <syedfaizan824@gmail.com>
Date: Sun, 2 Feb 2020 16:54:22 +0530
Subject: [PATCH 029/243] SCRAPY_CHECK will be set while running contact

---
 scrapy/commands/check.py | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index 9d4437a47..09a76ca7a 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -78,19 +78,19 @@ class Command(ScrapyCommand):
                 elif tested_methods:
                     self.crawler_process.crawl(spidercls)
 
-        # start checks
-        if opts.list:
-            for spider, methods in sorted(contract_reqs.items()):
-                if not methods and not opts.verbose:
-                    continue
-                print(spider)
-                for method in sorted(methods):
-                    print('  * %s' % method)
-        else:
-            start = time.time()
-            self.crawler_process.start()
-            stop = time.time()
+            # start checks
+            if opts.list:
+                for spider, methods in sorted(contract_reqs.items()):
+                    if not methods and not opts.verbose:
+                        continue
+                    print(spider)
+                    for method in sorted(methods):
+                        print('  * %s' % method)
+            else:
+                start = time.time()
+                self.crawler_process.start()
+                stop = time.time()
 
-            result.printErrors()
-            result.printSummary(start, stop)
-            self.exitcode = int(not result.wasSuccessful())
+                result.printErrors()
+                result.printSummary(start, stop)
+                self.exitcode = int(not result.wasSuccessful())

From 7f2d3051feb0bc8f868a3eea8310e2fc8c461287 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 6 Feb 2020 18:19:40 -0300
Subject: [PATCH 030/243] Fix Flake8 issue

---
 tests/test_crawl.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index f1d502ff7..225fe7a0e 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -14,8 +14,8 @@ from tests.mockserver import MockServer
 from tests.spiders import (
     AsyncDefAsyncioReturnSpider,
     AsyncDefAsyncioSpider,
-    AsyncDefSpider, 
-    BrokenStartRequestsSpider, 
+    AsyncDefSpider,
+    BrokenStartRequestsSpider,
     CrawlSpiderWithErrback,
     CrawlSpiderWithParseMethod,
     DelaySpider,

From a64fa2f0866c10594f1e5cf00a0161f9fea1eb62 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 10 Feb 2020 10:16:05 -0300
Subject: [PATCH 031/243] Keyword arguments when creating a _ResponseReader

---
 scrapy/core/downloader/handlers/http11.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 49c9eacac..7a1a77b23 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -417,14 +417,14 @@ class ScrapyAgent:
         d = defer.Deferred(_cancel)
         txresponse.deliverBody(
             _ResponseReader(
-                d,
-                txresponse,
-                request,
-                maxsize,
-                warnsize,
-                fail_on_dataloss,
-                self._crawler,
-                self._source,
+                finished=d,
+                txresponse=txresponse,
+                request=request,
+                maxsize=maxsize,
+                warnsize=warnsize,
+                fail_on_dataloss=fail_on_dataloss,
+                crawler=self._crawler,
+                source=self._source,
             )
         )
 

From 122ce6d6fb3861d99ba2f2810b2370056bae1190 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 10 Feb 2020 10:20:26 -0300
Subject: [PATCH 032/243] Check bytes are received in order (bytes_received
 signal)

---
 tests/test_engine.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index c83a23b55..0d970928b 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -12,7 +12,6 @@ module with the ``runserver`` argument::
 
 import os
 import re
-import string
 import sys
 from collections import defaultdict
 from urllib.parse import urlparse
@@ -91,7 +90,8 @@ def start_test_site(debug=False):
     r = static.File(root_dir)
     r.putChild(b"redirect", util.Redirect(b"/redirected"))
     r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
-    r.putChild(b"random", static.Data(string.ascii_letters.encode("utf8") * 2**14, "text/plain"))
+    numbers = [str(x).encode("utf8") for x in range(2**14)]
+    r.putChild(b"numbers", static.Data(b"".join(numbers), "text/plain"))
 
     port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1")
     if debug:
@@ -124,7 +124,7 @@ class CrawlerRun(object):
             self.geturl("/"),
             self.geturl("/redirect"),
             self.geturl("/redirect"),  # duplicate
-            self.geturl("/random"),
+            self.geturl("/numbers"),
         ]
 
         for name, signal in vars(signals).items():
@@ -315,8 +315,12 @@ class EngineTest(unittest.TestCase):
                     b"  </body>\n"
                     b"</html>\n"
                 )
-            elif self.run.getpath(request.url) == "/random":
-                self.assertTrue(len(data) > 1)  # signal was fired multiple times
+            elif self.run.getpath(request.url) == "/numbers":
+                # signal was fired multiple times
+                self.assertTrue(len(data) > 1)
+                # bytes were received in order
+                numbers = [str(x).encode("utf8") for x in range(2**14)]
+                self.assertEqual(joined_data, b"".join(numbers))
 
     def _assert_signals_caught(self):
         assert signals.engine_started in self.run.signals_caught

From 42b4e9b3372ce3f9da57c7512b31a3c455b8a161 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 10 Feb 2020 11:23:38 -0300
Subject: [PATCH 033/243] Reword signal docs

---
 docs/topics/signals.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 3a15bf95c..dfb87cef3 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -279,7 +279,7 @@ request_scheduled
     Sent when the engine schedules a :class:`~scrapy.http.Request`, to be
     downloaded later.
 
-    The signal does not support returning deferreds from its handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param request: the request that reached the scheduler
     :type request: :class:`~scrapy.http.Request` object
@@ -296,7 +296,7 @@ request_dropped
     Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be
     downloaded later, is rejected by the scheduler.
 
-    The signal does not support returning deferreds from its handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param request: the request that reached the scheduler
     :type request: :class:`~scrapy.http.Request` object
@@ -312,7 +312,7 @@ request_reached_downloader
 
     Sent when a :class:`~scrapy.http.Request` reached downloader.
 
-    The signal does not support returning deferreds from its handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param request: the request that reached downloader
     :type request: :class:`~scrapy.http.Request` object

From 7025c18b159f1ce14b5732dc7d262efbdc72cf5e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 10 Feb 2020 19:43:23 +0100
Subject: [PATCH 034/243] Clear line of spaces

---
 docs/topics/feed-exports.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 9f837221e..8dc08710f 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -184,7 +184,7 @@ passed through the following settings:
 
  * :setting:`AWS_ACCESS_KEY_ID`
  * :setting:`AWS_SECRET_ACCESS_KEY`
- 
+
 You can also define a custom ACL for exported feeds using this setting:
 
  * :setting:`FEED_STORAGE_S3_ACL`

From 73e88d036c72aa628af50371aace4cfce7e286e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 12 Feb 2020 17:17:38 +0100
Subject: [PATCH 035/243] Import mock from unittest

---
 scrapy/utils/test.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 4ff6d73a5..00002c303 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -2,14 +2,13 @@
 This module contains some assorted functions used in tests
 """
 
-from __future__ import absolute_import
-from posixpath import split
 import asyncio
 import os
+from posixpath import split
+from unittest import mock
 
 from importlib import import_module
 from twisted.trial.unittest import SkipTest
-from tests import mock
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.boto import is_botocore

From e1be078eaa17a8df72716932fde07e225f79745f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 12 Feb 2020 17:38:06 +0100
Subject: [PATCH 036/243] Fix Flake8-reported issues

---
 scrapy/utils/test.py     |  2 ++
 tests/test_feedexport.py | 27 ++++++++++++++-------------
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 00002c303..7442a2f33 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -120,12 +120,14 @@ def assert_samelines(testcase, text1, text2, msg=None):
     """
     testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)
 
+
 def get_from_asyncio_queue(value):
     q = asyncio.Queue()
     getter = q.get()
     q.put_nowait(value)
     return getter
 
+
 def mock_google_cloud_storage():
     """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
     classes and set their proper return values.
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index d97b199fe..2b299503b 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -25,9 +25,13 @@ from scrapy.extensions.feedexport import (
     IFeedStorage, FileFeedStorage, FTPFeedStorage, GCSFeedStorage,
     S3FeedStorage, StdoutFeedStorage,
     BlockingFeedStorage)
-from scrapy.utils.test import (assert_aws_environ, get_s3_content_and_delete,
-    get_crawler, mock_google_cloud_storage)
 from scrapy.utils.python import to_unicode
+from scrapy.utils.test import (
+    assert_aws_environ,
+    get_s3_content_and_delete,
+    get_crawler,
+    mock_google_cloud_storage,
+)
 
 
 class FileFeedStorageTest(unittest.TestCase):
@@ -362,15 +366,13 @@ class S3FeedStorageTest(unittest.TestCase):
 
 class GCSFeedStorageTest(unittest.TestCase):
 
-    @mock.patch('scrapy.conf.settings',
-                new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': None }, create=True)
     def test_parse_settings(self):
         try:
-            from google.cloud.storage import Client
+            from google.cloud.storage import Client  # noqa
         except ImportError:
             raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
 
-        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead' }
+        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead'}
         crawler = get_crawler(settings_dict=settings)
         storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
         assert storage.project_id == '123'
@@ -378,20 +380,18 @@ class GCSFeedStorageTest(unittest.TestCase):
         assert storage.bucket_name == 'mybucket'
         assert storage.blob_name == 'export.csv'
 
-    @mock.patch('scrapy.conf.settings',
-                new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': '' }, create=True)
     def test_parse_empty_acl(self):
         try:
-            from google.cloud.storage import Client
+            from google.cloud.storage import Client  # noqa
         except ImportError:
             raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
 
-        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': '' }
+        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': ''}
         crawler = get_crawler(settings_dict=settings)
         storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
         assert storage.acl is None
 
-        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': None }
+        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': None}
         crawler = get_crawler(settings_dict=settings)
         storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
         assert storage.acl is None
@@ -399,7 +399,7 @@ class GCSFeedStorageTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_store(self):
         try:
-            from google.cloud.storage import Client
+            from google.cloud.storage import Client  # noqa
         except ImportError:
             raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
 
@@ -407,7 +407,7 @@ class GCSFeedStorageTest(unittest.TestCase):
         project_id = 'myproject-123'
         acl = 'publicRead'
         (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()
-        with mock.patch('google.cloud.storage.Client') as m:    
+        with mock.patch('google.cloud.storage.Client') as m:
             m.return_value = client_mock
 
             f = mock.Mock()
@@ -420,6 +420,7 @@ class GCSFeedStorageTest(unittest.TestCase):
             bucket_mock.blob.assert_called_once_with('export.csv')
             blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
 
+
 class StdoutFeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks

From 36a3913a41a033e57c6119edd8bbf967044c927f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 22 Mar 2020 21:34:20 -0300
Subject: [PATCH 037/243] E501 compliance

---
 pytest.ini                           | 176 +++++++++------------------
 scrapy/cmdline.py                    |   5 +-
 scrapy/core/downloader/middleware.py |  18 ++-
 scrapy/robotstxt.py                  |  10 +-
 scrapy/spiders/crawl.py              |   8 +-
 5 files changed, 85 insertions(+), 132 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 141a13a4f..781a45541 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -20,6 +20,7 @@ addopts =
 twisted = 1
 markers =
     only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
+flake8-max-line-length = 119
 flake8-ignore =
     W503
     # Files that are only meant to provide top-level imports are expected not
@@ -27,146 +28,85 @@ flake8-ignore =
     scrapy/core/downloader/handlers/http.py F401
     scrapy/http/__init__.py F401
     # Issues pending a review:
-    # extras
-    extras/qps-bench-server.py E501
-    extras/qpsclient.py E501 E501
     # scrapy/commands
-    scrapy/commands/__init__.py E128 E501
-    scrapy/commands/check.py E501
-    scrapy/commands/crawl.py E501
-    scrapy/commands/edit.py E501
-    scrapy/commands/fetch.py E401 E501 E128 E731
-    scrapy/commands/genspider.py E128 E501 E502
-    scrapy/commands/parse.py E128 E501 E731
-    scrapy/commands/runspider.py E501
+    scrapy/commands/__init__.py E128
+    scrapy/commands/fetch.py E401 E128 E731
+    scrapy/commands/genspider.py E128 E502
+    scrapy/commands/parse.py E128 E731
+    scrapy/commands/runspider.py
     scrapy/commands/settings.py E128
-    scrapy/commands/shell.py E128 E501 E502
-    scrapy/commands/startproject.py E127 E501 E128
-    scrapy/commands/version.py E501 E128
+    scrapy/commands/shell.py E128 E502
+    scrapy/commands/startproject.py E127 E128
+    scrapy/commands/version.py E128
     # scrapy/contracts
-    scrapy/contracts/__init__.py E501 W504
+    scrapy/contracts/__init__.py W504
     scrapy/contracts/default.py E128
     # scrapy/core
-    scrapy/core/engine.py E501 E128 E127 E502
-    scrapy/core/scheduler.py E501
-    scrapy/core/scraper.py E501 E128 W504
-    scrapy/core/spidermw.py E501 E731 E126
-    scrapy/core/downloader/__init__.py E501
-    scrapy/core/downloader/contextfactory.py E501 E128 E126
-    scrapy/core/downloader/middleware.py E501 E502
-    scrapy/core/downloader/tls.py E501 E241
-    scrapy/core/downloader/webclient.py E731 E501 E128 E126
-    scrapy/core/downloader/handlers/__init__.py E501
-    scrapy/core/downloader/handlers/ftp.py E501 E128 E127
-    scrapy/core/downloader/handlers/http10.py E501
-    scrapy/core/downloader/handlers/http11.py E501
-    scrapy/core/downloader/handlers/s3.py E501 E128 E126
+    scrapy/core/engine.py E128 E127 E502
+    scrapy/core/scheduler.py
+    scrapy/core/scraper.py E128 W504
+    scrapy/core/spidermw.py E731 E126
+    scrapy/core/downloader/__init__.py
+    scrapy/core/downloader/contextfactory.py E128 E126
+    scrapy/core/downloader/middleware.py E502
+    scrapy/core/downloader/tls.py E241
+    scrapy/core/downloader/webclient.py E731 E128 E126
+    scrapy/core/downloader/handlers/ftp.py E128 E127
+    scrapy/core/downloader/handlers/s3.py E128 E126
     # scrapy/downloadermiddlewares
-    scrapy/downloadermiddlewares/ajaxcrawl.py E501
-    scrapy/downloadermiddlewares/decompression.py E501
-    scrapy/downloadermiddlewares/defaultheaders.py E501
-    scrapy/downloadermiddlewares/httpcache.py E501 E126
-    scrapy/downloadermiddlewares/httpcompression.py E501 E128
-    scrapy/downloadermiddlewares/httpproxy.py E501
-    scrapy/downloadermiddlewares/redirect.py E501 W504
-    scrapy/downloadermiddlewares/retry.py E501 E126
-    scrapy/downloadermiddlewares/robotstxt.py E501
-    scrapy/downloadermiddlewares/stats.py E501
+    scrapy/downloadermiddlewares/httpcache.py E126
+    scrapy/downloadermiddlewares/httpcompression.py E128
+    scrapy/downloadermiddlewares/redirect.py W504
+    scrapy/downloadermiddlewares/retry.py E126
     # scrapy/extensions
-    scrapy/extensions/closespider.py E501 E128 E123
-    scrapy/extensions/corestats.py E501
-    scrapy/extensions/feedexport.py E128 E501
-    scrapy/extensions/httpcache.py E128 E501
-    scrapy/extensions/memdebug.py E501
-    scrapy/extensions/spiderstate.py E501
-    scrapy/extensions/telnet.py E501 W504
-    scrapy/extensions/throttle.py E501
+    scrapy/extensions/closespider.py E128 E123
+    scrapy/extensions/feedexport.py E128
+    scrapy/extensions/httpcache.py E128
+    scrapy/extensions/telnet.py W504
     # scrapy/http
-    scrapy/http/common.py E501
-    scrapy/http/cookies.py E501
-    scrapy/http/request/__init__.py E501
-    scrapy/http/request/form.py E501 E123
-    scrapy/http/request/json_request.py E501
-    scrapy/http/response/__init__.py E501 E128
-    scrapy/http/response/text.py E501 E128 E124
+    scrapy/http/request/form.py E123
+    scrapy/http/response/__init__.py E128
+    scrapy/http/response/text.py E128 E124
     # scrapy/linkextractors
-    scrapy/linkextractors/__init__.py E731 E501 E402 W504
-    scrapy/linkextractors/lxmlhtml.py E501 E731
+    scrapy/linkextractors/__init__.py E731 E402 W504
+    scrapy/linkextractors/lxmlhtml.py E731
     # scrapy/loader
-    scrapy/loader/__init__.py E501 E128
-    scrapy/loader/processors.py E501
+    scrapy/loader/__init__.py E128
+    scrapy/loader/processors.py
     # scrapy/pipelines
-    scrapy/pipelines/__init__.py E501
-    scrapy/pipelines/files.py E116 E501 E266
-    scrapy/pipelines/images.py E265 E501
-    scrapy/pipelines/media.py E125 E501 E266
+    scrapy/pipelines/files.py E116 E266
+    scrapy/pipelines/images.py E265
+    scrapy/pipelines/media.py E125 E266
     # scrapy/selector
     scrapy/selector/__init__.py F403
-    scrapy/selector/unified.py E501 E111
+    scrapy/selector/unified.py E111
     # scrapy/settings
-    scrapy/settings/__init__.py E501
-    scrapy/settings/default_settings.py E501 E114 E116
-    scrapy/settings/deprecated.py E501
+    scrapy/settings/default_settings.py E114 E116
     # scrapy/spidermiddlewares
-    scrapy/spidermiddlewares/httperror.py E501
-    scrapy/spidermiddlewares/offsite.py E501
-    scrapy/spidermiddlewares/referer.py E501 E129 W504
-    scrapy/spidermiddlewares/urllength.py E501
+    scrapy/spidermiddlewares/referer.py E129 W504
     # scrapy/spiders
-    scrapy/spiders/__init__.py E501 E402
-    scrapy/spiders/crawl.py E501
-    scrapy/spiders/feed.py E501
-    scrapy/spiders/sitemap.py E501
+    scrapy/spiders/__init__.py E402
     # scrapy/utils
-    scrapy/utils/asyncio.py E501
-    scrapy/utils/benchserver.py E501
-    scrapy/utils/conf.py E402 E501
-    scrapy/utils/datatypes.py E501
-    scrapy/utils/decorators.py E501
-    scrapy/utils/defer.py E501 E128
-    scrapy/utils/deprecate.py E128 E501 E127 E502
-    scrapy/utils/gz.py E501 W504
+    scrapy/utils/defer.py E128
+    scrapy/utils/deprecate.py E128 E127 E502
+    scrapy/utils/gz.py W504
     scrapy/utils/http.py F403
-    scrapy/utils/httpobj.py E501
-    scrapy/utils/iterators.py E501
-    scrapy/utils/log.py E128 E501
+    scrapy/utils/log.py E128
     scrapy/utils/markup.py F403
-    scrapy/utils/misc.py E501
     scrapy/utils/multipart.py F403
-    scrapy/utils/project.py E501
-    scrapy/utils/python.py E501
-    scrapy/utils/reactor.py E501
-    scrapy/utils/reqser.py E501
-    scrapy/utils/request.py E127 E501
-    scrapy/utils/response.py E501 E128
-    scrapy/utils/signal.py E501 E128
-    scrapy/utils/sitemap.py E501
-    scrapy/utils/spider.py E501
-    scrapy/utils/ssl.py E501
-    scrapy/utils/test.py E501
-    scrapy/utils/url.py E501 F403 E128 F405
+    scrapy/utils/request.py E127
+    scrapy/utils/response.py E128
+    scrapy/utils/signal.py E128
+    scrapy/utils/url.py F403 E128 F405
     # scrapy
-    scrapy/__init__.py E402 E501
-    scrapy/cmdline.py E501
-    scrapy/crawler.py E501
-    scrapy/dupefilters.py E501 E202
-    scrapy/exceptions.py E501
-    scrapy/exporters.py E501
-    scrapy/interfaces.py E501
-    scrapy/item.py E501 E128
-    scrapy/link.py E501
-    scrapy/logformatter.py E501
-    scrapy/mail.py E402 E128 E501 E502
-    scrapy/middleware.py E128 E501
-    scrapy/pqueues.py E501
-    scrapy/resolver.py E501
-    scrapy/responsetypes.py E128 E501
-    scrapy/robotstxt.py E501
-    scrapy/shell.py E501
-    scrapy/signalmanager.py E501
-    scrapy/spiderloader.py F841 E501 E126
+    scrapy/__init__.py E402
+    scrapy/dupefilters.py E202
+    scrapy/item.py E128
+    scrapy/mail.py E402 E128 E502
+    scrapy/middleware.py E128
+    scrapy/responsetypes.py E128
+    scrapy/spiderloader.py F841 E126
     scrapy/squeues.py E128
-    scrapy/statscollectors.py E501
     # tests
     tests/__init__.py E402 E501
     tests/mockserver.py E401 E501 E126 E123
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index a4ec7c8ae..b189e016b 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -165,6 +165,7 @@ if __name__ == '__main__':
     try:
         execute()
     finally:
-        # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect()
-        # on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies
+        # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect() on exit:
+        # http://doc.pypy.org/en/latest/cpython_differences.html
+        # ?highlight=gc.collect#differences-related-to-garbage-collection-strategies
         garbage_collect()
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 9c0014206..83c7b1f19 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -35,8 +35,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             for method in self.methods['process_request']:
                 response = yield deferred_from_coro(method(request=request, spider=spider))
                 if response is not None and not isinstance(response, (Response, Request)):
-                    raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
-                                         (method.__self__.__class__.__name__, response.__class__.__name__))
+                    raise _InvalidOutput(
+                        "Middleware %s.process_request must return None, Response or Request, got %s"
+                        % (method.__self__.__class__.__name__, response.__class__.__name__)
+                    )
                 if response:
                     defer.returnValue(response)
             defer.returnValue((yield download_func(request=request, spider=spider)))
@@ -50,8 +52,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             for method in self.methods['process_response']:
                 response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
                 if not isinstance(response, (Response, Request)):
-                    raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
-                                         (method.__self__.__class__.__name__, type(response)))
+                    raise _InvalidOutput(
+                        "Middleware %s.process_response must return Response or Request, got %s"
+                        % (method.__self__.__class__.__name__, type(response))
+                    )
                 if isinstance(response, Request):
                     defer.returnValue(response)
             defer.returnValue(response)
@@ -62,8 +66,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             for method in self.methods['process_exception']:
                 response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
                 if response is not None and not isinstance(response, (Response, Request)):
-                    raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
-                                         (method.__self__.__class__.__name__, type(response)))
+                    raise _InvalidOutput(
+                        "Middleware %s.process_exception must return None, Response or Request, got %s"
+                        % (method.__self__.__class__.__name__, type(response))
+                    )
                 if response:
                     defer.returnValue(response)
             defer.returnValue(_failure)
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 0a9af3a62..52cf09844 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -17,10 +17,12 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
     except UnicodeDecodeError:
         # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
         # Switch to 'allow all' state.
-        logger.warning("Failure while parsing robots.txt. "
-                       "File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",
-                       exc_info=sys.exc_info(),
-                       extra={'spider': spider})
+        logger.warning(
+            "Failure while parsing robots.txt. File either contains garbage or "
+            "is in an encoding other than UTF-8, treating it as an empty file.",
+            exc_info=sys.exc_info(),
+            extra={'spider': spider},
+        )
         robotstxt_body = ''
     return robotstxt_body
 
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index d76a96451..3d62224bf 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -54,8 +54,12 @@ class Rule:
         self.process_request = _get_method(self.process_request, spider)
         self.process_request_argcount = len(get_func_args(self.process_request))
         if self.process_request_argcount == 1:
-            msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated'
-            warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)
+            warnings.warn(
+                "Rule.process_request should accept two arguments"
+                " (request, response), accepting only one is deprecated",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
 
     def _process_request(self, request, response):
         """

From 182394bcecf556854291f1a0d2e0d2c406bc7b48 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 22 Mar 2020 23:29:30 -0300
Subject: [PATCH 038/243] E501 compliance (tests)

---
 pytest.ini                                    | 129 ++++++++----------
 tests/test_command_shell.py                   |   4 +-
 tests/test_crawler.py                         |   4 +-
 tests/test_downloader_handlers.py             |  10 +-
 ...st_downloadermiddleware_httpcompression.py |   9 +-
 tests/test_downloadermiddleware_redirect.py   |  10 +-
 tests/test_exporters.py                       |  25 +++-
 tests/test_feedexport.py                      |  31 +++--
 tests/test_http_request.py                    |   5 +-
 tests/test_http_response.py                   |  33 +++--
 tests/test_linkextractors.py                  |  49 ++++---
 tests/test_pipeline_files.py                  |  49 ++++---
 tests/test_pipeline_images.py                 |  39 +++---
 tests/test_request_cb_kwargs.py               |  10 +-
 tests/test_responsetypes.py                   |   9 +-
 tests/test_selector.py                        |  32 +++--
 tests/test_spider.py                          |   4 +-
 tests/test_spidermiddleware_output_chain.py   |  20 ++-
 tests/test_spidermiddleware_referer.py        |  86 ++++++++----
 tests/test_utils_iterators.py                 |  25 +++-
 tests/test_utils_request.py                   |   8 +-
 tests/test_utils_sitemap.py                   |  36 +++--
 tests/test_utils_url.py                       |  23 +++-
 23 files changed, 415 insertions(+), 235 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 781a45541..6f7e09478 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -108,84 +108,63 @@ flake8-ignore =
     scrapy/spiderloader.py F841 E126
     scrapy/squeues.py E128
     # tests
-    tests/__init__.py E402 E501
-    tests/mockserver.py E401 E501 E126 E123
+    tests/__init__.py E402
+    tests/mockserver.py E401 E126 E123
     tests/pipelines.py F841
-    tests/spiders.py E501 E127
-    tests/test_closespider.py E501 E127
-    tests/test_command_fetch.py E501
-    tests/test_command_parse.py E501 E128
-    tests/test_command_shell.py E501 E128
-    tests/test_commands.py E128 E501
-    tests/test_contracts.py E501 E128
-    tests/test_crawl.py E501 E741 E265
-    tests/test_crawler.py F841 E501
-    tests/test_dependencies.py F841 E501
-    tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E126 E123
-    tests/test_downloadermiddleware.py E501
-    tests/test_downloadermiddleware_ajaxcrawlable.py E501
-    tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126
+    tests/spiders.py E127
+    tests/test_closespider.py E127
+    tests/test_command_parse.py E128
+    tests/test_command_shell.py E128
+    tests/test_commands.py E128
+    tests/test_contracts.py E128
+    tests/test_crawl.py E741 E265
+    tests/test_crawler.py F841
+    tests/test_dependencies.py F841
+    tests/test_downloader_handlers.py E124 E127 E128 E265 E126 E123
+    tests/test_downloadermiddleware_cookies.py E731 E741 E128 E265 E126
     tests/test_downloadermiddleware_decompression.py E127
-    tests/test_downloadermiddleware_defaultheaders.py E501
-    tests/test_downloadermiddleware_downloadtimeout.py E501
-    tests/test_downloadermiddleware_httpcache.py E501
-    tests/test_downloadermiddleware_httpcompression.py E501 E126 E123
-    tests/test_downloadermiddleware_httpproxy.py E501 E128
-    tests/test_downloadermiddleware_redirect.py E501 E128 E127
-    tests/test_downloadermiddleware_retry.py E501 E128 E126
-    tests/test_downloadermiddleware_robotstxt.py E501
-    tests/test_downloadermiddleware_stats.py E501
-    tests/test_dupefilters.py E501 E741 E128 E124
-    tests/test_engine.py E401 E501 E128
-    tests/test_exporters.py E501 E731 E128 E124
+    tests/test_downloadermiddleware_httpcompression.py E126 E123
+    tests/test_downloadermiddleware_httpproxy.py E128
+    tests/test_downloadermiddleware_redirect.py E128 E127
+    tests/test_downloadermiddleware_retry.py E128 E126
+    tests/test_dupefilters.py E741 E128 E124
+    tests/test_engine.py E401 E128
+    tests/test_exporters.py E731 E128 E124
     tests/test_extension_telnet.py F841
-    tests/test_feedexport.py E501 F841 E241
-    tests/test_http_cookies.py E501
-    tests/test_http_headers.py E501
-    tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
-    tests/test_http_response.py E501 E128 E265
+    tests/test_feedexport.py F841 E241
+    tests/test_http_request.py E402 E127 E128 E128 E126 E123
+    tests/test_http_response.py E128 E265
     tests/test_item.py E128 F841
-    tests/test_link.py E501
-    tests/test_linkextractors.py E501 E128 E124
-    tests/test_loader.py E501 E731 E741 E128 E117 E241
-    tests/test_logformatter.py E128 E501 E122
-    tests/test_mail.py E128 E501
-    tests/test_middleware.py E501 E128
-    tests/test_pipeline_crawl.py E501 E128 E126
-    tests/test_pipeline_files.py E501
-    tests/test_pipeline_images.py F841 E501
-    tests/test_pipeline_media.py E501 E741 E731 E128 E502
-    tests/test_proxy_connect.py E501 E741
-    tests/test_request_cb_kwargs.py E501
-    tests/test_responsetypes.py E501
-    tests/test_robotstxt_interface.py E501 E501
-    tests/test_scheduler.py E501 E126 E123
-    tests/test_selector.py E501 E127
-    tests/test_spider.py E501
-    tests/test_spidermiddleware.py E501
-    tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
-    tests/test_spidermiddleware_offsite.py E501 E128 E111
-    tests/test_spidermiddleware_output_chain.py E501
-    tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
-    tests/test_squeues.py E501 E741
-    tests/test_utils_asyncio.py E501
-    tests/test_utils_conf.py E501 E128
-    tests/test_utils_curl.py E501
-    tests/test_utils_datatypes.py E402 E501
-    tests/test_utils_defer.py E501 F841
-    tests/test_utils_deprecate.py F841 E501
-    tests/test_utils_http.py E501 E128 W504
-    tests/test_utils_iterators.py E501 E128 E129 E241
+    tests/test_linkextractors.py E128 E124
+    tests/test_loader.py E731 E741 E128 E117 E241
+    tests/test_logformatter.py E128 E122
+    tests/test_mail.py E128
+    tests/test_middleware.py E128
+    tests/test_pipeline_crawl.py E128 E126
+    tests/test_pipeline_images.py F841
+    tests/test_pipeline_media.py E741 E731 E128 E502
+    tests/test_proxy_connect.py E741
+    tests/test_scheduler.py E126 E123
+    tests/test_selector.py E127
+    tests/test_spidermiddleware_httperror.py E128 E127 E121
+    tests/test_spidermiddleware_offsite.py E128 E111
+    tests/test_spidermiddleware_referer.py F841 E125 E201 E124 E241 E121
+    tests/test_squeues.py E741
+    tests/test_utils_conf.py E128
+    tests/test_utils_datatypes.py E402
+    tests/test_utils_defer.py F841
+    tests/test_utils_deprecate.py F841
+    tests/test_utils_http.py E128 W504
+    tests/test_utils_iterators.py E128 E129 E241
     tests/test_utils_log.py E741
-    tests/test_utils_python.py E501 E731
-    tests/test_utils_reqser.py E501 E128
-    tests/test_utils_request.py E501 E128
-    tests/test_utils_response.py E501
+    tests/test_utils_python.py E731
+    tests/test_utils_reqser.py E128
+    tests/test_utils_request.py E128
     tests/test_utils_signal.py E741 F841 E731
-    tests/test_utils_sitemap.py E128 E501 E124
-    tests/test_utils_url.py E501 E127 E125 E501 E241 E126 E123
-    tests/test_webclient.py E501 E128 E122 E402 E241 E123 E126
-    tests/test_cmdline/__init__.py E501
-    tests/test_settings/__init__.py E501 E128
-    tests/test_spiderloader/__init__.py E128 E501
-    tests/test_utils_misc/__init__.py E501
+    tests/test_utils_sitemap.py E128 E124
+    tests/test_utils_url.py E127 E125 E241 E126 E123
+    tests/test_webclient.py E128 E122 E402 E241 E123 E126
+    tests/test_cmdline/__init__.py
+    tests/test_settings/__init__.py E128
+    tests/test_spiderloader/__init__.py E128
+    tests/test_utils_misc/__init__.py
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index d664b6ade..5dc0ba3d2 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -56,7 +56,9 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_redirect_not_follow_302(self):
-        _, out, _ = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status'])
+        _, out, _ = yield self.execute(
+            ['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status']
+        )
         assert out.strip().endswith(b'302')
 
     @defer.inlineCallbacks
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 169e763f0..bfc9a609a 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -303,8 +303,10 @@ class CrawlerProcessSubprocess(unittest.TestCase):
     def test_ipv6_default_name_resolver(self):
         log = self.run_script('default_name_resolver.py')
         self.assertIn('Spider closed (finished)', log)
-        self.assertIn("twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", log)
         self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log)
+        self.assertIn(
+            "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.",
+            log)
 
     def test_ipv6_alternative_name_resolver(self):
         log = self.run_script('alternative_name_resolver.py')
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 29d06bab4..1a7be5c1b 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -493,7 +493,10 @@ class Http11TestCase(HttpTestCase):
 class Https11TestCase(Http11TestCase):
     scheme = 'https'
 
-    tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", subject "/C=IE/O=Scrapy/CN=localhost"'
+    tls_log_message = (
+        'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", '
+        'subject "/C=IE/O=Scrapy/CN=localhost"'
+    )
 
     @defer.inlineCallbacks
     def test_tls_logging(self):
@@ -542,7 +545,10 @@ class Https11InvalidDNSPattern(Https11TestCase):
             from service_identity.exceptions import CertificateError  # noqa: F401
         except ImportError:
             raise unittest.SkipTest("cryptography lib is too old")
-        self.tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
+        self.tls_log_message = (
+            'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", '
+            'subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
+        )
         super(Https11InvalidDNSPattern, self).setUp()
 
 
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 106ca3360..4d43e164f 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -124,7 +124,8 @@ class HttpCompressionTest(TestCase):
             'Content-Encoding': 'gzip',
         }
         f = BytesIO()
-        plainbody = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=gb2312">"""
+        plainbody = (b'<html><head><title>Some page</title>'
+                     b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">')
         zf = GzipFile(fileobj=f, mode='wb')
         zf.write(plainbody)
         zf.close()
@@ -142,7 +143,8 @@ class HttpCompressionTest(TestCase):
             'Content-Encoding': 'gzip',
         }
         f = BytesIO()
-        plainbody = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=gb2312">"""
+        plainbody = (b'<html><head><title>Some page</title>'
+                     b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">')
         zf = GzipFile(fileobj=f, mode='wb')
         zf.write(plainbody)
         zf.close()
@@ -158,7 +160,8 @@ class HttpCompressionTest(TestCase):
         headers = {
             'Content-Encoding': 'identity',
         }
-        plainbody = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=gb2312">"""
+        plainbody = (b'<html><head><title>Some page</title>'
+                     b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">')
         respcls = responsetypes.from_args(url="http://www.example.com/index", headers=headers, body=plainbody)
         response = respcls("http://www.example.com/index", headers=headers, body=plainbody)
         request = Request("http://www.example.com/index")
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 053e26fc3..3044c5b21 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -151,7 +151,10 @@ class RedirectMiddlewareTest(unittest.TestCase):
         self.assertEqual(req2.url, 'http://scrapytest.org/redirected')
         self.assertEqual(req2.meta['redirect_urls'], ['http://scrapytest.org/first'])
         self.assertEqual(req3.url, 'http://scrapytest.org/redirected2')
-        self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'])
+        self.assertEqual(
+            req3.meta['redirect_urls'],
+            ['http://scrapytest.org/first', 'http://scrapytest.org/redirected']
+        )
 
     def test_redirect_reasons(self):
         req1 = Request('http://scrapytest.org/first')
@@ -282,7 +285,10 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
         self.assertEqual(req2.url, 'http://scrapytest.org/redirected')
         self.assertEqual(req2.meta['redirect_urls'], ['http://scrapytest.org/first'])
         self.assertEqual(req3.url, 'http://scrapytest.org/redirected2')
-        self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'])
+        self.assertEqual(
+            req3.meta['redirect_urls'],
+            ['http://scrapytest.org/first', 'http://scrapytest.org/redirected']
+        )
 
     def test_redirect_reasons(self):
         req1 = Request('http://scrapytest.org/first')
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 6e2507508..b8301a567 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -111,7 +111,10 @@ class PythonItemExporterTest(BaseItemExporterTest):
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(type(exported), dict)
-        self.assertEqual(exported, {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'})
+        self.assertEqual(
+            exported,
+            {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'}
+        )
         self.assertEqual(type(exported['age']), dict)
         self.assertEqual(type(exported['age']['age']), dict)
 
@@ -121,7 +124,10 @@ class PythonItemExporterTest(BaseItemExporterTest):
         i3 = TestItem(name=u'Jesus', age=[i2])
         ie = self._get_exporter()
         exported = ie.export_item(i3)
-        self.assertEqual(exported, {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'})
+        self.assertEqual(
+            exported,
+            {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}
+        )
         self.assertEqual(type(exported['age'][0]), dict)
         self.assertEqual(type(exported['age'][0]['age'][0]), dict)
 
@@ -131,7 +137,10 @@ class PythonItemExporterTest(BaseItemExporterTest):
         i3 = TestItem(name=u'Jesus', age=[i2])
         ie = self._get_exporter()
         exported = ie.export_item(i3)
-        self.assertEqual(exported, {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'})
+        self.assertEqual(
+            exported,
+            {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}
+        )
         self.assertEqual(type(exported['age'][0]), dict)
         self.assertEqual(type(exported['age'][0]['age'][0]), dict)
 
@@ -327,13 +336,19 @@ class XmlItemExporterTest(BaseItemExporterTest):
         self.assertXmlEquivalent(fp.getvalue(), expected_value)
 
     def _check_output(self):
-        expected_value = b'<?xml version="1.0" encoding="utf-8"?>\n<items><item><age>22</age><name>John\xc2\xa3</name></item></items>'
+        expected_value = (
+            b'<?xml version="1.0" encoding="utf-8"?>\n'
+            b'<items><item><age>22</age><name>John\xc2\xa3</name></item></items>'
+        )
         self.assertXmlEquivalent(self.output.getvalue(), expected_value)
 
     def test_multivalued_fields(self):
         self.assertExportResult(
             TestItem(name=[u'John\xa3', u'Doe']),
-            b'<?xml version="1.0" encoding="utf-8"?>\n<items><item><name><value>John\xc2\xa3</value><value>Doe</value></name></item></items>'
+            (
+                b'<?xml version="1.0" encoding="utf-8"?>\n'
+                b'<items><item><name><value>John\xc2\xa3</value><value>Doe</value></name></item></items>'
+            )
         )
 
     def test_nested_item(self):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index c5589e52f..e248969db 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -718,10 +718,13 @@ class FeedExportTest(unittest.TestCase):
         header = ['foo']
 
         formats = {
-            'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
-            'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'),
-            'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'),
-            'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),
+            'json': '[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
+            'jsonlines': '{"foo": "Test\\u00d6"}\n'.encode('utf-8'),
+            'xml': (
+                '<?xml version="1.0" encoding="utf-8"?>\n'
+                '<items><item><foo>Test\xd6</foo></item></items>'
+            ).encode('utf-8'),
+            'csv': 'foo\r\nTest\xd6\r\n'.encode('utf-8'),
         }
 
         for fmt, expected in formats.items():
@@ -735,10 +738,13 @@ class FeedExportTest(unittest.TestCase):
             self.assertEqual(expected, data[fmt])
 
         formats = {
-            'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'),
-            'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'),
-            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'),
-            'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'),
+            'json': '[{"foo": "Test\xd6"}]'.encode('latin-1'),
+            'jsonlines': '{"foo": "Test\xd6"}\n'.encode('latin-1'),
+            'xml': (
+                '<?xml version="1.0" encoding="latin-1"?>\n'
+                '<items><item><foo>Test\xd6</foo></item></items>'
+            ).encode('latin-1'),
+            'csv': 'foo\r\nTest\xd6\r\n'.encode('latin-1'),
         }
 
         for fmt, expected in formats.items():
@@ -757,9 +763,12 @@ class FeedExportTest(unittest.TestCase):
         items = [dict({'foo': u'FOO', 'bar': u'BAR'})]
 
         formats = {
-            'json': u'[\n{"bar": "BAR"}\n]'.encode('utf-8'),
-            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'.encode('latin-1'),
-            'csv': u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
+            'json': '[\n{"bar": "BAR"}\n]'.encode('utf-8'),
+            'xml': (
+                '<?xml version="1.0" encoding="latin-1"?>\n'
+                '<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'
+            ).encode('latin-1'),
+            'csv': 'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
         }
 
         settings = {
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index cc2cddda4..87d98d6ee 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -1262,7 +1262,10 @@ class XmlRpcRequestTest(RequestTest):
 class JsonRequestTest(RequestTest):
     request_class = JsonRequest
     default_method = 'GET'
-    default_headers = {b'Content-Type': [b'application/json'], b'Accept': [b'application/json, text/javascript, */*; q=0.01']}
+    default_headers = {
+        b'Content-Type': [b'application/json'],
+        b'Accept': [b'application/json, text/javascript, */*; q=0.01'],
+    }
 
     def setUp(self):
         warnings.simplefilter("always")
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index eafc3560e..a6b717bd1 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -25,7 +25,11 @@ class BaseResponseTest(unittest.TestCase):
         self.assertTrue(isinstance(self.response_class('http://example.com/', body=b''), self.response_class))
         self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'body'), self.response_class))
         # test presence of all optional parameters
-        self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'', headers={}, status=200), self.response_class))
+        self.assertTrue(
+            isinstance(
+                self.response_class('http://example.com/', body=b'', headers={}, status=200), self.response_class
+            )
+        )
 
         r = self.response_class("http://www.example.com")
         assert isinstance(r.url, str)
@@ -323,13 +327,16 @@ class TextResponseTest(BaseResponseTest):
         self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
         resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1')
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
-        resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]})
+        resp = self.response_class(u"http://www.example.com/price/\xa3",
+                                   headers={"Content-type": ["text/html; charset=utf-8"]})
         self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
-        resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]})
+        resp = self.response_class(u"http://www.example.com/price/\xa3",
+                                   headers={"Content-type": ["text/html; charset=iso-8859-1"]})
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
 
     def test_unicode_body(self):
-        unicode_string = u'\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u0442\u0435\u043a\u0441\u0442'
+        unicode_string = ('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 '
+                          '\u0442\u0435\u043a\u0441\u0442')
         self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body=u'unicode body')
 
         original_string = unicode_string.encode('cp1251')
@@ -344,13 +351,18 @@ class TextResponseTest(BaseResponseTest):
         self.assertEqual(r1.text, unicode_string)
 
     def test_encoding(self):
-        r1 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xc2\xa3")
+        r1 = self.response_class("http://www.example.com", body=b"\xc2\xa3",
+                                 headers={"Content-type": ["text/html; charset=utf-8"]})
         r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3")
-        r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body=b"\xa3")
+        r3 = self.response_class("http://www.example.com", body=b"\xa3",
+                                 headers={"Content-type": ["text/html; charset=iso-8859-1"]})
         r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3")
-        r5 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=None"]}, body=b"\xc2\xa3")
-        r6 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gb2312"]}, body=b"\xa8D")
-        r7 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gbk"]}, body=b"\xa8D")
+        r5 = self.response_class("http://www.example.com", body=b"\xc2\xa3",
+                                 headers={"Content-type": ["text/html; charset=None"]})
+        r6 = self.response_class("http://www.example.com", body=b"\xa8D",
+                                 headers={"Content-type": ["text/html; charset=gb2312"]})
+        r7 = self.response_class("http://www.example.com", body=b"\xa8D",
+                                 headers={"Content-type": ["text/html; charset=gbk"]})
 
         self.assertEqual(r1._headers_encoding(), "utf-8")
         self.assertEqual(r2._headers_encoding(), None)
@@ -685,7 +697,8 @@ class HtmlResponseTest(TextResponseTest):
         body = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=utf-8">
         </head><body>Price: \xa3100</body></html>'
         """
-        r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body=body)
+        r3 = self.response_class("http://www.example.com", body=body,
+                                 headers={"Content-type": ["text/html; charset=iso-8859-1"]})
         self._assert_response_values(r3, 'iso-8859-1', body)
 
         # make sure replace() preserves the encoding of the original response
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 53968e60e..ed5f2560a 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -279,8 +279,8 @@ class Base:
         def test_process_value(self):
             """Test restrict_xpaths with encodings"""
             html = b"""
-            <a href="javascript:goToPage('../other/page.html','photo','width=600,height=540,scrollbars'); return false">Link text</a>
-            <a href="/about.html">About us</a>
+<a href="javascript:goToPage('../other/page.html','photo','width=600,height=540,scrollbars'); return false">Text</a>
+<a href="/about.html">About us</a>
             """
             response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
 
@@ -291,7 +291,7 @@ class Base:
 
             lx = self.extractor_cls(process_value=process_value)
             self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.org/other/page.html', text='Link text')])
+                             [Link(url='http://example.org/other/page.html', text='Text')])
 
         def test_base_url_with_restrict_xpaths(self):
             html = b"""<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
@@ -332,7 +332,10 @@ class Base:
             self.assertEqual(lx.extract_links(self.response), [])
 
         def test_tags(self):
-            html = b"""<html><area href="sample1.html"></area><a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>"""
+            html = (
+                b'<html><area href="sample1.html"></area>'
+                b'<a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>'
+            )
             response = HtmlResponse("http://example.com/index.html", body=html)
 
             lx = self.extractor_cls(tags=None)
@@ -413,24 +416,34 @@ class Base:
             response = HtmlResponse("http://example.com/index.xhtml", body=xhtml)
 
             lx = self.extractor_cls()
-            self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
-                              Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
-                              Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
-                              Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
-                              Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
-                            )
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
+                    Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
+                    Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='',
+                         nofollow=True),
+                    Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='',
+                         nofollow=False),
+                    Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+                ]
+            )
 
             response = XmlResponse("http://example.com/index.xhtml", body=xhtml)
 
             lx = self.extractor_cls()
-            self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
-                              Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
-                              Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
-                              Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
-                              Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
-                            )
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
+                    Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
+                    Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='',
+                         nofollow=True),
+                    Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='',
+                         nofollow=False),
+                    Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+                ]
+            )
 
         def test_link_wrong_href(self):
             html = b"""
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index f155db4ce..34fc8dcb6 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -38,27 +38,36 @@ class FilesPipelineTestCase(unittest.TestCase):
 
     def test_file_path(self):
         file_path = self.pipeline.file_path
-        self.assertEqual(file_path(Request("https://dev.mydeco.com/mydeco.pdf")),
-                         'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf')
-        self.assertEqual(file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt")),
-                         'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt')
-        self.assertEqual(file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc")),
-                         'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc')
-        self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")),
-                         'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg')
-        self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")),
-                         'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2')
-        self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532")),
-                         'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
-        self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532"),
-                                   response=Response("http://www.dorma.co.uk/images/product_details/2532"),
-                                   info=object()),
-                         'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
-        self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha")),
-                         'full/76c00cef2ef669ae65052661f68d451162829507')
-        self.assertEqual(file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\
+        self.assertEqual(
+            file_path(Request("https://dev.mydeco.com/mydeco.pdf")),
+            'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf')
+        self.assertEqual(
+            file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt")),
+            'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt')
+        self.assertEqual(
+            file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc")),
+            'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc')
+        self.assertEqual(
+            file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")),
+            'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg')
+        self.assertEqual(
+            file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")),
+            'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2')
+        self.assertEqual(
+            file_path(Request("http://www.dorma.co.uk/images/product_details/2532")),
+            'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
+        self.assertEqual(
+            file_path(Request("http://www.dorma.co.uk/images/product_details/2532"),
+                      response=Response("http://www.dorma.co.uk/images/product_details/2532"),
+                      info=object()),
+            'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
+        self.assertEqual(
+            file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha")),
+            'full/76c00cef2ef669ae65052661f68d451162829507')
+        self.assertEqual(
+            file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\
                                     //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y")),
-                         'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png')
+            'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png')
 
     def test_fs_store(self):
         assert isinstance(self.pipeline.store, FSFilesStore)
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 5018d6802..e1c3a55dd 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -41,22 +41,29 @@ class ImagesPipelineTestCase(unittest.TestCase):
 
     def test_file_path(self):
         file_path = self.pipeline.file_path
-        self.assertEqual(file_path(Request("https://dev.mydeco.com/mydeco.gif")),
-                         'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
-        self.assertEqual(file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg")),
-                         'full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg')
-        self.assertEqual(file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif")),
-                         'full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg')
-        self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")),
-                         'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg')
-        self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")),
-                         'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg')
-        self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532")),
-                         'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg')
-        self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532"),
-                                   response=Response("http://www.dorma.co.uk/images/product_details/2532"),
-                                   info=object()),
-                         'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg')
+        self.assertEqual(
+            file_path(Request("https://dev.mydeco.com/mydeco.gif")),
+            'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
+        self.assertEqual(
+            file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg")),
+            'full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg')
+        self.assertEqual(
+            file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif")),
+            'full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg')
+        self.assertEqual(
+            file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")),
+            'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg')
+        self.assertEqual(
+            file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")),
+            'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg')
+        self.assertEqual(
+            file_path(Request("http://www.dorma.co.uk/images/product_details/2532")),
+            'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg')
+        self.assertEqual(
+            file_path(Request("http://www.dorma.co.uk/images/product_details/2532"),
+                      response=Response("http://www.dorma.co.uk/images/product_details/2532"),
+                      info=object()),
+            'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg')
 
     def test_thumbnail_name(self):
         thumb_path = self.pipeline.thumb_path
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index a3ddd50f4..bd49179aa 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -158,6 +158,12 @@ class CallbackKeywordArgumentsTestCase(TestCase):
                 if key in line.getMessage():
                     exceptions[key] = line
         self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)
-        self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'")
+        self.assertEqual(
+            str(exceptions['takes_less'].exc_info[1]),
+            "parse_takes_less() got an unexpected keyword argument 'number'"
+        )
         self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
-        self.assertEqual(str(exceptions['takes_more'].exc_info[1]), "parse_takes_more() missing 1 required positional argument: 'other'")
+        self.assertEqual(
+            str(exceptions['takes_more'].exc_info[1]),
+            "parse_takes_more() missing 1 required positional argument: 'other'"
+        )
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 8cdf7a176..9f4d56412 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -64,8 +64,9 @@ class ResponseTypesTest(unittest.TestCase):
     def test_from_headers(self):
         mappings = [
             ({'Content-Type': ['text/html; charset=utf-8']}, HtmlResponse),
-            ({'Content-Type': ['application/octet-stream'], 'Content-Disposition': ['attachment; filename=data.txt']}, TextResponse),
             ({'Content-Type': ['text/html; charset=utf-8'], 'Content-Encoding': ['gzip']}, Response),
+            ({'Content-Type': ['application/octet-stream'],
+              'Content-Disposition': ['attachment; filename=data.txt']}, TextResponse),
         ]
         for source, cls in mappings:
             source = Headers(source)
@@ -77,8 +78,10 @@ class ResponseTypesTest(unittest.TestCase):
         mappings = [
             ({'url': 'http://www.example.com/data.csv'}, TextResponse),
             # headers takes precedence over url
-            ({'headers': Headers({'Content-Type': ['text/html; charset=utf-8']}), 'url': 'http://www.example.com/item/'}, HtmlResponse),
-            ({'headers': Headers({'Content-Disposition': ['attachment; filename="data.xml.gz"']}), 'url': 'http://www.example.com/page/'}, Response),
+            ({'headers': Headers({'Content-Type': ['text/html; charset=utf-8']}),
+              'url': 'http://www.example.com/item/'}, HtmlResponse),
+            ({'headers': Headers({'Content-Disposition': ['attachment; filename="data.xml.gz"']}),
+              'url': 'http://www.example.com/page/'}, Response),
 
 
         ]
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 09c2546fb..3629feb16 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -19,18 +19,26 @@ class SelectorTestCase(unittest.TestCase):
         for x in xl:
             assert isinstance(x, Selector)
 
-        self.assertEqual(sel.xpath('//input').getall(),
-                         [x.get() for x in sel.xpath('//input')])
-
-        self.assertEqual([x.get() for x in sel.xpath("//input[@name='a']/@name")],
-                         [u'a'])
-        self.assertEqual([x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
-                         [u'12.0'])
-
-        self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(),
-                         [u'xpathrules'])
-        self.assertEqual([x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
-                         [u'12'])
+        self.assertEqual(
+            sel.xpath('//input').getall(),
+            [x.get() for x in sel.xpath('//input')]
+        )
+        self.assertEqual(
+            [x.get() for x in sel.xpath("//input[@name='a']/@name")],
+            [u'a']
+        )
+        self.assertEqual(
+            [x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
+            [u'12.0']
+        )
+        self.assertEqual(
+            sel.xpath("concat('xpath', 'rules')").getall(),
+            [u'xpathrules']
+        )
+        self.assertEqual(
+            [x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
+            [u'12']
+        )
 
     def test_root_base_url(self):
         body = b'<html><form action="/path"><input name="a" /></form></html>'
diff --git a/tests/test_spider.py b/tests/test_spider.py
index bb00c8f42..805d70459 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -120,7 +120,9 @@ class XMLFeedSpiderTest(SpiderTest):
         body = b"""<?xml version="1.0" encoding="UTF-8"?>
         <urlset xmlns:x="http://www.google.com/schemas/sitemap/0.84"
                 xmlns:y="http://www.example.com/schemas/extras/1.0">
-        <url><x:loc>http://www.example.com/Special-Offers.html</loc><y:updated>2009-08-16</updated><other value="bar" y:custom="fuu"/></url>
+        <url><x:loc>http://www.example.com/Special-Offers.html</loc><y:updated>2009-08-16</updated>
+            <other value="bar" y:custom="fuu"/>
+        </url>
         <url><loc>http://www.example.com/</loc><y:updated>2009-08-16</updated><other value="foo"/></url>
         </urlset>"""
         response = XmlResponse(url='http://example.com/sitemap.xml', body=body)
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index b26353d6c..78d2de347 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -385,9 +385,15 @@ class TestSpiderMiddleware(TestCase):
         log4 = yield self.crawl_log(GeneratorOutputChainSpider)
         self.assertIn("'item_scraped_count': 2", str(log4))
         self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: LookupError caught", str(log4))
-        self.assertIn("GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught", str(log4))
-        self.assertNotIn("GeneratorFailMiddleware.process_spider_exception: LookupError caught", str(log4))
-        self.assertNotIn("GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught", str(log4))
+        self.assertIn(
+            "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught",
+            str(log4))
+        self.assertNotIn(
+            "GeneratorFailMiddleware.process_spider_exception: LookupError caught",
+            str(log4))
+        self.assertNotIn(
+            "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught",
+            str(log4))
         item_from_callback = {'processed': [
             'parse-first-item',
             'GeneratorFailMiddleware.process_spider_output',
@@ -414,9 +420,13 @@ class TestSpiderMiddleware(TestCase):
         log5 = yield self.crawl_log(NotGeneratorOutputChainSpider)
         self.assertIn("'item_scraped_count': 1", str(log5))
         self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught", str(log5))
-        self.assertIn("GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught", str(log5))
+        self.assertIn(
+            "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught",
+            str(log5))
         self.assertNotIn("GeneratorFailMiddleware.process_spider_exception: ReferenceError caught", str(log5))
-        self.assertNotIn("GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught", str(log5))
+        self.assertNotIn(
+            "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught",
+            str(log5))
         item_recovered = {'processed': [
             'NotGeneratorRecoverMiddleware.process_spider_exception',
             'NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']}
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 4c6ede70b..3974f7884 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -114,12 +114,16 @@ class MixinNoReferrerWhenDowngrade:
 class MixinSameOrigin:
     scenarii = [
         # Same origin (protocol, host, port): send referrer
-        ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
-        ('http://example.com/page.html',        'http://example.com/not-page.html',         b'http://example.com/page.html'),
-        ('https://example.com:443/page.html',   'https://example.com/not-page.html',        b'https://example.com/page.html'),
-        ('http://example.com:80/page.html',     'http://example.com/not-page.html',         b'http://example.com/page.html'),
-        ('http://example.com/page.html',        'http://example.com:80/not-page.html',      b'http://example.com/page.html'),
-        ('http://example.com:8888/page.html',   'http://example.com:8888/not-page.html',    b'http://example.com:8888/page.html'),
+        ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
+        ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
+        ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
+        ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
+        ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
+        (
+            'http://example.com:8888/page.html',
+            'http://example.com:8888/not-page.html',
+            b'http://example.com:8888/page.html',
+        ),
 
         # Different host: do NOT send referrer
         ('https://example.com/page.html',       'https://not.example.com/otherpage.html',   None),
@@ -139,8 +143,12 @@ class MixinSameOrigin:
         ('ftps://example.com/urls.zip',     'https://example.com/not-page.html',    None),
 
         # test for user/password stripping
-        ('https://user:password@example.com/page.html', 'https://example.com/not-page.html',    b'https://example.com/page.html'),
-        ('https://user:password@example.com/page.html', 'http://example.com/not-page.html',     None),
+        ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None),
+        (
+            'https://user:password@example.com/page.html',
+            'https://example.com/not-page.html',
+            b'https://example.com/page.html',
+        ),
     ]
 
 
@@ -179,12 +187,16 @@ class MixinStrictOrigin:
 class MixinOriginWhenCrossOrigin:
     scenarii = [
         # Same origin (protocol, host, port): send referrer
-        ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
-        ('http://example.com/page.html',        'http://example.com/not-page.html',         b'http://example.com/page.html'),
-        ('https://example.com:443/page.html',   'https://example.com/not-page.html',        b'https://example.com/page.html'),
-        ('http://example.com:80/page.html',     'http://example.com/not-page.html',         b'http://example.com/page.html'),
-        ('http://example.com/page.html',        'http://example.com:80/not-page.html',      b'http://example.com/page.html'),
-        ('http://example.com:8888/page.html',   'http://example.com:8888/not-page.html',    b'http://example.com:8888/page.html'),
+        ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
+        ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
+        ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
+        ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
+        ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
+        (
+            'http://example.com:8888/page.html',
+            'http://example.com:8888/not-page.html',
+            b'http://example.com:8888/page.html',
+        ),
 
         # Different host: send origin as referrer
         ('https://example2.com/page.html',  'https://scrapy.org/otherpage.html',        b'https://example2.com/'),
@@ -205,21 +217,33 @@ class MixinOriginWhenCrossOrigin:
         ('ftps://example4.com/urls.zip',    'https://example4.com/not-page.html',   b'ftps://example4.com/'),
 
         # test for user/password stripping
-        ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html',  b'https://example5.com/page.html'),
+        (
+            'https://user:password@example5.com/page.html',
+            'https://example5.com/not-page.html',
+            b'https://example5.com/page.html',
+        ),
         # TLS to non-TLS downgrade: send origin
-        ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html',   b'https://example5.com/'),
+        (
+            'https://user:password@example5.com/page.html',
+            'http://example5.com/not-page.html',
+            b'https://example5.com/',
+        ),
     ]
 
 
 class MixinStrictOriginWhenCrossOrigin:
     scenarii = [
         # Same origin (protocol, host, port): send referrer
-        ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
-        ('http://example.com/page.html',        'http://example.com/not-page.html',         b'http://example.com/page.html'),
-        ('https://example.com:443/page.html',   'https://example.com/not-page.html',        b'https://example.com/page.html'),
-        ('http://example.com:80/page.html',     'http://example.com/not-page.html',         b'http://example.com/page.html'),
-        ('http://example.com/page.html',        'http://example.com:80/not-page.html',      b'http://example.com/page.html'),
-        ('http://example.com:8888/page.html',   'http://example.com:8888/not-page.html',    b'http://example.com:8888/page.html'),
+        ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
+        ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
+        ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
+        ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
+        ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
+        (
+            'http://example.com:8888/page.html',
+            'http://example.com:8888/not-page.html',
+            b'http://example.com:8888/page.html',
+        ),
 
         # Different host: send origin as referrer
         ('https://example2.com/page.html',  'https://scrapy.org/otherpage.html',        b'https://example2.com/'),
@@ -248,7 +272,11 @@ class MixinStrictOriginWhenCrossOrigin:
         ('ftps://example4.com/urls.zip',    'https://example4.com/not-page.html',   b'ftps://example4.com/'),
 
         # test for user/password stripping
-        ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html',  b'https://example5.com/page.html'),
+        (
+            'https://user:password@example5.com/page.html',
+            'https://example5.com/not-page.html',
+            b'https://example5.com/page.html',
+        ),
 
         # TLS to non-TLS downgrade: send nothing
         ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html',   None),
@@ -281,8 +309,16 @@ class MixinUnsafeUrl:
         ('ftp://example3.com/urls.zip',         'https://scrapy.org/',          b'ftp://example3.com/urls.zip'),
 
         # test for user/password stripping
-        ('http://user:password@example4.com/page.html',     'https://not.example4.com/',    b'http://example4.com/page.html'),
-        ('https://user:password@example4.com/page.html',    'http://scrapy.org/',           b'https://example4.com/page.html'),
+        (
+            'http://user:password@example4.com/page.html',
+            'https://not.example4.com/',
+            b'http://example4.com/page.html',
+        ),
+        (
+            'https://user:password@example4.com/page.html',
+            'http://scrapy.org/',
+            b'https://example4.com/page.html',
+        ),
     ]
 
 
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 33fc4d570..7c4011142 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -17,7 +17,8 @@ class XmliterTestCase(unittest.TestCase):
 
     def test_xmliter(self):
         body = b"""<?xml version="1.0" encoding="UTF-8"?>\
-            <products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="someschmea.xsd">\
+            <products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                      xsi:noNamespaceSchemaLocation="someschmea.xsd">\
               <product id="001">\
                 <type>Type 1</type>\
                 <name>Name 1</name>\
@@ -107,7 +108,10 @@ class XmliterTestCase(unittest.TestCase):
                               (u'27', [u'A'], [u'27'])])
 
     def test_xmliter_text(self):
-        body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
+        body = (
+            '<?xml version="1.0" encoding="UTF-8"?>'
+            '<products><product>one</product><product>two</product></products>'
+        )
 
         self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')],
                          [[u'one'], [u'two']])
@@ -139,7 +143,10 @@ class XmliterTestCase(unittest.TestCase):
         self.assertEqual(node.xpath('title/text()').getall(), ['Item 1'])
         self.assertEqual(node.xpath('description/text()').getall(), ['This is item 1'])
         self.assertEqual(node.xpath('link/text()').getall(), ['http://www.mydummycompany.com/items/1'])
-        self.assertEqual(node.xpath('g:image_link/text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg'])
+        self.assertEqual(
+            node.xpath('g:image_link/text()').getall(),
+            ['http://www.mydummycompany.com/images/item1.jpg']
+        )
         self.assertEqual(node.xpath('g:id/text()').getall(), ['ITEM_1'])
         self.assertEqual(node.xpath('g:price/text()').getall(), ['400'])
         self.assertEqual(node.xpath('image_link/text()').getall(), [])
@@ -147,7 +154,10 @@ class XmliterTestCase(unittest.TestCase):
         self.assertEqual(node.xpath('price/text()').getall(), [])
 
     def test_xmliter_exception(self):
-        body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
+        body = (
+            '<?xml version="1.0" encoding="UTF-8"?>'
+            '<products><product>one</product><product>two</product></products>'
+        )
 
         iter = self.xmliter(body, 'product')
         next(iter)
@@ -160,7 +170,12 @@ class XmliterTestCase(unittest.TestCase):
         self.assertRaises(AssertionError, next, i)
 
     def test_xmliter_encoding(self):
-        body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n    <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n'
+        body = (
+            b'<?xml version="1.0" encoding="ISO-8859-9"?>\n'
+            b'<xml>\n'
+            b'    <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n'
+            b'</xml>\n\n'
+        )
         response = XmlResponse('http://www.example.com', body=body)
         self.assertEqual(
             next(self.xmliter(response, 'item')).get(),
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 45f0f59e4..3b1994019 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -75,8 +75,12 @@ class UtilsRequestTest(unittest.TestCase):
         r1 = Request("http://www.example.com/some/page.html?arg=1")
         self.assertEqual(request_httprepr(r1), b'GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n')
 
-        r1 = Request("http://www.example.com", method='POST', headers={"Content-type": b"text/html"}, body=b"Some body")
-        self.assertEqual(request_httprepr(r1), b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body')
+        r1 = Request("http://www.example.com", method='POST',
+                     headers={"Content-type": b"text/html"}, body=b"Some body")
+        self.assertEqual(
+            request_httprepr(r1),
+            b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body'
+        )
 
     def test_request_httprepr_for_non_http_request(self):
         # the representation is not important but it must not fail.
diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py
index db323ab31..4c09d33b2 100644
--- a/tests/test_utils_sitemap.py
+++ b/tests/test_utils_sitemap.py
@@ -22,8 +22,14 @@ class SitemapTest(unittest.TestCase):
   </url>
 </urlset>""")
         assert s.type == 'urlset'
-        self.assertEqual(list(s),
-            [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'}])
+        self.assertEqual(
+            list(s),
+            [
+                {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
+                {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html',
+                 'lastmod': '2009-08-16', 'changefreq': 'weekly'},
+            ]
+        )
 
     def test_sitemap_index(self):
         s = Sitemap(b"""<?xml version="1.0" encoding="UTF-8"?>
@@ -38,7 +44,13 @@ class SitemapTest(unittest.TestCase):
    </sitemap>
 </sitemapindex>""")
         assert s.type == 'sitemapindex'
-        self.assertEqual(list(s), [{'loc': 'http://www.example.com/sitemap1.xml.gz', 'lastmod': '2004-10-01T18:23:17+00:00'}, {'loc': 'http://www.example.com/sitemap2.xml.gz', 'lastmod': '2005-01-01'}])
+        self.assertEqual(
+            list(s),
+            [
+                {'loc': 'http://www.example.com/sitemap1.xml.gz', 'lastmod': '2004-10-01T18:23:17+00:00'},
+                {'loc': 'http://www.example.com/sitemap2.xml.gz', 'lastmod': '2005-01-01'},
+            ]
+        )
 
     def test_sitemap_strip(self):
         """Assert we can deal with trailing spaces inside <loc> tags - we've
@@ -195,11 +207,19 @@ Disallow: /forum/active/
         </url>
     </urlset>""")
 
-        self.assertEqual(list(s), [
-            {'loc': 'http://www.example.com/english/',
-             'alternate': ['http://www.example.com/deutsch/', 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/english/']
-            }
-        ])
+        self.assertEqual(
+            list(s),
+            [
+                {
+                    'loc': 'http://www.example.com/english/',
+                    'alternate': [
+                        'http://www.example.com/deutsch/',
+                        'http://www.example.com/schweiz-deutsch/',
+                        'http://www.example.com/english/',
+                    ],
+                }
+            ]
+        )
 
     def test_xml_entity_expansion(self):
         s = Sitemap(b"""<?xml version="1.0" encoding="utf-8"?>
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 7abff8281..a12b6dda2 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -28,7 +28,10 @@ class UrlUtilsTest(unittest.TestCase):
         self.assertTrue(url_is_from_any_domain(url, ['192.169.0.15:8080']))
         self.assertFalse(url_is_from_any_domain(url, ['192.169.0.15']))
 
-        url = 'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20javascript:%20document.orderform_2581_1190810811.submit%28%29'
+        url = (
+            'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20'
+            'javascript:%20document.orderform_2581_1190810811.submit%28%29'
+        )
         self.assertFalse(url_is_from_any_domain(url, ['testdomain.com']))
         self.assertFalse(url_is_from_any_domain(url + '.testdomain.com', ['testdomain.com']))
 
@@ -105,8 +108,10 @@ class AddHttpIfNoScheme(unittest.TestCase):
                                                'http://username:password@www.example.com')
 
     def test_complete_url(self):
-        self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
-                                               'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
+            'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'
+        )
 
     def test_preserve_http(self):
         self.assertEqual(add_http_if_no_scheme('http://www.example.com'),
@@ -137,8 +142,10 @@ class AddHttpIfNoScheme(unittest.TestCase):
                                                'http://username:password@www.example.com')
 
     def test_preserve_http_complete_url(self):
-        self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
-                                               'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
+            'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'
+        )
 
     def test_protocol_relative(self):
         self.assertEqual(add_http_if_no_scheme('//www.example.com'),
@@ -169,8 +176,10 @@ class AddHttpIfNoScheme(unittest.TestCase):
                                                'http://username:password@www.example.com')
 
     def test_protocol_relative_complete_url(self):
-        self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
-                                               'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
+            'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'
+        )
 
     def test_preserve_https(self):
         self.assertEqual(add_http_if_no_scheme('https://www.example.com'),

From 4663f0b9df7f687bf05ca57e5b6e8442b0997df7 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 22 Mar 2020 23:38:51 -0300
Subject: [PATCH 039/243] Update pytest.ini after removing E501

---
 pytest.ini | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 6f7e09478..1b00bfb84 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -33,7 +33,6 @@ flake8-ignore =
     scrapy/commands/fetch.py E401 E128 E731
     scrapy/commands/genspider.py E128 E502
     scrapy/commands/parse.py E128 E731
-    scrapy/commands/runspider.py
     scrapy/commands/settings.py E128
     scrapy/commands/shell.py E128 E502
     scrapy/commands/startproject.py E127 E128
@@ -43,10 +42,8 @@ flake8-ignore =
     scrapy/contracts/default.py E128
     # scrapy/core
     scrapy/core/engine.py E128 E127 E502
-    scrapy/core/scheduler.py
     scrapy/core/scraper.py E128 W504
     scrapy/core/spidermw.py E731 E126
-    scrapy/core/downloader/__init__.py
     scrapy/core/downloader/contextfactory.py E128 E126
     scrapy/core/downloader/middleware.py E502
     scrapy/core/downloader/tls.py E241
@@ -72,7 +69,6 @@ flake8-ignore =
     scrapy/linkextractors/lxmlhtml.py E731
     # scrapy/loader
     scrapy/loader/__init__.py E128
-    scrapy/loader/processors.py
     # scrapy/pipelines
     scrapy/pipelines/files.py E116 E266
     scrapy/pipelines/images.py E265
@@ -87,6 +83,7 @@ flake8-ignore =
     # scrapy/spiders
     scrapy/spiders/__init__.py E402
     # scrapy/utils
+    scrapy/utils/conf.py E402
     scrapy/utils/defer.py E128
     scrapy/utils/deprecate.py E128 E127 E502
     scrapy/utils/gz.py W504
@@ -164,7 +161,5 @@ flake8-ignore =
     tests/test_utils_sitemap.py E128 E124
     tests/test_utils_url.py E127 E125 E241 E126 E123
     tests/test_webclient.py E128 E122 E402 E241 E123 E126
-    tests/test_cmdline/__init__.py
     tests/test_settings/__init__.py E128
     tests/test_spiderloader/__init__.py E128
-    tests/test_utils_misc/__init__.py

From ee510cf0d6dda87bd5d472d177c04ceef742d1e3 Mon Sep 17 00:00:00 2001
From: elacuesta <elacuesta@users.noreply.github.com>
Date: Tue, 24 Mar 2020 13:31:44 -0300
Subject: [PATCH 040/243] Update scrapy/spiders/crawl.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 scrapy/spiders/crawl.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 3d62224bf..cb021a5a7 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -55,8 +55,8 @@ class Rule:
         self.process_request_argcount = len(get_func_args(self.process_request))
         if self.process_request_argcount == 1:
             warnings.warn(
-                "Rule.process_request should accept two arguments"
-                " (request, response), accepting only one is deprecated",
+                "Rule.process_request should accept two arguments "
+                "(request, response), accepting only one is deprecated",
                 category=ScrapyDeprecationWarning,
                 stacklevel=2,
             )

From a175b6efc319ecbfd03b19705c25bd8bd44ca339 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Fri, 27 Mar 2020 02:10:10 -0300
Subject: [PATCH 041/243] Set up CI with Azure Pipelines

[skip ci]
---
 azure-pipelines.yml | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 azure-pipelines.yml

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
new file mode 100644
index 000000000..9bc324c32
--- /dev/null
+++ b/azure-pipelines.yml
@@ -0,0 +1,32 @@
+# Python package
+# Create and test a Python package on multiple Python versions.
+# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
+# https://docs.microsoft.com/azure/devops/pipelines/languages/python
+
+
+pool:
+  vmImage: 'windows-2019'
+strategy:
+  matrix:
+    Python35:
+      python.version: '3.5'
+    Python36:
+      python.version: '3.6'
+    Python37:
+      python.version: '3.7'
+
+steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: '$(python.version)'
+  displayName: 'Use Python $(python.version)'
+
+- script: |
+    python -m pip install --upgrade pip
+    pip install -r requirements.txt
+  displayName: 'Install dependencies'
+
+- script: |
+    pip install pytest pytest-azurepipelines
+    pytest
+  displayName: 'pytest'

From 02206e5ffe74fe8107c272ad35920928544323b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Fri, 27 Mar 2020 02:20:39 -0300
Subject: [PATCH 042/243] Run tox

---
 azure-pipelines.yml        | 9 ++++++---
 tests/requirements-py3.txt | 1 +
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 9bc324c32..489cfe53b 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -10,10 +10,13 @@ strategy:
   matrix:
     Python35:
       python.version: '3.5'
+      TOXENV: py35
     Python36:
       python.version: '3.6'
+      TOXENV: py36
     Python37:
       python.version: '3.7'
+      TOXENV: py37
 
 steps:
 - task: UsePythonVersion@0
@@ -27,6 +30,6 @@ steps:
   displayName: 'Install dependencies'
 
 - script: |
-    pip install pytest pytest-azurepipelines
-    pytest
-  displayName: 'pytest'
+    pip install -U tox twine wheel codecov
+    tox
+  displayName: 'Run test suite'
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index d207c5fb0..8896f4614 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -6,6 +6,7 @@ pytest < 5.4
 pytest-cov
 pytest-twisted >= 1.11
 pytest-xdist
+pytest-azurepipelines
 sybil
 testfixtures
 

From 0699e6bb1600ff943e131a3b7a299aa038145198 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Fri, 27 Mar 2020 02:22:05 -0300
Subject: [PATCH 043/243] no need to install requirements.txt

---
 azure-pipelines.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 489cfe53b..ffc4d549b 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -24,11 +24,6 @@ steps:
     versionSpec: '$(python.version)'
   displayName: 'Use Python $(python.version)'
 
-- script: |
-    python -m pip install --upgrade pip
-    pip install -r requirements.txt
-  displayName: 'Install dependencies'
-
 - script: |
     pip install -U tox twine wheel codecov
     tox

From 3fb0027138ab44b16be6e11677626d69b8c90c95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Sat, 28 Mar 2020 17:36:50 +0100
Subject: [PATCH 044/243] =?UTF-8?q?Require=20sybil=20=E2=89=A5=201.3.0=20f?=
 =?UTF-8?q?or=20tests?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 tests/requirements-py3.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index d207c5fb0..e7c86e0e9 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -6,7 +6,7 @@ pytest < 5.4
 pytest-cov
 pytest-twisted >= 1.11
 pytest-xdist
-sybil
+sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
 testfixtures
 
 # optional for shell wrapper tests

From c4a5e3f0da3e674ecb7393c0894098984d6aa571 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 6 Apr 2020 09:26:13 -0300
Subject: [PATCH 045/243] Simplify bytes_received signal

Remove "source" parameter
---
 docs/topics/signals.rst                   |  6 +-----
 scrapy/core/downloader/handlers/http11.py | 18 +++++-------------
 scrapy/core/downloader/handlers/s3.py     |  1 -
 tests/test_engine.py                      |  5 +----
 4 files changed, 7 insertions(+), 23 deletions(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 97be46f2a..02fa6e287 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -163,7 +163,7 @@ bytes_received
 --------------
 
 .. signal:: bytes_received
-.. function:: bytes_received(data, request, spider, source)
+.. function:: bytes_received(data, request, spider)
 
     Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is
     received for a specific request. This signal might be fired multiple
@@ -180,10 +180,6 @@ bytes_received
     :param spider: the spider associated with the response
     :type spider: :class:`~scrapy.spiders.Spider` object
 
-    :param source: a string to identify which handler sent the signal
-        (current values could be "http11" or "s3")
-    :type source: :class:`str` object
-
 spider_closed
 -------------
 
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index c350cd3c2..bda21b6b9 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -33,9 +33,8 @@ logger = logging.getLogger(__name__)
 class HTTP11DownloadHandler:
     lazy = False
 
-    def __init__(self, settings, crawler=None, source="http11"):
+    def __init__(self, settings, crawler=None):
         self._crawler = crawler
-        self._source = source
 
         from twisted.internet import reactor
         self._pool = HTTPConnectionPool(reactor, persistent=True)
@@ -71,8 +70,8 @@ class HTTP11DownloadHandler:
         self._disconnect_timeout = 1
 
     @classmethod
-    def from_crawler(cls, crawler, **kwargs):
-        return cls(crawler.settings, crawler, **kwargs)
+    def from_crawler(cls, crawler):
+        return cls(crawler.settings, crawler)
 
     def download_request(self, request, spider):
         """Return a deferred for the HTTP download"""
@@ -83,7 +82,6 @@ class HTTP11DownloadHandler:
             warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
             fail_on_dataloss=self._fail_on_dataloss,
             crawler=self._crawler,
-            source=self._source,
         )
         return agent.download_request(request)
 
@@ -281,7 +279,7 @@ class ScrapyAgent:
     _TunnelingAgent = TunnelingAgent
 
     def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None,
-                 maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None, source=None):
+                 maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None):
         self._contextFactory = contextFactory
         self._connectTimeout = connectTimeout
         self._bindAddress = bindAddress
@@ -291,7 +289,6 @@ class ScrapyAgent:
         self._fail_on_dataloss = fail_on_dataloss
         self._txresponse = None
         self._crawler = crawler
-        self._source = source
 
     def _get_agent(self, request, timeout):
         from twisted.internet import reactor
@@ -430,7 +427,6 @@ class ScrapyAgent:
                 warnsize=warnsize,
                 fail_on_dataloss=fail_on_dataloss,
                 crawler=self._crawler,
-                source=self._source,
             )
         )
 
@@ -468,9 +464,7 @@ class _RequestBodyProducer:
 
 class _ResponseReader(protocol.Protocol):
 
-    def __init__(
-        self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler, source
-    ):
+    def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler):
         self._finished = finished
         self._txresponse = txresponse
         self._request = request
@@ -483,7 +477,6 @@ class _ResponseReader(protocol.Protocol):
         self._bytes_received = 0
         self._certificate = None
         self._crawler = crawler
-        self._source = source
 
     def connectionMade(self):
         if self._certificate is None:
@@ -503,7 +496,6 @@ class _ResponseReader(protocol.Protocol):
             data=bodyBytes,
             request=self._request,
             spider=self._crawler.spider,
-            source=self._source,
         )
 
         if self._maxsize and self._bytes_received > self._maxsize:
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 2366b6394..40a1fa48e 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -73,7 +73,6 @@ class S3DownloadHandler:
             objcls=httpdownloadhandler,
             settings=settings,
             crawler=crawler,
-            source="s3",
         )
         self._download_http = _http_handler.download_request
 
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 26f3163cf..acfe94f63 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -112,7 +112,6 @@ class CrawlerRun:
         self.itemerror = []
         self.itemresp = []
         self.bytes = defaultdict(lambda: list())
-        self.bytes_source = set()
         self.signals_caught = {}
         self.spider_class = spider_class
 
@@ -166,9 +165,8 @@ class CrawlerRun:
     def item_scraped(self, item, spider, response):
         self.itemresp.append((item, response))
 
-    def bytes_received(self, data, request, spider, source):
+    def bytes_received(self, data, request, spider):
         self.bytes[request].append(data)
-        self.bytes_source.add(source)
 
     def request_scheduled(self, request, spider):
         self.reqplug.append((request, spider))
@@ -281,7 +279,6 @@ class EngineTest(unittest.TestCase):
 
     def _assert_bytes_received(self):
         self.assertEqual(9, len(self.run.bytes))
-        self.assertEqual(self.run.bytes_source, set(["http11"]))
         for request, data in self.run.bytes.items():
             joined_data = b"".join(data)
             if self.run.getpath(request.url) == "/":

From 83a0cc6cdf4b8d55ebc594f2635beb75d93898cf Mon Sep 17 00:00:00 2001
From: ilias-ant <ilias.antonopoulos@yahoo.gr>
Date: Sun, 12 Apr 2020 23:22:17 +0300
Subject: [PATCH 046/243] Add status to files information

---
 docs/topics/media-pipeline.rst | 13 +++++++++++--
 scrapy/pipelines/files.py      |  4 ++--
 tests/test_pipeline_crawl.py   |  5 +++++
 3 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index cd84905c5..86550d7a4 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -50,7 +50,7 @@ this:
 4. When the files are downloaded, another field (``files``) will be populated
    with the results. This field will contain a list of dicts with information
    about the downloaded files, such as the downloaded path, the original
-   scraped url (taken from the ``file_urls`` field) , and the file checksum.
+   scraped url (taken from the ``file_urls`` field), the file checksum and the file status.
    The files in the list of the ``files`` field will retain the same order of
    the original ``file_urls`` field. If some file failed downloading, an
    error will be logged and the file won't be present in the ``files`` field.
@@ -470,6 +470,14 @@ See here the methods that you can override in your custom Files Pipeline:
 
         * ``checksum`` - a `MD5 hash`_ of the image contents
 
+        * ``status`` - the file status indication. It can be one of the following:
+
+          * ``downloaded`` - file was downloaded.
+          * ``uptodate`` - file was not downloaded, as it was downloaded recently,
+            according to the file expiration policy.
+          * ``cached`` - file was already scheduled for download, by another item
+            sharing the same file.
+
       The list of tuples received by :meth:`~item_completed` is
       guaranteed to retain the same order of the requests returned from the
       :meth:`~get_media_requests` method.
@@ -479,7 +487,8 @@ See here the methods that you can override in your custom Files Pipeline:
           [(True,
             {'checksum': '2b00042f7481c7b056c4b410d28f33cf',
              'path': 'full/0a79c461a4062ac383dc4fade7bc09f1384a3910.jpg',
-             'url': 'http://www.example.com/files/product1.pdf'}),
+             'url': 'http://www.example.com/files/product1.pdf',
+             'status': 'downloaded'}),
            (False,
             Failure(...))]
 
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 101bf5fbc..747dfa065 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -417,7 +417,7 @@ class FilesPipeline(MediaPipeline):
             self.inc_stats(info.spider, 'uptodate')
 
             checksum = result.get('checksum', None)
-            return {'url': request.url, 'path': path, 'checksum': checksum}
+            return {'url': request.url, 'path': path, 'checksum': checksum, 'status': 'uptodate'}
 
         path = self.file_path(request, info=info)
         dfd = defer.maybeDeferred(self.store.stat_file, path, info)
@@ -494,7 +494,7 @@ class FilesPipeline(MediaPipeline):
             )
             raise FileException(str(exc))
 
-        return {'url': request.url, 'path': path, 'checksum': checksum}
+        return {'url': request.url, 'path': path, 'checksum': checksum, 'status': status}
 
     def inc_stats(self, spider, status):
         spider.crawler.stats.inc_value('file_count', spider=spider)
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 962c33144..73b03e9d2 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -94,6 +94,11 @@ class FileDownloadCrawlTestCase(TestCase):
         file_dl_success = 'File (downloaded): Downloaded file from'
         self.assertEqual(logs.count(file_dl_success), 3)
 
+        # check that the images/files status is `downloaded`
+        for item in items:
+            for i in item[self.media_key]:
+                self.assertEqual(i['status'], 'downloaded')
+
         # check that the images/files checksums are what we know they should be
         if self.expected_checksums is not None:
             checksums = set(

From f242751b625c4a559d9e80f62a79223f911a8551 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 15 Apr 2020 09:39:28 -0300
Subject: [PATCH 047/243] Remove empty line

---
 pytest.ini | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pytest.ini b/pytest.ini
index 47f34e8f1..a9014ae55 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -68,7 +68,6 @@ flake8-ignore =
     # scrapy/loader
     scrapy/loader/__init__.py E128
     # scrapy/pipelines
-
     scrapy/pipelines/files.py E116
     scrapy/pipelines/media.py E125
     # scrapy/selector

From 2f510fd47d8d10217f7b18b531205dd8c252eaef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 15 Apr 2020 21:10:05 +0200
Subject: [PATCH 048/243] Fix ShellTest.test_local_file on Windows

---
 scrapy/utils/url.py         | 13 +++++++------
 tests/test_command_shell.py |  2 +-
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index c9abb12d5..c29ed4461 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -85,11 +85,9 @@ def add_http_if_no_scheme(url):
 
 
 def guess_scheme(url):
-    """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise."""
-    parts = urlparse(url)
-    if parts.scheme:
-        return url
-    # Note: this does not match Windows filepath
+    """Add an URL scheme if missing: file:// for filepath-like input or
+    http:// otherwise."""
+    # POSIX path
     if re.match(r'''^                   # start with...
                     (
                         \.              # ...a single dot,
@@ -99,7 +97,10 @@ def guess_scheme(url):
                     )?      # optional match of ".", ".." or ".blabla"
                     /       # at least one "/" for a file path,
                     .       # and something after the "/"
-                    ''', parts.path, flags=re.VERBOSE):
+                    ''', url, flags=re.VERBOSE):
+        return any_to_uri(url)
+    # Windows drive-letter path
+    elif re.match(r'''^[a-z]:\\''', url, flags=re.IGNORECASE):
         return any_to_uri(url)
     else:
         return add_http_if_no_scheme(url)
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index d664b6ade..acf8e9f71 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -94,7 +94,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_local_file(self):
-        filepath = join(tests_datadir, 'test_site/index.html')
+        filepath = join(tests_datadir, 'test_site', 'index.html')
         _, out, _ = yield self.execute([filepath, '-c', 'item'])
         assert b'{}' in out
 

From e5b23f4b00962df76d8302ebf869ef4a4319e142 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Wed, 18 Mar 2020 11:26:59 +0300
Subject: [PATCH 049/243] fix #4250: add batch deliveries

---
 scrapy/extensions/feedexport.py     | 54 +++++++++++++++++++++--------
 scrapy/settings/default_settings.py |  1 +
 2 files changed, 41 insertions(+), 14 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 998d2a5d1..906f99fee 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -241,6 +241,7 @@ class FeedExporter:
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
+        self.storage_batch = self.settings.getint('FEED_STORAGE_BATCH')
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
@@ -250,19 +251,7 @@ class FeedExporter:
     def open_spider(self, spider):
         for uri, feed in self.feeds.items():
             uri = uri % self._get_uri_params(spider, feed['uri_params'])
-            storage = self._get_storage(uri)
-            file = storage.open(spider)
-            exporter = self._get_exporter(
-                file=file,
-                format=feed['format'],
-                fields_to_export=feed['fields'],
-                encoding=feed['encoding'],
-                indent=feed['indent'],
-            )
-            slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'])
-            self.slots.append(slot)
-            if slot.store_empty:
-                slot.start_exporting()
+            self.slots.append(self._start_new_batch(None, uri, feed, spider))
 
     def close_spider(self, spider):
         deferred_list = []
@@ -285,11 +274,48 @@ class FeedExporter:
             deferred_list.append(d)
         return defer.DeferredList(deferred_list) if deferred_list else None
 
+    def _start_new_batch(self, previous_batch_slot, uri, feed, spider):
+        """
+        Redirect the output data stream to a new file.
+        Execute multiple times if 'FEED_STORAGE_BATCH' setting is greater than zero.
+        """
+        if previous_batch_slot is not None:
+            previous_batch_slot.exporter.finish_exporting()
+            previous_batch_slot.storage.store(previous_batch_slot.file)
+        storage = self._get_storage(uri)
+        file = storage.open(spider)
+        exporter = self._get_exporter(
+            file=file,
+            format=feed['format'],
+            fields_to_export=feed['fields'],
+            encoding=feed['encoding'],
+            indent=feed['indent']
+        )
+        slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'])
+        if slot.store_empty:
+            slot.start_exporting()
+        return slot
+
+    def _get_uri_of_partial(self, slot, feed, spider):
+        """Get uri for each partial using datetime.now().isoformat()"""
+        uri = (slot.uri % self._get_uri_params(spider, feed['uri_params'])).split('.')[0] + '.'
+        uri = uri + datetime.now().isoformat() + '.' + feed['format']
+        return uri
+
     def item_scraped(self, item, spider):
-        for slot in self.slots:
+        slots = []
+        for idx, slot in enumerate(self.slots):
             slot.start_exporting()
             slot.exporter.export_item(item)
             slot.itemcount += 1
+            if self.storage_batch and slot.itemcount % self.storage_batch == 0:
+                uri = self._get_uri_of_partial(slot, self.feeds[slot.uri], spider)
+                slots.append(self._start_new_batch(slot, uri, self.feeds[slot.uri], spider))
+                self.feeds[uri] = self.feeds[slot.uri]
+                self.feeds.pop(slot.uri)
+                self.slots[idx] = None
+        self.slots = [slot for slot in self.slots if slot is not None]
+        self.slots.extend(slots)
 
     def _load_components(self, setting_prefix):
         conf = without_none_values(self.settings.getwithbase(setting_prefix))
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 077317c81..690e044c5 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -146,6 +146,7 @@ FEED_STORAGES_BASE = {
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
 }
+FEED_STORAGE_BATCH = 0
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
     'json': 'scrapy.exporters.JsonItemExporter',

From 8b4566ff93843cdf17ada069dc09261a99971d26 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Wed, 18 Mar 2020 14:21:21 +0300
Subject: [PATCH 050/243] fix wrong name of first file in partial deliveries

---
 scrapy/extensions/feedexport.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 906f99fee..4f7c6bf07 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -249,6 +249,8 @@ class FeedExporter:
                 raise NotConfigured
 
     def open_spider(self, spider):
+        if self.storage_batch:
+            self.feeds = {self._get_uri_of_partial(uri, feed, spider): feed for uri, feed in self.feeds.items()}
         for uri, feed in self.feeds.items():
             uri = uri % self._get_uri_params(spider, feed['uri_params'])
             self.slots.append(self._start_new_batch(None, uri, feed, spider))
@@ -296,11 +298,11 @@ class FeedExporter:
             slot.start_exporting()
         return slot
 
-    def _get_uri_of_partial(self, slot, feed, spider):
+    def _get_uri_of_partial(self, template_uri, feed, spider):
         """Get uri for each partial using datetime.now().isoformat()"""
-        uri = (slot.uri % self._get_uri_params(spider, feed['uri_params'])).split('.')[0] + '.'
-        uri = uri + datetime.now().isoformat() + '.' + feed['format']
-        return uri
+        template_uri = (template_uri % self._get_uri_params(spider, feed['uri_params']))
+        uri_name = template_uri.split('.')[0]
+        return '{}.{}.{}'.format(uri_name, datetime.now().isoformat(), feed["format"])
 
     def item_scraped(self, item, spider):
         slots = []
@@ -309,11 +311,12 @@ class FeedExporter:
             slot.exporter.export_item(item)
             slot.itemcount += 1
             if self.storage_batch and slot.itemcount % self.storage_batch == 0:
-                uri = self._get_uri_of_partial(slot, self.feeds[slot.uri], spider)
+                uri = self._get_uri_of_partial(slot.uri, self.feeds[slot.uri], spider)
                 slots.append(self._start_new_batch(slot, uri, self.feeds[slot.uri], spider))
                 self.feeds[uri] = self.feeds[slot.uri]
                 self.feeds.pop(slot.uri)
                 self.slots[idx] = None
+
         self.slots = [slot for slot in self.slots if slot is not None]
         self.slots.extend(slots)
 

From 0723e3f4f9777a87d0df3b2e2fddfeac9099dd3b Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 19 Mar 2020 21:17:02 +0300
Subject: [PATCH 051/243] add batch_id, add error if uri is specified
 incorrectly

---
 scrapy/extensions/feedexport.py     | 73 ++++++++++++++++++++---------
 scrapy/settings/default_settings.py |  2 +-
 2 files changed, 52 insertions(+), 23 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 4f7c6bf07..38b25bf4a 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -180,14 +180,16 @@ class FTPFeedStorage(BlockingFeedStorage):
 
 
 class _FeedSlot:
-    def __init__(self, file, exporter, storage, uri, format, store_empty):
+    def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, template_uri):
         self.file = file
         self.exporter = exporter
         self.storage = storage
         # feed params
-        self.uri = uri
+        self.batch_id = batch_id
         self.format = format
         self.store_empty = store_empty
+        self.template_uri = template_uri
+        self.uri = uri
         # flags
         self.itemcount = 0
         self._exporting = False
@@ -241,19 +243,28 @@ class FeedExporter:
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
-        self.storage_batch = self.settings.getint('FEED_STORAGE_BATCH')
+        self.storage_batch_size = self.settings.getint('FEED_STORAGE_BATCH_SIZE')
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
+            if not self._batch_deliveries_supported(uri):
+                raise NotConfigured
             if not self._exporter_supported(feed['format']):
                 raise NotConfigured
 
     def open_spider(self, spider):
-        if self.storage_batch:
-            self.feeds = {self._get_uri_of_partial(uri, feed, spider): feed for uri, feed in self.feeds.items()}
         for uri, feed in self.feeds.items():
-            uri = uri % self._get_uri_params(spider, feed['uri_params'])
-            self.slots.append(self._start_new_batch(None, uri, feed, spider))
+            batch_id = 1
+            uri_params = self._get_uri_params(spider, feed['uri_params'])
+            uri_params['batch_id'] = batch_id
+            self.slots.append(self._start_new_batch(
+                previous_batch_slot=None,
+                uri=uri % uri_params,
+                feed=feed,
+                spider=spider,
+                batch_id=batch_id,
+                template_uri=uri
+            ))
 
     def close_spider(self, spider):
         deferred_list = []
@@ -276,10 +287,17 @@ class FeedExporter:
             deferred_list.append(d)
         return defer.DeferredList(deferred_list) if deferred_list else None
 
-    def _start_new_batch(self, previous_batch_slot, uri, feed, spider):
+    def _start_new_batch(self, previous_batch_slot, uri, feed, spider, batch_id, template_uri):
         """
         Redirect the output data stream to a new file.
         Execute multiple times if 'FEED_STORAGE_BATCH' setting is greater than zero.
+        :param previous_batch_slot: slot of previous batch. We need to call slot.storage.store
+        to get the file properly closed.
+        :param uri: uri of the new batch to start
+        :param feed: dict with parameters of feed
+        :param spider: user spider
+        :param batch_id: sequential batch id starting at 1
+        :param template_uri: template uri which contains %(time)s or %(batch_id)s to create new uri
         """
         if previous_batch_slot is not None:
             previous_batch_slot.exporter.finish_exporting()
@@ -293,30 +311,30 @@ class FeedExporter:
             encoding=feed['encoding'],
             indent=feed['indent']
         )
-        slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'])
+        slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'], batch_id, template_uri)
         if slot.store_empty:
             slot.start_exporting()
         return slot
 
-    def _get_uri_of_partial(self, template_uri, feed, spider):
-        """Get uri for each partial using datetime.now().isoformat()"""
-        template_uri = (template_uri % self._get_uri_params(spider, feed['uri_params']))
-        uri_name = template_uri.split('.')[0]
-        return '{}.{}.{}'.format(uri_name, datetime.now().isoformat(), feed["format"])
-
     def item_scraped(self, item, spider):
         slots = []
         for idx, slot in enumerate(self.slots):
             slot.start_exporting()
             slot.exporter.export_item(item)
             slot.itemcount += 1
-            if self.storage_batch and slot.itemcount % self.storage_batch == 0:
-                uri = self._get_uri_of_partial(slot.uri, self.feeds[slot.uri], spider)
-                slots.append(self._start_new_batch(slot, uri, self.feeds[slot.uri], spider))
-                self.feeds[uri] = self.feeds[slot.uri]
-                self.feeds.pop(slot.uri)
+            if self.storage_batch_size and slot.itemcount % self.storage_batch_size == 0:
+                batch_id = slot.batch_id + 1
+                uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'])
+                uri_params['batch_id'] = batch_id
+                self.slots.append(self._start_new_batch(
+                    previous_batch_slot=slot,
+                    uri=slot.template_uri % uri_params,
+                    feed=self.feeds[slot.template_uri],
+                    spider=spider,
+                    batch_id=batch_id,
+                    template_uri=slot.template_uri
+                ))
                 self.slots[idx] = None
-
         self.slots = [slot for slot in self.slots if slot is not None]
         self.slots.extend(slots)
 
@@ -335,6 +353,17 @@ class FeedExporter:
             return True
         logger.error("Unknown feed format: %(format)s", {'format': format})
 
+    def _batch_deliveries_supported(self, uri):
+        """
+        If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time)s or %(batch_id)s
+        to distinguish different files of partial output
+        """
+        if not self.storage_batch_size:
+            return True
+        if '%(time)s' in uri or '%(batch_id)s' in uri:
+            return True
+        logger.error('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
+
     def _storage_supported(self, uri):
         scheme = urlparse(uri).scheme
         if scheme in self.storages:
@@ -364,7 +393,7 @@ class FeedExporter:
         params = {}
         for k in dir(spider):
             params[k] = getattr(spider, k)
-        ts = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')
+        ts = datetime.utcnow().isoformat().replace(':', '-')
         params['time'] = ts
         uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
         uripar_function(params, spider)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 690e044c5..7f90a2280 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -146,7 +146,7 @@ FEED_STORAGES_BASE = {
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
 }
-FEED_STORAGE_BATCH = 0
+FEED_STORAGE_BATCH_SIZE = 0
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
     'json': 'scrapy.exporters.JsonItemExporter',

From d11411b402ae68874c6ccc2883836be0b9cf8326 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Sat, 21 Mar 2020 10:48:13 +0300
Subject: [PATCH 052/243] fix comments

---
 scrapy/extensions/feedexport.py     | 31 ++++++++++++++++++-----------
 scrapy/settings/default_settings.py |  2 +-
 2 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 38b25bf4a..ab0a0de37 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -25,7 +25,6 @@ from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import without_none_values
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -243,7 +242,7 @@ class FeedExporter:
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
-        self.storage_batch_size = self.settings.getint('FEED_STORAGE_BATCH_SIZE')
+        self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_SIZE', None)
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
@@ -263,7 +262,7 @@ class FeedExporter:
                 feed=feed,
                 spider=spider,
                 batch_id=batch_id,
-                template_uri=uri
+                template_uri=uri,
             ))
 
     def close_spider(self, spider):
@@ -290,7 +289,7 @@ class FeedExporter:
     def _start_new_batch(self, previous_batch_slot, uri, feed, spider, batch_id, template_uri):
         """
         Redirect the output data stream to a new file.
-        Execute multiple times if 'FEED_STORAGE_BATCH' setting is greater than zero.
+        Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified.
         :param previous_batch_slot: slot of previous batch. We need to call slot.storage.store
         to get the file properly closed.
         :param uri: uri of the new batch to start
@@ -309,9 +308,18 @@ class FeedExporter:
             format=feed['format'],
             fields_to_export=feed['fields'],
             encoding=feed['encoding'],
-            indent=feed['indent']
+            indent=feed['indent'],
+        )
+        slot = _FeedSlot(
+            file=file,
+            exporter=exporter,
+            storage=storage,
+            uri=uri,
+            format=feed['format'],
+            store_empty=feed['store_empty'],
+            batch_id=batch_id,
+            template_uri=template_uri,
         )
-        slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'], batch_id, template_uri)
         if slot.store_empty:
             slot.start_exporting()
         return slot
@@ -326,13 +334,13 @@ class FeedExporter:
                 batch_id = slot.batch_id + 1
                 uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'])
                 uri_params['batch_id'] = batch_id
-                self.slots.append(self._start_new_batch(
+                slots.append(self._start_new_batch(
                     previous_batch_slot=slot,
                     uri=slot.template_uri % uri_params,
                     feed=self.feeds[slot.template_uri],
                     spider=spider,
                     batch_id=batch_id,
-                    template_uri=slot.template_uri
+                    template_uri=slot.template_uri,
                 ))
                 self.slots[idx] = None
         self.slots = [slot for slot in self.slots if slot is not None]
@@ -358,11 +366,10 @@ class FeedExporter:
         If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time)s or %(batch_id)s
         to distinguish different files of partial output
         """
-        if not self.storage_batch_size:
+        if self.storage_batch_size is None or '%(time)s' in uri or '%(batch_id)s' in uri:
             return True
-        if '%(time)s' in uri or '%(batch_id)s' in uri:
-            return True
-        logger.error('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
+        logger.warning('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
+        return False
 
     def _storage_supported(self, uri):
         scheme = urlparse(uri).scheme
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 7f90a2280..c3463a505 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -146,7 +146,7 @@ FEED_STORAGES_BASE = {
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
 }
-FEED_STORAGE_BATCH_SIZE = 0
+FEED_STORAGE_BATCH_SIZE = None
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
     'json': 'scrapy.exporters.JsonItemExporter',

From 39d0d13d3f7bd671d5b29646b209c62e23373fab Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 26 Mar 2020 14:18:35 +0300
Subject: [PATCH 053/243] Add partial deliveries tests

---
 tests/test_feedexport.py | 195 +++++++++++++++++++++++++++++++--------
 1 file changed, 159 insertions(+), 36 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index c5589e52f..1ebe44e12 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -6,6 +6,7 @@ import shutil
 import string
 import tempfile
 import warnings
+from abc import ABC, abstractmethod
 from io import BytesIO
 from pathlib import Path
 from string import ascii_letters, digits
@@ -21,8 +22,9 @@ from zope.interface.verify import verifyObject
 
 import scrapy
 from scrapy.crawler import CrawlerRunner
+from scrapy.exceptions import NotConfigured
 from scrapy.exporters import CsvItemExporter
-from scrapy.extensions.feedexport import (BlockingFeedStorage, FileFeedStorage, FTPFeedStorage,
+from scrapy.extensions.feedexport import (BlockingFeedStorage, FeedExporter, FileFeedStorage, FTPFeedStorage,
                                           IFeedStorage, S3FeedStorage, StdoutFeedStorage)
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
@@ -76,6 +78,7 @@ class FTPFeedStorageTest(unittest.TestCase):
     def get_test_spider(self, settings=None):
         class TestSpider(scrapy.Spider):
             name = 'test_spider'
+
         crawler = get_crawler(settings_dict=settings)
         spider = TestSpider.from_crawler(crawler)
         return spider
@@ -129,6 +132,7 @@ class BlockingFeedStorageTest(unittest.TestCase):
     def get_test_spider(self, settings=None):
         class TestSpider(scrapy.Spider):
             name = 'test_spider'
+
         crawler = get_crawler(settings_dict=settings)
         spider = TestSpider.from_crawler(crawler)
         return spider
@@ -390,23 +394,63 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):
     pass
 
 
-class FeedExportTest(unittest.TestCase):
+class FeedExportTestBase(ABC, unittest.TestCase):
+    __test__ = False
 
     class MyItem(scrapy.Item):
         foo = scrapy.Field()
         egg = scrapy.Field()
         baz = scrapy.Field()
 
+    def _random_temp_filename(self, inter_dir=''):
+        chars = [random.choice(ascii_letters + digits) for _ in range(15)]
+        filename = ''.join(chars)
+        return os.path.join(self.temp_dir, inter_dir, filename)
+
     def setUp(self):
         self.temp_dir = tempfile.mkdtemp()
 
     def tearDown(self):
         shutil.rmtree(self.temp_dir, ignore_errors=True)
 
-    def _random_temp_filename(self):
-        chars = [random.choice(ascii_letters + digits) for _ in range(15)]
-        filename = ''.join(chars)
-        return os.path.join(self.temp_dir, filename)
+    @defer.inlineCallbacks
+    def exported_data(self, items, settings):
+        """
+        Return exported data which a spider yielding ``items`` would return.
+        """
+
+        class TestSpider(scrapy.Spider):
+            name = 'testspider'
+
+            def parse(self, response):
+                for item in items:
+                    yield item
+
+        data = yield self.run_and_export(TestSpider, settings)
+        defer.returnValue(data)
+
+    @defer.inlineCallbacks
+    def exported_no_data(self, settings):
+        """
+        Return exported data which a spider yielding no ``items`` would return.
+        """
+
+        class TestSpider(scrapy.Spider):
+            name = 'testspider'
+
+            def parse(self, response):
+                pass
+
+        data = yield self.run_and_export(TestSpider, settings)
+        defer.returnValue(data)
+
+    @abstractmethod
+    def run_and_export(self, spider_cls, settings):
+        pass
+
+
+class FeedExportTest(FeedExportTestBase):
+    __test__ = True
 
     @defer.inlineCallbacks
     def run_and_export(self, spider_cls, settings):
@@ -417,7 +461,6 @@ class FeedExportTest(unittest.TestCase):
             urljoin('file:', pathname2url(str(file_path))): feed
             for file_path, feed in FEEDS.items()
         }
-
         content = {}
         try:
             with MockServer() as s:
@@ -435,35 +478,6 @@ class FeedExportTest(unittest.TestCase):
 
         defer.returnValue(content)
 
-    @defer.inlineCallbacks
-    def exported_data(self, items, settings):
-        """
-        Return exported data which a spider yielding ``items`` would return.
-        """
-        class TestSpider(scrapy.Spider):
-            name = 'testspider'
-
-            def parse(self, response):
-                for item in items:
-                    yield item
-
-        data = yield self.run_and_export(TestSpider, settings)
-        defer.returnValue(data)
-
-    @defer.inlineCallbacks
-    def exported_no_data(self, settings):
-        """
-        Return exported data which a spider yielding no ``items`` would return.
-        """
-        class TestSpider(scrapy.Spider):
-            name = 'testspider'
-
-            def parse(self, response):
-                pass
-
-        data = yield self.run_and_export(TestSpider, settings)
-        defer.returnValue(data)
-
     @defer.inlineCallbacks
     def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
         settings = settings or {}
@@ -970,3 +984,112 @@ class FeedExportTest(unittest.TestCase):
         }
         data = yield self.exported_no_data(settings)
         self.assertEqual(data['csv'], b'')
+
+
+class PartialDeliveriesTest(FeedExportTestBase):
+    __test__ = True
+    _file_mark = '_%(time)s_#%(batch_id)s'
+
+    @defer.inlineCallbacks
+    def run_and_export(self, spider_cls, settings):
+        """ Run spider with specified settings; return exported data. """
+
+        FEEDS = settings.get('FEEDS') or {}
+        settings['FEEDS'] = {
+            urljoin('file:', file_path): feed
+            for file_path, feed in FEEDS.items()
+        }
+        from collections import defaultdict
+        content = defaultdict(list)
+        try:
+            with MockServer() as s:
+                runner = CrawlerRunner(Settings(settings))
+                spider_cls.start_urls = [s.url('/')]
+                yield runner.crawl(spider_cls)
+
+            for path, feed in FEEDS.items():
+                dir_name = os.path.dirname(path)
+                for file in sorted(os.listdir(dir_name)):
+                    with open(os.path.join(dir_name, file), 'rb') as f:
+                        data = f.read()
+                        content[feed['format']].append(data)
+        finally:
+            pass
+        defer.returnValue(content)
+
+    @defer.inlineCallbacks
+    def assertPartialExported(self, items, rows, settings=None):
+        settings = settings or {}
+        settings.update({
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
+            },
+        })
+        data = yield self.exported_data(items, settings)
+        data['jl'] = b''.join(data['jl'])
+        parsed = [json.loads(to_unicode(line)) for line in data['jl'].splitlines()]
+
+        rows = [{k: v for k, v in row.items() if v} for row in rows]
+        self.assertEqual(rows, parsed)
+
+    @defer.inlineCallbacks
+    def test_partial_deliveries(self):
+        items = [
+            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+            self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
+            self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
+        ]
+        rows = [
+            {'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
+            {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'},
+            {'foo': 'bar3', 'baz': 'quux3'}
+        ]
+        settings = {
+            'FEED_STORAGE_BATCH_SIZE': 1
+        }
+        yield self.assertPartialExported(items, rows, settings=settings)
+
+    def test_wrong_path(self):
+        settings = {
+            'FEEDS': {
+                self._random_temp_filename(): {'format': 'xml'},
+            },
+            'FEED_STORAGE_BATCH_SIZE': 1
+        }
+        crawler = get_crawler(settings_dict=settings)
+        self.assertRaises(NotConfigured, FeedExporter, crawler)
+
+    @defer.inlineCallbacks
+    def test_export_no_items_not_store_empty(self):
+        for fmt in ('json', 'jsonlines', 'xml', 'csv'):
+            settings = {
+                'FEEDS': {
+                    os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
+                },
+                'FEED_STORAGE_BATCH_SIZE': 1
+            }
+            data = yield self.exported_no_data(settings)
+            data[fmt] = b''.join(data[fmt])
+            self.assertEqual(data[fmt], b'')
+
+    @defer.inlineCallbacks
+    def test_export_no_items_store_empty(self):
+        formats = (
+            ('json', b'[]'),
+            ('jsonlines', b''),
+            ('xml', b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),
+            ('csv', b''),
+        )
+
+        for fmt, expctd in formats:
+            settings = {
+                'FEEDS': {
+                    os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
+                },
+                'FEED_STORE_EMPTY': True,
+                'FEED_EXPORT_INDENT': None,
+                'FEED_STORAGE_BATCH_SIZE': 1
+            }
+            data = yield self.exported_no_data(settings)
+            data[fmt] = b''.join(data[fmt])
+            self.assertEqual(data[fmt], expctd)

From ffa8a533e74478a5c81fbf453f2c65601bb1d244 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Sat, 28 Mar 2020 11:40:16 +0300
Subject: [PATCH 054/243] Set batch_id in _get_uri_params

---
 scrapy/extensions/feedexport.py | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index ab0a0de37..06ea6c5b2 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -253,15 +253,12 @@ class FeedExporter:
 
     def open_spider(self, spider):
         for uri, feed in self.feeds.items():
-            batch_id = 1
-            uri_params = self._get_uri_params(spider, feed['uri_params'])
-            uri_params['batch_id'] = batch_id
+            uri_params = self._get_uri_params(spider, feed['uri_params'], None)
             self.slots.append(self._start_new_batch(
                 previous_batch_slot=None,
                 uri=uri % uri_params,
                 feed=feed,
                 spider=spider,
-                batch_id=batch_id,
                 template_uri=uri,
             ))
 
@@ -286,7 +283,7 @@ class FeedExporter:
             deferred_list.append(d)
         return defer.DeferredList(deferred_list) if deferred_list else None
 
-    def _start_new_batch(self, previous_batch_slot, uri, feed, spider, batch_id, template_uri):
+    def _start_new_batch(self, previous_batch_slot, uri, feed, spider, template_uri):
         """
         Redirect the output data stream to a new file.
         Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified.
@@ -295,12 +292,15 @@ class FeedExporter:
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
         :param spider: user spider
-        :param batch_id: sequential batch id starting at 1
         :param template_uri: template uri which contains %(time)s or %(batch_id)s to create new uri
         """
         if previous_batch_slot is not None:
+            previous_batch_id = previous_batch_slot.batch_id
             previous_batch_slot.exporter.finish_exporting()
             previous_batch_slot.storage.store(previous_batch_slot.file)
+        else:
+            previous_batch_id = 0
+
         storage = self._get_storage(uri)
         file = storage.open(spider)
         exporter = self._get_exporter(
@@ -317,7 +317,7 @@ class FeedExporter:
             uri=uri,
             format=feed['format'],
             store_empty=feed['store_empty'],
-            batch_id=batch_id,
+            batch_id=previous_batch_id + 1,
             template_uri=template_uri,
         )
         if slot.store_empty:
@@ -331,15 +331,12 @@ class FeedExporter:
             slot.exporter.export_item(item)
             slot.itemcount += 1
             if self.storage_batch_size and slot.itemcount % self.storage_batch_size == 0:
-                batch_id = slot.batch_id + 1
-                uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'])
-                uri_params['batch_id'] = batch_id
+                uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot)
                 slots.append(self._start_new_batch(
                     previous_batch_slot=slot,
                     uri=slot.template_uri % uri_params,
                     feed=self.feeds[slot.template_uri],
                     spider=spider,
-                    batch_id=batch_id,
                     template_uri=slot.template_uri,
                 ))
                 self.slots[idx] = None
@@ -396,12 +393,12 @@ class FeedExporter:
     def _get_storage(self, uri):
         return self._get_instance(self.storages[urlparse(uri).scheme], uri)
 
-    def _get_uri_params(self, spider, uri_params):
+    def _get_uri_params(self, spider, uri_params, slot):
         params = {}
         for k in dir(spider):
             params[k] = getattr(spider, k)
-        ts = datetime.utcnow().isoformat().replace(':', '-')
-        params['time'] = ts
+        params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
+        params['time'] = datetime.utcnow().isoformat().replace(':', '-')
         uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
         uripar_function(params, spider)
         return params

From 963580463b96315eb58319e6d35b4cd52672371a Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Wed, 15 Apr 2020 20:14:33 +0300
Subject: [PATCH 055/243] Update tests

---
 tests/test_feedexport.py | 203 ++++++++++++++++++++++++++++++---------
 1 file changed, 159 insertions(+), 44 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 1ebe44e12..c6cd867b1 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -7,6 +7,7 @@ import string
 import tempfile
 import warnings
 from abc import ABC, abstractmethod
+from collections import defaultdict
 from io import BytesIO
 from pathlib import Path
 from string import ascii_letters, digits
@@ -444,10 +445,31 @@ class FeedExportTestBase(ABC, unittest.TestCase):
         data = yield self.run_and_export(TestSpider, settings)
         defer.returnValue(data)
 
+    @defer.inlineCallbacks
+    def assertExported(self, items, header, rows, settings=None, ordered=True):
+        yield self.assertExportedCsv(items, header, rows, settings, ordered)
+        yield self.assertExportedJsonLines(items, rows, settings)
+        yield self.assertExportedXml(items, rows, settings)
+        yield self.assertExportedPickle(items, rows, settings)
+        yield self.assertExportedMarshal(items, rows, settings)
+        yield self.assertExportedMultiple(items, rows, settings)
+
     @abstractmethod
     def run_and_export(self, spider_cls, settings):
         pass
 
+    def _load_until_eof(self, data, load_func):
+        result = []
+        with tempfile.TemporaryFile() as temp:
+            temp.write(data)
+            temp.seek(0)
+            while True:
+                try:
+                    result.append(load_func(temp))
+                except EOFError:
+                    break
+        return result
+
 
 class FeedExportTest(FeedExportTestBase):
     __test__ = True
@@ -478,6 +500,22 @@ class FeedExportTest(FeedExportTestBase):
 
         defer.returnValue(content)
 
+    @defer.inlineCallbacks
+    def exported_data(self, items, settings):
+        """
+        Return exported data which a spider yielding ``items`` would return.
+        """
+
+        class TestSpider(scrapy.Spider):
+            name = 'testspider'
+
+            def parse(self, response):
+                for item in items:
+                    yield item
+
+        data = yield self.run_and_export(TestSpider, settings)
+        defer.returnValue(data)
+
     @defer.inlineCallbacks
     def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
         settings = settings or {}
@@ -543,18 +581,6 @@ class FeedExportTest(FeedExportTestBase):
         json_rows = json.loads(to_unicode(data['json']))
         self.assertEqual(rows, json_rows)
 
-    def _load_until_eof(self, data, load_func):
-        result = []
-        with tempfile.TemporaryFile() as temp:
-            temp.write(data)
-            temp.seek(0)
-            while True:
-                try:
-                    result.append(load_func(temp))
-                except EOFError:
-                    break
-        return result
-
     @defer.inlineCallbacks
     def assertExportedPickle(self, items, rows, settings=None):
         settings = settings or {}
@@ -583,15 +609,6 @@ class FeedExportTest(FeedExportTestBase):
         result = self._load_until_eof(data['marshal'], load_func=marshal.load)
         self.assertEqual(expected, result)
 
-    @defer.inlineCallbacks
-    def assertExported(self, items, header, rows, settings=None, ordered=True):
-        yield self.assertExportedCsv(items, header, rows, settings, ordered)
-        yield self.assertExportedJsonLines(items, rows, settings)
-        yield self.assertExportedXml(items, rows, settings)
-        yield self.assertExportedPickle(items, rows, settings)
-        yield self.assertExportedMarshal(items, rows, settings)
-        yield self.assertExportedMultiple(items, rows, settings)
-
     @defer.inlineCallbacks
     def test_export_items(self):
         # feed exporters use field names from Item
@@ -615,7 +632,7 @@ class FeedExportTest(FeedExportTestBase):
                 },
             }
             data = yield self.exported_no_data(settings)
-            self.assertEqual(data[fmt], b'')
+            self.assertEqual(b'', data[fmt])
 
     @defer.inlineCallbacks
     def test_export_no_items_store_empty(self):
@@ -635,7 +652,7 @@ class FeedExportTest(FeedExportTestBase):
                 'FEED_EXPORT_INDENT': None,
             }
             data = yield self.exported_no_data(settings)
-            self.assertEqual(data[fmt], expctd)
+            self.assertEqual(expctd, data[fmt])
 
     @defer.inlineCallbacks
     def test_export_multiple_item_classes(self):
@@ -734,7 +751,8 @@ class FeedExportTest(FeedExportTestBase):
         formats = {
             'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
             'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'),
-            'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'),
+            'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode(
+                'utf-8'),
             'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),
         }
 
@@ -751,7 +769,8 @@ class FeedExportTest(FeedExportTestBase):
         formats = {
             'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'),
             'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'),
-            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'),
+            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode(
+                'latin-1'),
             'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'),
         }
 
@@ -772,7 +791,8 @@ class FeedExportTest(FeedExportTestBase):
 
         formats = {
             'json': u'[\n{"bar": "BAR"}\n]'.encode('utf-8'),
-            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'.encode('latin-1'),
+            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'.encode(
+                'latin-1'),
             'csv': u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
         }
 
@@ -988,7 +1008,7 @@ class FeedExportTest(FeedExportTestBase):
 
 class PartialDeliveriesTest(FeedExportTestBase):
     __test__ = True
-    _file_mark = '_%(time)s_#%(batch_id)s'
+    _file_mark = '_%(time)s_#%(batch_id)s_'
 
     @defer.inlineCallbacks
     def run_and_export(self, spider_cls, settings):
@@ -999,7 +1019,6 @@ class PartialDeliveriesTest(FeedExportTestBase):
             urljoin('file:', file_path): feed
             for file_path, feed in FEEDS.items()
         }
-        from collections import defaultdict
         content = defaultdict(list)
         try:
             with MockServer() as s:
@@ -1014,26 +1033,120 @@ class PartialDeliveriesTest(FeedExportTestBase):
                         data = f.read()
                         content[feed['format']].append(data)
         finally:
-            pass
+            self.tearDown()
         defer.returnValue(content)
 
     @defer.inlineCallbacks
-    def assertPartialExported(self, items, rows, settings=None):
+    def assertExportedJsonLines(self, items, rows, settings=None):
         settings = settings or {}
         settings.update({
             'FEEDS': {
                 os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
             },
         })
-        data = yield self.exported_data(items, settings)
-        data['jl'] = b''.join(data['jl'])
-        parsed = [json.loads(to_unicode(line)) for line in data['jl'].splitlines()]
-
+        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
         rows = [{k: v for k, v in row.items() if v} for row in rows]
-        self.assertEqual(rows, parsed)
+        data = yield self.exported_data(items, settings)
+        for batch in data['jl']:
+            got_batch = [json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines()]
+            expected_batch, rows = rows[:batch_size], rows[batch_size:]
+            self.assertEqual(expected_batch, got_batch)
 
     @defer.inlineCallbacks
-    def test_partial_deliveries(self):
+    def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
+        settings = settings or {}
+        settings.update({
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
+            },
+        })
+        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        data = yield self.exported_data(items, settings)
+        for batch in data['csv']:
+            got_batch = csv.DictReader(to_unicode(batch).splitlines())
+            self.assertEqual(list(header), got_batch.fieldnames)
+            expected_batch, rows = rows[:batch_size], rows[batch_size:]
+            self.assertEqual(expected_batch, list(got_batch))
+
+    @defer.inlineCallbacks
+    def assertExportedXml(self, items, rows, settings=None):
+        settings = settings or {}
+        settings.update({
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
+            },
+        })
+        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        rows = [{k: v for k, v in row.items() if v} for row in rows]
+        data = yield self.exported_data(items, settings)
+        for batch in data['xml']:
+            root = lxml.etree.fromstring(batch)
+            got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')]
+            expected_batch, rows = rows[:batch_size], rows[batch_size:]
+            self.assertEqual(expected_batch, got_batch)
+
+    @defer.inlineCallbacks
+    def assertExportedMultiple(self, items, rows, settings=None):
+        settings = settings or {}
+        settings.update({
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
+                os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
+            },
+        })
+        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        rows = [{k: v for k, v in row.items() if v} for row in rows]
+        data = yield self.exported_data(items, settings)
+        # XML
+        xml_rows = rows.copy()
+        for batch in data['xml']:
+            root = lxml.etree.fromstring(batch)
+            got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')]
+            expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:]
+            self.assertEqual(expected_batch, got_batch)
+        # JSON
+        json_rows = rows.copy()
+        for batch in data['json']:
+            got_batch = json.loads(batch)
+            expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:]
+            self.assertEqual(expected_batch, got_batch)
+
+    @defer.inlineCallbacks
+    def assertExportedPickle(self, items, rows, settings=None):
+        settings = settings or {}
+        settings.update({
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
+            },
+        })
+        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        rows = [{k: v for k, v in row.items() if v} for row in rows]
+        data = yield self.exported_data(items, settings)
+        import pickle
+        for batch in data['pickle']:
+            got_batch = self._load_until_eof(batch, load_func=pickle.load)
+            expected_batch, rows = rows[:batch_size], rows[batch_size:]
+            self.assertEqual(expected_batch, got_batch)
+
+    @defer.inlineCallbacks
+    def assertExportedMarshal(self, items, rows, settings=None):
+        settings = settings or {}
+        settings.update({
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
+            },
+        })
+        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        rows = [{k: v for k, v in row.items() if v} for row in rows]
+        data = yield self.exported_data(items, settings)
+        import marshal
+        for batch in data['marshal']:
+            got_batch = self._load_until_eof(batch, load_func=marshal.load)
+            expected_batch, rows = rows[:batch_size], rows[batch_size:]
+            self.assertEqual(expected_batch, got_batch)
+
+    @defer.inlineCallbacks
+    def test_export_items(self):
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
             self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
@@ -1042,14 +1155,16 @@ class PartialDeliveriesTest(FeedExportTestBase):
         rows = [
             {'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
             {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'},
-            {'foo': 'bar3', 'baz': 'quux3'}
+            {'foo': 'bar3', 'baz': 'quux3', 'egg': ''}
         ]
         settings = {
-            'FEED_STORAGE_BATCH_SIZE': 1
+            'FEED_STORAGE_BATCH_SIZE': 2
         }
-        yield self.assertPartialExported(items, rows, settings=settings)
+        header = self.MyItem.fields.keys()
+        yield self.assertExported(items, header, rows, settings=settings)
 
     def test_wrong_path(self):
+        """If path without %(time)s or %(batch_id)s an exception must be raised"""
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},
@@ -1069,8 +1184,8 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 'FEED_STORAGE_BATCH_SIZE': 1
             }
             data = yield self.exported_no_data(settings)
-            data[fmt] = b''.join(data[fmt])
-            self.assertEqual(data[fmt], b'')
+            data = dict(data)
+            self.assertEqual(b'', data[fmt][0])
 
     @defer.inlineCallbacks
     def test_export_no_items_store_empty(self):
@@ -1088,8 +1203,8 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 },
                 'FEED_STORE_EMPTY': True,
                 'FEED_EXPORT_INDENT': None,
-                'FEED_STORAGE_BATCH_SIZE': 1
+                'FEED_STORAGE_BATCH_SIZE': 1,
             }
             data = yield self.exported_no_data(settings)
-            data[fmt] = b''.join(data[fmt])
-            self.assertEqual(data[fmt], expctd)
+            data = dict(data)
+            self.assertEqual(expctd, data[fmt][0])

From cac1f3a6adedc32977e0fb1830917a5e7d758bef Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 16 Apr 2020 10:06:56 +0300
Subject: [PATCH 056/243] Update documentation

---
 docs/topics/feed-exports.rst | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 9e5968a29..0bba03a7c 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -220,6 +220,7 @@ These are the settings used for configuring the feed exports:
  * :setting:`FEED_STORAGE_FTP_ACTIVE`
  * :setting:`FEED_STORAGE_S3_ACL`
  * :setting:`FEED_EXPORTERS`
+ * :setting:`FEED_EXPORT_BATCH_SIZE`
 
 .. currentmodule:: scrapy.extensions.feedexport
 
@@ -429,3 +430,37 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 .. _Amazon S3: https://aws.amazon.com/s3/
 .. _botocore: https://github.com/boto/botocore
 .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+
+.. setting:: FEED_EXPORT_BATCH_SIZE
+
+FEED_EXPORT_BATCH_SIZE
+----------------------
+Default: ``None``
+
+An integer number which represent number of scraped items stored in each output
+file. Whenever the number of items exceeds this setting, a new file
+creates and output redirects to it.  The name of the new file will be selected
+based on timestamp when the feed is being created and/or batch sequence number.
+Therefore you must specify %(time)s or %(batch_id)s or both in the file path.
+
+* ``%(time)s`` - gets replaced by a timestamp when the feed is being created
+* ``%(batch_id)s`` - gets replaced by sequence number of batch
+
+For instance::
+
+    FEED_EXPORT_BATCH_SIZE=100
+
+Your request can be like::
+
+  scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time)s.json
+
+The result directory tree of above can be like::
+
+->projectname
+-->dirname
+--->1-filename2020-03-28T14-45-08.237134.json
+--->2-filename2020-03-28T14-45-09.148903.json
+--->3-filename2020-03-28T14-45-10.046092.json
+
+Where first and second files contain exactly 100 items. The last one contains
+<= 100 items.
\ No newline at end of file

From 5980ae72c6cb177f47fbb41d17837e8d98d50025 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 16 Apr 2020 10:13:39 +0300
Subject: [PATCH 057/243] Some minor fixes and refactoring

---
 tests/test_feedexport.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 60e19d1df..e97e50e8e 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -428,7 +428,7 @@ class FeedExportTestBase(ABC, unittest.TestCase):
                     yield item
 
         data = yield self.run_and_export(TestSpider, settings)
-        defer.returnValue(data)
+        return data
 
     @defer.inlineCallbacks
     def exported_no_data(self, settings):
@@ -443,7 +443,7 @@ class FeedExportTestBase(ABC, unittest.TestCase):
                 pass
 
         data = yield self.run_and_export(TestSpider, settings)
-        defer.returnValue(data)
+        return data
 
     @defer.inlineCallbacks
     def assertExported(self, items, header, rows, settings=None, ordered=True):
@@ -735,8 +735,7 @@ class FeedExportTest(FeedExportTestBase):
         formats = {
             'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
             'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'),
-            'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode(
-                'utf-8'),
+            'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'),
             'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),
         }
 
@@ -753,8 +752,7 @@ class FeedExportTest(FeedExportTestBase):
         formats = {
             'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'),
             'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'),
-            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode(
-                'latin-1'),
+            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'),
             'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'),
         }
 
@@ -775,8 +773,7 @@ class FeedExportTest(FeedExportTestBase):
 
         formats = {
             'json': u'[\n{"bar": "BAR"}\n]'.encode('utf-8'),
-            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'.encode(
-                'latin-1'),
+            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'.encode('latin-1'),
             'csv': u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
         }
 
@@ -1148,7 +1145,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
         yield self.assertExported(items, header, rows, settings=settings)
 
     def test_wrong_path(self):
-        """If path without %(time)s or %(batch_id)s an exception must be raised"""
+        """If path is without %(time)s or %(batch_id)s an exception must be raised"""
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},

From 94ee68695a42ec8d3ccdff71bfc2fa33d5ea7049 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 16 Apr 2020 15:36:56 +0200
Subject: [PATCH 058/243] Mock server: use 127.0.0.1 also for HTTPS

Windows throws an error about 0.0.0.0 being external:
https://stackoverflow.com/a/23857995/939364
---
 tests/mockserver.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index a45277db9..e3dbdcc68 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -218,9 +218,8 @@ class MockServer():
         self.proc.communicate()
 
     def url(self, path, is_secure=False):
-        host = self.http_address.replace('0.0.0.0', '127.0.0.1')
-        if is_secure:
-            host = self.https_address
+        host = self.https_address if is_secure else self.http_address
+        host = host.replace('0.0.0.0', '127.0.0.1')
         return host + path
 
 

From 7cc9601029274124804e63ebebfa8783ac175205 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 16 Apr 2020 16:57:48 +0200
Subject: [PATCH 059/243] Improve reporting on
 test_ipv6_alternative_name_resolver

---
 tests/test_crawler.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 169e763f0..a6b079395 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -309,14 +309,8 @@ class CrawlerProcessSubprocess(unittest.TestCase):
     def test_ipv6_alternative_name_resolver(self):
         log = self.run_script('alternative_name_resolver.py')
         self.assertIn('Spider closed (finished)', log)
-        self.assertTrue(any([
-            "twisted.internet.error.ConnectionRefusedError" in log,
-            "twisted.internet.error.ConnectError" in log,
-        ]))
-        self.assertTrue(any([
-            "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log,
-            "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
-        ]))
+        self.assertRegex(log, r"twisted\.internet\.error\.(?:ConnectionRefusedError|ConnectError)")
+        self.assertRegex(log, r"'downloader/exception_type_count/twisted\.internet\.error\.(?:ConnectionRefusedError|ConnectError)': 1,")
 
     def test_reactor_select(self):
         log = self.run_script("twisted_reactor_select.py")

From cf4180308982d9cd017167238a3df8de7900646d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 16 Apr 2020 17:07:29 +0200
Subject: [PATCH 060/243] Skip test_reactor_poll on Windows

---
 tests/test_crawler.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index a6b079395..3d166e14c 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,5 +1,6 @@
 import logging
 import os
+import platform
 import subprocess
 import sys
 import warnings
@@ -317,6 +318,7 @@ class CrawlerProcessSubprocess(unittest.TestCase):
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
 
+    @mark.skipif(platform.system() == 'Windows', reason="PollReactor is not supported on Windows")
     def test_reactor_poll(self):
         log = self.run_script("twisted_reactor_poll.py")
         self.assertIn("Spider closed (finished)", log)

From ea3e675801fe41c7517c58e21b46831940fbd064 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 16 Apr 2020 17:10:45 +0200
Subject: [PATCH 061/243] test_utils_iterators: use os.linesep

---
 tests/test_utils_iterators.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 33fc4d570..5b0073fd1 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -8,7 +8,7 @@ from scrapy.http import XmlResponse, TextResponse, Response
 from tests import get_testdata
 
 
-FOOBAR_NL = u"foo\nbar"
+FOOBAR_NL = "foo{}bar".format(os.linesep)
 
 
 class XmliterTestCase(unittest.TestCase):

From ec76445dd6753074c1531571f66467eecf22b498 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Sat, 18 Apr 2020 09:29:23 +0300
Subject: [PATCH 062/243] Update tests

---
 tests/test_feedexport.py | 66 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 65 insertions(+), 1 deletion(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index e97e50e8e..8e03a91c8 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1128,6 +1128,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_export_items(self):
+        """ Test partial deliveries in all supported formats """
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
             self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
@@ -1145,7 +1146,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
         yield self.assertExported(items, header, rows, settings=settings)
 
     def test_wrong_path(self):
-        """If path is without %(time)s or %(batch_id)s an exception must be raised"""
+        """ If path is without %(time)s or %(batch_id)s an exception must be raised """
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},
@@ -1189,3 +1190,66 @@ class PartialDeliveriesTest(FeedExportTestBase):
             data = yield self.exported_no_data(settings)
             data = dict(data)
             self.assertEqual(expctd, data[fmt][0])
+
+    @defer.inlineCallbacks
+    def test_export_multiple_configs(self):
+        items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]
+
+        formats = {
+            'json': [u'[\n{"bar": "BAR"}\n]'.encode('utf-8'),
+                     u'[\n{"bar": "BAR1"}\n]'.encode('utf-8')],
+            'xml': [u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'.encode('latin-1'),
+                    u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO1</foo>\n  </item>\n</items>'.encode('latin-1')],
+            'csv': [u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
+                    u'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')],
+        }
+
+        settings = {
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'json', self._file_mark): {
+                    'format': 'json',
+                    'indent': 0,
+                    'fields': ['bar'],
+                    'encoding': 'utf-8',
+                },
+                os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {
+                    'format': 'xml',
+                    'indent': 2,
+                    'fields': ['foo'],
+                    'encoding': 'latin-1',
+                },
+                os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {
+                    'format': 'csv',
+                    'indent': None,
+                    'fields': ['bar', 'foo'],
+                    'encoding': 'utf-8',
+                },
+            },
+            'FEED_STORAGE_BATCH_SIZE': 1,
+        }
+        data = yield self.exported_data(items, settings)
+        for fmt, expected in formats.items():
+            for expected_batch, got_batch in zip(expected, data[fmt]):
+                self.assertEqual(expected_batch, got_batch)
+
+    @defer.inlineCallbacks
+    def test_batch_path_differ(self):
+        """
+        Test that the name of all batch files differ from each other.
+        So %(time)s replaced with the current date.
+        """
+        items = [
+            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+            self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
+            self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
+        ]
+        settings = {
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), '%(time)s'): {
+                    'format': 'json',
+                },
+            },
+            'FEED_STORAGE_BATCH_SIZE': 1,
+        }
+        data = yield self.exported_data(items, settings)
+        self.assertEqual(len(items) + 1, len(data['json']))

From 773ddf77ba36613a90d35e583b29e9020cc2aace Mon Sep 17 00:00:00 2001
From: ilias-ant <ilias.antonopoulos@yahoo.gr>
Date: Sun, 19 Apr 2020 14:14:17 +0300
Subject: [PATCH 063/243] added more tests to cover the (downloaded, uptodate,
 cached) status functionality

---
 tests/test_pipeline_files.py | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index f155db4ce..9eb6462f2 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -84,6 +84,7 @@ class FilesPipelineTestCase(unittest.TestCase):
 
         result = yield self.pipeline.process_item(item, None)
         self.assertEqual(result['files'][0]['checksum'], 'abc')
+        self.assertEqual(result['files'][0]['status'], 'uptodate')
 
         for p in patchers:
             p.stop()
@@ -105,6 +106,29 @@ class FilesPipelineTestCase(unittest.TestCase):
 
         result = yield self.pipeline.process_item(item, None)
         self.assertNotEqual(result['files'][0]['checksum'], 'abc')
+        self.assertEqual(result['files'][0]['status'], 'downloaded')
+
+        for p in patchers:
+            p.stop()
+
+    @defer.inlineCallbacks
+    def test_file_cached(self):
+        item_url = "http://example.com/file3.pdf"
+        item = _create_item_with_files(item_url)
+        patchers = [
+            mock.patch.object(FilesPipeline, 'inc_stats', return_value=True),
+            mock.patch.object(FSFilesStore, 'stat_file', return_value={
+                'checksum': 'abc',
+                'last_modified': time.time() - (self.pipeline.expires * 60 * 60 * 24 * 2)}),
+            mock.patch.object(FilesPipeline, 'get_media_requests',
+                              return_value=[_prepare_request_object(item_url, flags=['cached'])])
+        ]
+        for p in patchers:
+            p.start()
+
+        result = yield self.pipeline.process_item(item, None)
+        self.assertNotEqual(result['files'][0]['checksum'], 'abc')
+        self.assertEqual(result['files'][0]['status'], 'cached')
 
         for p in patchers:
             p.stop()
@@ -403,10 +427,10 @@ def _create_item_with_files(*files):
     return item
 
 
-def _prepare_request_object(item_url):
+def _prepare_request_object(item_url, flags=None):
     return Request(
         item_url,
-        meta={'response': Response(item_url, status=200, body=b'data')})
+        meta={'response': Response(item_url, status=200, body=b'data', flags=flags)})
 
 
 if __name__ == "__main__":

From 1fecacbb1a3c2c4c61c202561f9c51f7f6b191ba Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 20 Apr 2020 12:05:15 -0300
Subject: [PATCH 064/243] IPv6 test: check for the absence of DNSLookupError

---
 tests/test_crawler.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 3d166e14c..d6756c266 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -310,8 +310,7 @@ class CrawlerProcessSubprocess(unittest.TestCase):
     def test_ipv6_alternative_name_resolver(self):
         log = self.run_script('alternative_name_resolver.py')
         self.assertIn('Spider closed (finished)', log)
-        self.assertRegex(log, r"twisted\.internet\.error\.(?:ConnectionRefusedError|ConnectError)")
-        self.assertRegex(log, r"'downloader/exception_type_count/twisted\.internet\.error\.(?:ConnectionRefusedError|ConnectError)': 1,")
+        self.assertNotIn("twisted.internet.error.DNSLookupError", log)
 
     def test_reactor_select(self):
         log = self.run_script("twisted_reactor_select.py")

From f0f1be76d1e6cef65ac9a01d13c5d5060a03f648 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Mon, 27 Apr 2020 09:56:57 +0300
Subject: [PATCH 065/243] Using time_id instead of time as a timestamp

---
 docs/topics/feed-exports.rst    |  6 +++---
 scrapy/extensions/feedexport.py | 11 ++++++-----
 tests/test_feedexport.py        |  8 ++++----
 3 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 0bba03a7c..2017be78f 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -441,9 +441,9 @@ An integer number which represent number of scraped items stored in each output
 file. Whenever the number of items exceeds this setting, a new file
 creates and output redirects to it.  The name of the new file will be selected
 based on timestamp when the feed is being created and/or batch sequence number.
-Therefore you must specify %(time)s or %(batch_id)s or both in the file path.
+Therefore you must specify %(time_id)s or %(batch_id)s or both in the file path.
 
-* ``%(time)s`` - gets replaced by a timestamp when the feed is being created
+* ``%(time_id)s`` - gets replaced by a timestamp when the feed is being created
 * ``%(batch_id)s`` - gets replaced by sequence number of batch
 
 For instance::
@@ -452,7 +452,7 @@ For instance::
 
 Your request can be like::
 
-  scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time)s.json
+  scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time_id)s.json
 
 The result directory tree of above can be like::
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 06ea6c5b2..72baa6269 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -292,7 +292,7 @@ class FeedExporter:
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
         :param spider: user spider
-        :param template_uri: template uri which contains %(time)s or %(batch_id)s to create new uri
+        :param template_uri: template uri which contains %(time_id)s or %(batch_id)s to create new uri
         """
         if previous_batch_slot is not None:
             previous_batch_id = previous_batch_slot.batch_id
@@ -360,12 +360,12 @@ class FeedExporter:
 
     def _batch_deliveries_supported(self, uri):
         """
-        If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time)s or %(batch_id)s
+        If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time_id)s or %(batch_id)s
         to distinguish different files of partial output
         """
-        if self.storage_batch_size is None or '%(time)s' in uri or '%(batch_id)s' in uri:
+        if self.storage_batch_size is None or '%(time_id)s' in uri or '%(batch_id)s' in uri:
             return True
-        logger.warning('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
+        logger.warning('%(time_id)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
         return False
 
     def _storage_supported(self, uri):
@@ -397,8 +397,9 @@ class FeedExporter:
         params = {}
         for k in dir(spider):
             params[k] = getattr(spider, k)
+        params['time'] = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')
+        params['time_id'] = datetime.utcnow().isoformat().replace(':', '-')
         params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
-        params['time'] = datetime.utcnow().isoformat().replace(':', '-')
         uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
         uripar_function(params, spider)
         return params
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 8e03a91c8..da759917a 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -989,7 +989,7 @@ class FeedExportTest(FeedExportTestBase):
 
 class PartialDeliveriesTest(FeedExportTestBase):
     __test__ = True
-    _file_mark = '_%(time)s_#%(batch_id)s_'
+    _file_mark = '_%(time_id)s_#%(batch_id)s_'
 
     @defer.inlineCallbacks
     def run_and_export(self, spider_cls, settings):
@@ -1146,7 +1146,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
         yield self.assertExported(items, header, rows, settings=settings)
 
     def test_wrong_path(self):
-        """ If path is without %(time)s or %(batch_id)s an exception must be raised """
+        """ If path is without %(time_id)s or %(batch_id)s an exception must be raised """
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},
@@ -1236,7 +1236,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
     def test_batch_path_differ(self):
         """
         Test that the name of all batch files differ from each other.
-        So %(time)s replaced with the current date.
+        So %(time_id)s replaced with the current date.
         """
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
@@ -1245,7 +1245,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
         ]
         settings = {
             'FEEDS': {
-                os.path.join(self._random_temp_filename(), '%(time)s'): {
+                os.path.join(self._random_temp_filename(), '%(time_id)s'): {
                     'format': 'json',
                 },
             },

From 2eee6c81017e08bb492da560bc73c03f4f375fcc Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Mon, 27 Apr 2020 09:58:14 +0300
Subject: [PATCH 066/243] Documentation spelling fix

---
 docs/topics/feed-exports.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 2017be78f..6c463fc27 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -438,10 +438,10 @@ FEED_EXPORT_BATCH_SIZE
 Default: ``None``
 
 An integer number which represent number of scraped items stored in each output
-file. Whenever the number of items exceeds this setting, a new file
-creates and output redirects to it.  The name of the new file will be selected
+file. Whenever the number of items exceeds this setting, a new file is
+created and output redirects to it.  The name of the new file will be selected
 based on timestamp when the feed is being created and/or batch sequence number.
-Therefore you must specify %(time_id)s or %(batch_id)s or both in the file path.
+Therefore you must specify %(time_id)s or %(batch_id)s or both in FEED_URI.
 
 * ``%(time_id)s`` - gets replaced by a timestamp when the feed is being created
 * ``%(batch_id)s`` - gets replaced by sequence number of batch

From 204737042ac6672eee73c975d0bd6735893d684c Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Mon, 27 Apr 2020 12:52:18 +0300
Subject: [PATCH 067/243] Extract the slot closing functionality to the
 function; minor changes

---
 scrapy/extensions/feedexport.py | 56 ++++++++++++++++-----------------
 1 file changed, 27 insertions(+), 29 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 72baa6269..fe6061c33 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -255,7 +255,7 @@ class FeedExporter:
         for uri, feed in self.feeds.items():
             uri_params = self._get_uri_params(spider, feed['uri_params'], None)
             self.slots.append(self._start_new_batch(
-                previous_batch_slot=None,
+                batch_id=1,
                 uri=uri % uri_params,
                 feed=feed,
                 spider=spider,
@@ -265,42 +265,38 @@ class FeedExporter:
     def close_spider(self, spider):
         deferred_list = []
         for slot in self.slots:
-            if not slot.itemcount and not slot.store_empty:
-                # We need to call slot.storage.store nonetheless to get the file
-                # properly closed.
-                return defer.maybeDeferred(slot.storage.store, slot.file)
-            slot.finish_exporting()
-            logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
-            log_args = {'format': slot.format,
-                        'itemcount': slot.itemcount,
-                        'uri': slot.uri}
-            d = defer.maybeDeferred(slot.storage.store, slot.file)
-            d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args,
-                                                extra={'spider': spider}))
-            d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args,
-                                                exc_info=failure_to_exc_info(f),
-                                                extra={'spider': spider}))
+            d = self._close_slot(slot, spider)
             deferred_list.append(d)
         return defer.DeferredList(deferred_list) if deferred_list else None
 
-    def _start_new_batch(self, previous_batch_slot, uri, feed, spider, template_uri):
+    def _close_slot(self, slot, spider):
+        if not slot.itemcount and not slot.store_empty:
+            # We need to call slot.storage.store nonetheless to get the file
+            # properly closed.
+            return defer.maybeDeferred(slot.storage.store, slot.file)
+        slot.finish_exporting()
+        logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
+        log_args = {'format': slot.format,
+                    'itemcount': slot.itemcount,
+                    'uri': slot.uri}
+        d = defer.maybeDeferred(slot.storage.store, slot.file)
+        d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args,
+                                            extra={'spider': spider}))
+        d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args,
+                                            exc_info=failure_to_exc_info(f),
+                                            extra={'spider': spider}))
+        return d
+
+    def _start_new_batch(self, batch_id, uri, feed, spider, template_uri):
         """
         Redirect the output data stream to a new file.
         Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified.
-        :param previous_batch_slot: slot of previous batch. We need to call slot.storage.store
-        to get the file properly closed.
+        :param batch_id: sequence number of current batch
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
         :param spider: user spider
         :param template_uri: template uri which contains %(time_id)s or %(batch_id)s to create new uri
         """
-        if previous_batch_slot is not None:
-            previous_batch_id = previous_batch_slot.batch_id
-            previous_batch_slot.exporter.finish_exporting()
-            previous_batch_slot.storage.store(previous_batch_slot.file)
-        else:
-            previous_batch_id = 0
-
         storage = self._get_storage(uri)
         file = storage.open(spider)
         exporter = self._get_exporter(
@@ -317,7 +313,7 @@ class FeedExporter:
             uri=uri,
             format=feed['format'],
             store_empty=feed['store_empty'],
-            batch_id=previous_batch_id + 1,
+            batch_id=batch_id,
             template_uri=template_uri,
         )
         if slot.store_empty:
@@ -330,10 +326,12 @@ class FeedExporter:
             slot.start_exporting()
             slot.exporter.export_item(item)
             slot.itemcount += 1
-            if self.storage_batch_size and slot.itemcount % self.storage_batch_size == 0:
+            # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_SIZE and close the old one
+            if self.storage_batch_size and slot.itemcount == self.storage_batch_size:
                 uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot)
+                self._close_slot(slot, spider)
                 slots.append(self._start_new_batch(
-                    previous_batch_slot=slot,
+                    batch_id=slot.batch_id + 1,
                     uri=slot.template_uri % uri_params,
                     feed=self.feeds[slot.template_uri],
                     spider=spider,

From e3c3ec2ba988f654be1676586714fd96dba32c23 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 28 Apr 2020 13:48:50 +0200
Subject: [PATCH 068/243] Run quick tests first in Travis CI

---
 .travis.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 66e1a9617..dc91dfe4c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,6 +11,9 @@ matrix:
       python: 3.8
     - env: TOXENV=flake8
       python: 3.8
+    - env: TOXENV=docs
+      python: 3.7  # Keep in sync with .readthedocs.yml
+
     - env: TOXENV=pypy3
     - env: TOXENV=py35
       python: 3.5
@@ -28,8 +31,6 @@ matrix:
       python: 3.8
     - env: TOXENV=py38-asyncio
       python: 3.8
-    - env: TOXENV=docs
-      python: 3.7  # Keep in sync with .readthedocs.yml
 install:
   - |
       if [ "$TOXENV" = "pypy3" ]; then

From 15d96ab8b5fa6d349a45920f93d349b8ea1d0372 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 28 Apr 2020 17:09:05 +0200
Subject: [PATCH 069/243] Test the latest Ubuntu along the latest Python

---
 .travis.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 66e1a9617..02c8885e5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -24,10 +24,13 @@ matrix:
       python: 3.7
     - env: TOXENV=py38
       python: 3.8
+      dist: bionic
     - env: TOXENV=extra-deps
       python: 3.8
+      dist: bionic
     - env: TOXENV=py38-asyncio
       python: 3.8
+      dist: bionic
     - env: TOXENV=docs
       python: 3.7  # Keep in sync with .readthedocs.yml
 install:

From 5c0f11b4ef1d58de4245d9f4ac9a26f21faf082c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 28 Apr 2020 17:32:53 +0200
Subject: [PATCH 070/243] Simplify the asyncio Tox environment

---
 .travis.yml |  4 ++--
 tox.ini     | 12 +-----------
 2 files changed, 3 insertions(+), 13 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 66e1a9617..a924eb68c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,7 +16,7 @@ matrix:
       python: 3.5
     - env: TOXENV=pinned
       python: 3.5
-    - env: TOXENV=py35-asyncio
+    - env: TOXENV=asyncio
       python: 3.5.2
     - env: TOXENV=py36
       python: 3.6
@@ -26,7 +26,7 @@ matrix:
       python: 3.8
     - env: TOXENV=extra-deps
       python: 3.8
-    - env: TOXENV=py38-asyncio
+    - env: TOXENV=asyncio
       python: 3.8
     - env: TOXENV=docs
       python: 3.7  # Keep in sync with .readthedocs.yml
diff --git a/tox.ini b/tox.ini
index cd118c921..697328ebd 100644
--- a/tox.ini
+++ b/tox.ini
@@ -102,16 +102,6 @@ setenv = {[docs]setenv}
 commands =
     sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
 
-[asyncio]
+[testenv:asyncio]
 commands =
     {[testenv]commands} --reactor=asyncio
-
-[testenv:py35-asyncio]
-basepython = python3.5
-deps = {[testenv]deps}
-commands = {[asyncio]commands}
-
-[testenv:py38-asyncio]
-basepython = python3.8
-deps = {[testenv]deps}
-commands = {[asyncio]commands}

From 3a64f3eb2902ed8168b78c43f3516cf657873cef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 28 Apr 2020 17:44:19 +0200
Subject: [PATCH 071/243] Remove TOXENV from .travis.yml unless needed

---
 .travis.yml | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 66e1a9617..b029d8bda 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,17 +12,14 @@ matrix:
     - env: TOXENV=flake8
       python: 3.8
     - env: TOXENV=pypy3
-    - env: TOXENV=py35
-      python: 3.5
+    - python: 3.5
     - env: TOXENV=pinned
       python: 3.5
     - env: TOXENV=py35-asyncio
       python: 3.5.2
-    - env: TOXENV=py36
-      python: 3.6
-    - env: TOXENV=py37
-      python: 3.7
-    - env: TOXENV=py38
+    - python: 3.6
+    - python: 3.7
+    - env: PYPI_RELEASE_JOB=true
       python: 3.8
     - env: TOXENV=extra-deps
       python: 3.8
@@ -62,4 +59,4 @@ deploy:
   on:
     tags: true
     repo: scrapy/scrapy
-    condition: "$TOXENV == py37 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
+    condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"

From f787b8483ceb37ad8c9764d5a28be07028d85f70 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 20 Apr 2020 12:05:15 -0300
Subject: [PATCH 072/243] IPv6 test: check for the absence of DNSLookupError

---
 tests/test_crawler.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index b4144ea1d..9151278a5 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -311,14 +311,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
     def test_ipv6_alternative_name_resolver(self):
         log = self.run_script('alternative_name_resolver.py')
         self.assertIn('Spider closed (finished)', log)
-        self.assertTrue(any([
-            "twisted.internet.error.ConnectionRefusedError" in log,
-            "twisted.internet.error.ConnectError" in log,
-        ]))
-        self.assertTrue(any([
-            "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log,
-            "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
-        ]))
+        self.assertNotIn("twisted.internet.error.DNSLookupError", log)
 
     def test_reactor_select(self):
         log = self.run_script("twisted_reactor_select.py")

From 3f9874fac9f93c0956afa5975d7b2bbb21816894 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Fri, 1 May 2020 11:52:16 +0300
Subject: [PATCH 073/243] Add test s3 export

---
 tests/test_feedexport.py | 70 ++++++++++++++++++++++++++++++++++++++++
 tox.ini                  |  1 +
 2 files changed, 71 insertions(+)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index da759917a..9fc39c3a6 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1253,3 +1253,73 @@ class PartialDeliveriesTest(FeedExportTestBase):
         }
         data = yield self.exported_data(items, settings)
         self.assertEqual(len(items) + 1, len(data['json']))
+
+    @defer.inlineCallbacks
+    def test_s3_export(self):
+        """
+        Test export of items into s3 bucket.
+        S3_TEST_BUCKET_NAME, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY must be specified in tox.ini
+        to perform this test:
+        [testenv]
+        setenv =
+            AWS_SECRET_ACCESS_KEY = ABCD
+            AWS_ACCESS_KEY_ID = ABCD
+            S3_TEST_BUCKET_NAME = ABCD
+        """
+        try:
+            import boto3
+        except ImportError:
+            raise unittest.SkipTest("S3FeedStorage requires boto3")
+
+        assert_aws_environ()
+        s3_test_bucket_name = os.environ.get('S3_TEST_BUCKET_NAME')
+        access_key = os.environ.get('AWS_ACCESS_KEY_ID')
+        secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
+        if not s3_test_bucket_name:
+            raise unittest.SkipTest("No S3 BUCKET available for testing")
+
+        chars = [random.choice(ascii_letters + digits) for _ in range(15)]
+        filename = ''.join(chars)
+        prefix = 'tmp/{filename}'.format(filename=filename)
+        s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(time_id)s.json'.format(
+            bucket_name=s3_test_bucket_name, prefix=prefix
+        )
+        storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key)
+        settings = {
+            'FEEDS': {
+                s3_test_file_uri: {
+                    'format': 'json',
+                },
+            },
+            'FEED_STORAGE_BATCH_SIZE': 1,
+        }
+        items = [
+            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+            self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
+            self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
+        ]
+        verifyObject(IFeedStorage, storage)
+
+        class TestSpider(scrapy.Spider):
+            name = 'testspider'
+
+            def parse(self, response):
+                for item in items:
+                    yield item
+
+        s3 = boto3.resource('s3')
+        my_bucket = s3.Bucket(s3_test_bucket_name)
+        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+
+        with MockServer() as s:
+            runner = CrawlerRunner(Settings(settings))
+            TestSpider.start_urls = [s.url('/')]
+            yield runner.crawl(TestSpider)
+
+        for file_uri in my_bucket.objects.filter(Prefix=prefix):
+            content = get_s3_content_and_delete(s3_test_bucket_name, file_uri.key)
+            if not content and not items:
+                break
+            content = json.loads(content.decode('utf-8'))
+            expected_batch, items = items[:batch_size], items[batch_size:]
+            self.assertEqual(expected_batch, content)
diff --git a/tox.ini b/tox.ini
index cd118c921..c77fae1f0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -14,6 +14,7 @@ deps =
     # Extras
     botocore>=1.3.23
     Pillow>=3.4.2
+    boto3>=1.13.0
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID

From dad2ea75222d6240c569440d3221f5fc00925682 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Sat, 2 May 2020 01:21:03 +0300
Subject: [PATCH 074/243] Change time_id to batch_time

---
 docs/topics/feed-exports.rst    |  6 +++---
 scrapy/extensions/feedexport.py | 10 +++++-----
 tests/test_feedexport.py        | 10 +++++-----
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 6c463fc27..2106b41f5 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -441,9 +441,9 @@ An integer number which represent number of scraped items stored in each output
 file. Whenever the number of items exceeds this setting, a new file is
 created and output redirects to it.  The name of the new file will be selected
 based on timestamp when the feed is being created and/or batch sequence number.
-Therefore you must specify %(time_id)s or %(batch_id)s or both in FEED_URI.
+Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI.
 
-* ``%(time_id)s`` - gets replaced by a timestamp when the feed is being created
+* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
 * ``%(batch_id)s`` - gets replaced by sequence number of batch
 
 For instance::
@@ -452,7 +452,7 @@ For instance::
 
 Your request can be like::
 
-  scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time_id)s.json
+  scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
 
 The result directory tree of above can be like::
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index fe6061c33..a262f5d18 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -295,7 +295,7 @@ class FeedExporter:
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
         :param spider: user spider
-        :param template_uri: template uri which contains %(time_id)s or %(batch_id)s to create new uri
+        :param template_uri: template uri which contains %(batch_time)s or %(batch_id)s to create new uri
         """
         storage = self._get_storage(uri)
         file = storage.open(spider)
@@ -358,12 +358,12 @@ class FeedExporter:
 
     def _batch_deliveries_supported(self, uri):
         """
-        If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time_id)s or %(batch_id)s
+        If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(batch_time)s or %(batch_id)s
         to distinguish different files of partial output
         """
-        if self.storage_batch_size is None or '%(time_id)s' in uri or '%(batch_id)s' in uri:
+        if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
             return True
-        logger.warning('%(time_id)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
+        logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
         return False
 
     def _storage_supported(self, uri):
@@ -396,7 +396,7 @@ class FeedExporter:
         for k in dir(spider):
             params[k] = getattr(spider, k)
         params['time'] = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')
-        params['time_id'] = datetime.utcnow().isoformat().replace(':', '-')
+        params['batch_time'] = datetime.utcnow().isoformat().replace(':', '-')
         params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
         uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
         uripar_function(params, spider)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 9fc39c3a6..2217bb4ed 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -989,7 +989,7 @@ class FeedExportTest(FeedExportTestBase):
 
 class PartialDeliveriesTest(FeedExportTestBase):
     __test__ = True
-    _file_mark = '_%(time_id)s_#%(batch_id)s_'
+    _file_mark = '_%(batch_time)s_#%(batch_id)s_'
 
     @defer.inlineCallbacks
     def run_and_export(self, spider_cls, settings):
@@ -1146,7 +1146,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
         yield self.assertExported(items, header, rows, settings=settings)
 
     def test_wrong_path(self):
-        """ If path is without %(time_id)s or %(batch_id)s an exception must be raised """
+        """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},
@@ -1236,7 +1236,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
     def test_batch_path_differ(self):
         """
         Test that the name of all batch files differ from each other.
-        So %(time_id)s replaced with the current date.
+        So %(batch_time)s replaced with the current date.
         """
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
@@ -1245,7 +1245,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
         ]
         settings = {
             'FEEDS': {
-                os.path.join(self._random_temp_filename(), '%(time_id)s'): {
+                os.path.join(self._random_temp_filename(), '%(batch_time)s'): {
                     'format': 'json',
                 },
             },
@@ -1281,7 +1281,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
         chars = [random.choice(ascii_letters + digits) for _ in range(15)]
         filename = ''.join(chars)
         prefix = 'tmp/{filename}'.format(filename=filename)
-        s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(time_id)s.json'.format(
+        s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(batch_time)s.json'.format(
             bucket_name=s3_test_bucket_name, prefix=prefix
         )
         storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key)

From 83d7360bb709cf2c73680260c58b767006f42b12 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 4 May 2020 02:00:11 +0500
Subject: [PATCH 075/243] Don't mention unsupported package versions in docs

---
 docs/topics/settings.rst | 16 +++-------------
 1 file changed, 3 insertions(+), 13 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 18f81838f..e3da1bd12 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -420,10 +420,9 @@ connections (for ``HTTP10DownloadHandler``).
 .. note::
 
     HTTP/1.0 is rarely used nowadays so you can safely ignore this setting,
-    unless you use Twisted<11.1, or if you really want to use HTTP/1.0
-    and override :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme
-    accordingly, i.e. to
-    ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
+    unless you really want to use HTTP/1.0 and override
+    :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme accordingly,
+    i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
 
 .. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY
 
@@ -447,7 +446,6 @@ or even enable client-side authentication (and various other things).
     Scrapy also has another context factory class that you can set,
     ``'scrapy.core.downloader.contextfactory.BrowserLikeContextFactory'``,
     which uses the platform's certificates to validate remote endpoints.
-    **This is only available if you use Twisted>=14.0.**
 
 If you do use a custom ContextFactory, make sure its ``__init__`` method
 accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping
@@ -494,10 +492,6 @@ This setting must be one of these string values:
 - ``'TLSv1.2'``: forces TLS version 1.2
 - ``'SSLv3'``: forces SSL version 3 (**not recommended**)
 
-.. note::
-
-    We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13
-    or above (Twisted>=14.0 if you can).
 
 .. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
 
@@ -660,8 +654,6 @@ If you want to disable it set to 0.
     spider attribute and per-request using :reqmeta:`download_maxsize`
     Request.meta key.
 
-    This feature needs Twisted >= 11.1.
-
 .. setting:: DOWNLOAD_WARNSIZE
 
 DOWNLOAD_WARNSIZE
@@ -679,8 +671,6 @@ If you want to disable it set to 0.
     spider attribute and per-request using :reqmeta:`download_warnsize`
     Request.meta key.
 
-    This feature needs Twisted >= 11.1.
-
 .. setting:: DOWNLOAD_FAIL_ON_DATALOSS
 
 DOWNLOAD_FAIL_ON_DATALOSS

From f75941f79dfeb931649f9e8880cdbd1b0a8e4681 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 30 Apr 2020 11:56:52 -0300
Subject: [PATCH 076/243] Deprecate scrapy.item.BaseItem

---
 scrapy/item.py     |  7 ++++++-
 tests/test_item.py | 25 ++++++++++++++++++++-----
 2 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/scrapy/item.py b/scrapy/item.py
index 748368932..3558b2231 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -28,7 +28,12 @@ class BaseItem(object_ref):
     Unlike instances of :class:`dict`, instances of :class:`BaseItem` may be
     :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
     """
-    pass
+
+    def __new__(cls, *args, **kwargs):
+        if issubclass(cls, BaseItem) and not (issubclass(cls, Item) or issubclass(cls, DictItem)):
+            warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
+                 ScrapyDeprecationWarning, stacklevel=2)
+        return super(BaseItem, cls).__new__(cls, *args, **kwargs)
 
 
 class Field(dict):
diff --git a/tests/test_item.py b/tests/test_item.py
index 4017f6e84..1220bc98a 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -4,7 +4,7 @@ from unittest import mock
 from warnings import catch_warnings
 
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta
+from scrapy.item import ABCMeta, BaseItem, DictItem, Field, Item, ItemMeta
 
 
 PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6)
@@ -131,12 +131,12 @@ class ItemTest(unittest.TestCase):
         self.assertSortedEqual(list(item.values()), [u'New'])
 
     def test_metaclass_inheritance(self):
-        class BaseItem(Item):
+        class ParentItem(Item):
             name = Field()
             keys = Field()
             values = Field()
 
-        class TestItem(BaseItem):
+        class TestItem(ParentItem):
             keys = Field()
 
         i = TestItem()
@@ -321,13 +321,28 @@ class DictItemTest(unittest.TestCase):
 
     def test_deprecation_warning(self):
         with catch_warnings(record=True) as warnings:
-            dict_item = DictItem()
+            DictItem()
             self.assertEqual(len(warnings), 1)
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
         with catch_warnings(record=True) as warnings:
             class SubclassedDictItem(DictItem):
                 pass
-            subclassed_dict_item = SubclassedDictItem()
+            SubclassedDictItem()
+            self.assertEqual(len(warnings), 1)
+            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+
+
+class BaseItemTest(unittest.TestCase):
+
+    def test_deprecation_warning(self):
+        with catch_warnings(record=True) as warnings:
+            BaseItem()
+            self.assertEqual(len(warnings), 1)
+            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+        with catch_warnings(record=True) as warnings:
+            class SubclassedBaseItem(BaseItem):
+                pass
+            SubclassedBaseItem()
             self.assertEqual(len(warnings), 1)
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
 

From e1948b492317eb5b11550d119d91c61b74b3a37f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 4 May 2020 09:07:27 -0300
Subject: [PATCH 077/243] Add example about bytes_received signal

---
 docs/topics/signals.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index bc04faad5..7fe63a7b0 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -378,7 +378,9 @@ bytes_received
 
     Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is
     received for a specific request. This signal might be fired multiple
-    times for the same request, with partial data each time.
+    times for the same request, with partial data each time. For instance,
+    a possible scenario for a 25 kb response would be two signals fired
+    with 10 kb of data, and a final one with 5 kb of data.
 
     This signal does not support returning deferreds from its handlers.
 

From fe6154e4faee375e7f47d61ceafabde7a3289bf3 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 4 May 2020 18:18:38 +0500
Subject: [PATCH 078/243] clarify DOWNLOADER_HTTPCLIENTFACTORY docs

---
 docs/topics/settings.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index e3da1bd12..f06d9db3c 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -421,7 +421,7 @@ connections (for ``HTTP10DownloadHandler``).
 
     HTTP/1.0 is rarely used nowadays so you can safely ignore this setting,
     unless you really want to use HTTP/1.0 and override
-    :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme accordingly,
+    :setting:`DOWNLOAD_HANDLERS` for ``http(s)`` scheme accordingly,
     i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
 
 .. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY

From 622ce860669b8bd7fc581b74d414aef24f4fb041 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 4 May 2020 16:22:24 -0300
Subject: [PATCH 079/243] Test: make sure scrapy.item.Item does not issue a
 deprecation warning

---
 tests/test_item.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/test_item.py b/tests/test_item.py
index 1220bc98a..f35a2b9f9 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -347,5 +347,18 @@ class BaseItemTest(unittest.TestCase):
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
 
 
+class ItemNoDeprecationWarningTest(unittest.TestCase):
+
+    def test_no_deprecation_warning(self):
+        with catch_warnings(record=True) as warnings:
+            Item()
+            self.assertEqual(len(warnings), 0)
+        with catch_warnings(record=True) as warnings:
+            class SubclassedItem(Item):
+                pass
+            SubclassedItem()
+            self.assertEqual(len(warnings), 0)
+
+
 if __name__ == "__main__":
     unittest.main()

From 7988c676a956457843416f8586544a58bf121f63 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 5 May 2020 13:11:01 -0300
Subject: [PATCH 080/243] Update Item docstring, update BaseItem occurrences

---
 docs/faq.rst               |  4 ++--
 pytest.ini                 |  2 +-
 scrapy/item.py             | 35 ++++++++++++++++++-----------------
 scrapy/spiders/feed.py     |  2 +-
 tests/test_loader.py       |  2 +-
 tests/test_utils_spider.py |  4 ++--
 6 files changed, 25 insertions(+), 24 deletions(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index 75a0f4864..79ef6ca85 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -342,14 +342,14 @@ method for this purpose. For example::
 
     from copy import deepcopy
 
-    from scrapy.item import BaseItem
+    from scrapy.item import Item
 
 
     class MultiplyItemsMiddleware:
 
         def process_spider_output(self, response, result, spider):
             for item in result:
-                if isinstance(item, (BaseItem, dict)):
+                if isinstance(item, (Item, dict)):
                     for _ in range(item['multiply_by']):
                         yield deepcopy(item)
 
diff --git a/pytest.ini b/pytest.ini
index e8911ee3f..5a86ce2a7 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -153,7 +153,7 @@ flake8-ignore =
     scrapy/exceptions.py E501
     scrapy/exporters.py E501
     scrapy/interfaces.py E501
-    scrapy/item.py E501 E128
+    scrapy/item.py E501
     scrapy/link.py E501
     scrapy/logformatter.py E501
     scrapy/mail.py E402 E128 E501
diff --git a/scrapy/item.py b/scrapy/item.py
index 3558b2231..46d20d017 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -15,18 +15,8 @@ from scrapy.utils.trackref import object_ref
 
 
 class BaseItem(object_ref):
-    """Base class for all scraped items.
-
-    In Scrapy, an object is considered an *item* if it is an instance of either
-    :class:`BaseItem` or :class:`dict`. For example, when the output of a
-    spider callback is evaluated, only instances of :class:`BaseItem` or
-    :class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
-
-    If you need instances of a custom class to be considered items by Scrapy,
-    you must inherit from either :class:`BaseItem` or :class:`dict`.
-
-    Unlike instances of :class:`dict`, instances of :class:`BaseItem` may be
-    :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
+    """
+    Deprecated, please use :class:`scrapy.item.Item` instead
     """
 
     def __new__(cls, *args, **kwargs):
@@ -91,8 +81,7 @@ class DictItem(MutableMapping, BaseItem):
         if key in self.fields:
             self._values[key] = value
         else:
-            raise KeyError("%s does not support field: %s" %
-                (self.__class__.__name__, key))
+            raise KeyError("%s does not support field: %s" % (self.__class__.__name__, key))
 
     def __delitem__(self, key):
         del self._values[key]
@@ -104,8 +93,7 @@ class DictItem(MutableMapping, BaseItem):
 
     def __setattr__(self, name, value):
         if not name.startswith('_'):
-            raise AttributeError("Use item[%r] = %r to set field value" %
-                (name, value))
+            raise AttributeError("Use item[%r] = %r to set field value" % (name, value))
         super(DictItem, self).__setattr__(name, value)
 
     def __len__(self):
@@ -132,4 +120,17 @@ class DictItem(MutableMapping, BaseItem):
 
 
 class Item(DictItem, metaclass=ItemMeta):
-    pass
+    """
+    Base class for scraped items.
+
+    In Scrapy, an object is considered an *item* if it is an instance of either
+    :class:`Item` or :class:`dict`. For example, when the output of a
+    spider callback is evaluated, only instances of :class:`Item` or
+    :class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
+
+    If you need instances of a custom class to be considered items by Scrapy,
+    you must inherit from either :class:`Item` or :class:`dict`.
+
+    Unlike instances of :class:`dict`, instances of :class:`Item` may be
+    :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
+    """
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index c566f0236..a4ff8010d 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -52,7 +52,7 @@ class XMLFeedSpider(Spider):
         """This method is called for the nodes matching the provided tag name
         (itertag). Receives the response and an Selector for each node.
         Overriding this method is mandatory. Otherwise, you spider won't work.
-        This method must return either a BaseItem, a Request, or a list
+        This method must return either an item, a request, or a list
         containing any of them.
         """
 
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 701d568dc..f14714c75 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -601,7 +601,7 @@ class NoInputReprocessingItemLoader(BaseNoInputReprocessingLoader):
 
 class NoInputReprocessingFromItemTest(unittest.TestCase):
     """
-    Loaders initialized from loaded items must not reprocess fields (BaseItem instances)
+    Loaders initialized from loaded items must not reprocess fields (Item instances)
     """
     def test_avoid_reprocessing_with_initial_values_single(self):
         il = NoInputReprocessingItemLoader(item=NoInputReprocessingItem(title='foo'))
diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py
index ee7d17062..3c87268ab 100644
--- a/tests/test_utils_spider.py
+++ b/tests/test_utils_spider.py
@@ -2,7 +2,7 @@ import unittest
 
 from scrapy import Spider
 from scrapy.http import Request
-from scrapy.item import BaseItem
+from scrapy.item import Item
 from scrapy.utils.spider import iterate_spider_output, iter_spider_classes
 
 
@@ -17,7 +17,7 @@ class MySpider2(Spider):
 class UtilsSpidersTestCase(unittest.TestCase):
 
     def test_iterate_spider_output(self):
-        i = BaseItem()
+        i = Item()
         r = Request('http://scrapytest.org')
         o = object()
 

From 17c0cf64aee1641e1ad33c5b46a61435c5969f2f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Tue, 5 May 2020 19:14:48 -0300
Subject: [PATCH 081/243] Flake8: remove W504 code (#4525)

Co-authored-by: Mikhail Korobov <kmike84@gmail.com>
---
 pytest.ini                               | 16 ++++++++--------
 scrapy/contracts/__init__.py             |  4 ++--
 scrapy/downloadermiddlewares/redirect.py | 11 +++++++----
 scrapy/extensions/telnet.py              |  6 ++++--
 scrapy/linkextractors/__init__.py        |  3 +--
 scrapy/spidermiddlewares/referer.py      | 14 ++++++++------
 scrapy/utils/gz.py                       |  3 +--
 tests/test_utils_http.py                 |  8 ++++----
 8 files changed, 35 insertions(+), 30 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index e8911ee3f..4f3494e0e 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -44,12 +44,12 @@ flake8-ignore =
     scrapy/commands/startproject.py E127 E501 E128
     scrapy/commands/version.py E501 E128
     # scrapy/contracts
-    scrapy/contracts/__init__.py E501 W504
+    scrapy/contracts/__init__.py E501
     scrapy/contracts/default.py E128
     # scrapy/core
     scrapy/core/engine.py E501 E128 E127
     scrapy/core/scheduler.py E501
-    scrapy/core/scraper.py E501 E128 W504
+    scrapy/core/scraper.py E501 E128
     scrapy/core/spidermw.py E501 E126
     scrapy/core/downloader/__init__.py E501
     scrapy/core/downloader/contextfactory.py E501 E128 E126
@@ -68,7 +68,7 @@ flake8-ignore =
     scrapy/downloadermiddlewares/httpcache.py E501 E126
     scrapy/downloadermiddlewares/httpcompression.py E501 E128
     scrapy/downloadermiddlewares/httpproxy.py E501
-    scrapy/downloadermiddlewares/redirect.py E501 W504
+    scrapy/downloadermiddlewares/redirect.py E501
     scrapy/downloadermiddlewares/retry.py E501 E126
     scrapy/downloadermiddlewares/robotstxt.py E501
     scrapy/downloadermiddlewares/stats.py E501
@@ -79,7 +79,7 @@ flake8-ignore =
     scrapy/extensions/httpcache.py E128 E501
     scrapy/extensions/memdebug.py E501
     scrapy/extensions/spiderstate.py E501
-    scrapy/extensions/telnet.py E501 W504
+    scrapy/extensions/telnet.py E501
     scrapy/extensions/throttle.py E501
     # scrapy/http
     scrapy/http/common.py E501
@@ -90,7 +90,7 @@ flake8-ignore =
     scrapy/http/response/__init__.py E501 E128
     scrapy/http/response/text.py E501 E128 E124
     # scrapy/linkextractors
-    scrapy/linkextractors/__init__.py E501 E402 W504
+    scrapy/linkextractors/__init__.py E501 E402
     scrapy/linkextractors/lxmlhtml.py E501
     # scrapy/loader
     scrapy/loader/__init__.py E501 E128
@@ -110,7 +110,7 @@ flake8-ignore =
     # scrapy/spidermiddlewares
     scrapy/spidermiddlewares/httperror.py E501
     scrapy/spidermiddlewares/offsite.py E501
-    scrapy/spidermiddlewares/referer.py E501 E129 W504
+    scrapy/spidermiddlewares/referer.py E501 E129
     scrapy/spidermiddlewares/urllength.py E501
     # scrapy/spiders
     scrapy/spiders/__init__.py E501 E402
@@ -125,7 +125,7 @@ flake8-ignore =
     scrapy/utils/decorators.py E501
     scrapy/utils/defer.py E501 E128
     scrapy/utils/deprecate.py E128 E501 E127
-    scrapy/utils/gz.py E501 W504
+    scrapy/utils/gz.py E501
     scrapy/utils/http.py F403
     scrapy/utils/httpobj.py E501
     scrapy/utils/iterators.py E501
@@ -234,7 +234,7 @@ flake8-ignore =
     tests/test_utils_datatypes.py E402 E501
     tests/test_utils_defer.py E501 F841
     tests/test_utils_deprecate.py F841 E501
-    tests/test_utils_http.py E501 E128 W504
+    tests/test_utils_http.py E501 E128
     tests/test_utils_iterators.py E501 E128 E129
     tests/test_utils_log.py E741
     tests/test_utils_python.py E501
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 41d4f25b2..5af3831a2 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -17,10 +17,10 @@ class ContractsManager:
             self.contracts[contract.name] = contract
 
     def tested_methods_from_spidercls(self, spidercls):
+        is_method = re.compile(r"^\s*@", re.MULTILINE).search
         methods = []
         for key, value in getmembers(spidercls):
-            if (callable(value) and value.__doc__ and
-                    re.search(r'^\s*@', value.__doc__, re.MULTILINE)):
+            if callable(value) and value.__doc__ and is_method(value.__doc__):
                 methods.append(key)
 
         return methods
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 09ee8377e..b32afb8e4 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -60,11 +60,14 @@ class RedirectMiddleware(BaseRedirectMiddleware):
     Handle redirection of requests based on response status
     and meta-refresh html tag.
     """
+
     def process_response(self, request, response, spider):
-        if (request.meta.get('dont_redirect', False) or
-                response.status in getattr(spider, 'handle_httpstatus_list', []) or
-                response.status in request.meta.get('handle_httpstatus_list', []) or
-                request.meta.get('handle_httpstatus_all', False)):
+        if (
+            request.meta.get('dont_redirect', False)
+            or response.status in getattr(spider, 'handle_httpstatus_list', [])
+            or response.status in request.meta.get('handle_httpstatus_list', [])
+            or request.meta.get('handle_httpstatus_all', False)
+        ):
             return response
 
         allowed_status = (301, 302, 303, 307, 308)
diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index 04ffd7235..1663604e7 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -76,8 +76,10 @@ class TelnetConsole(protocol.ServerFactory):
             """An implementation of IPortal"""
             @defers
             def login(self_, credentials, mind, *interfaces):
-                if not (credentials.username == self.username.encode('utf8') and
-                        credentials.checkPassword(self.password.encode('utf8'))):
+                if not (
+                    credentials.username == self.username.encode('utf8')
+                    and credentials.checkPassword(self.password.encode('utf8'))
+                ):
                     raise ValueError("Invalid credentials")
 
                 protocol = telnet.TelnetBootstrapProtocol(
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index d0b5066b6..ae019c70f 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -61,8 +61,7 @@ class FilteringLinkExtractor:
 
     def __new__(cls, *args, **kwargs):
         from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
-        if (issubclass(cls, FilteringLinkExtractor) and
-                not issubclass(cls, LxmlLinkExtractor)):
+        if issubclass(cls, FilteringLinkExtractor) and not issubclass(cls, LxmlLinkExtractor):
             warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
                  'please use scrapy.linkextractors.LinkExtractor instead',
                  ScrapyDeprecationWarning, stacklevel=2)
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 3784de885..434067b00 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -163,9 +163,10 @@ class StrictOriginPolicy(ReferrerPolicy):
     name = POLICY_STRICT_ORIGIN
 
     def referrer(self, response_url, request_url):
-        if ((self.tls_protected(response_url) and
-             self.potentially_trustworthy(request_url))
-            or not self.tls_protected(response_url)):
+        if (
+            self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
+            or not self.tls_protected(response_url)
+        ):
             return self.origin_referrer(response_url)
 
 
@@ -213,9 +214,10 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
         origin = self.origin(response_url)
         if origin == self.origin(request_url):
             return self.stripped_referrer(response_url)
-        elif ((self.tls_protected(response_url) and
-               self.potentially_trustworthy(request_url))
-              or not self.tls_protected(response_url)):
+        elif (
+            self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
+            or not self.tls_protected(response_url)
+        ):
             return self.origin_referrer(response_url)
 
 
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index c291ae237..fbd7bd18f 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -52,8 +52,7 @@ def is_gzipped(response):
     """Return True if the response is gzipped, or False otherwise"""
     ctype = response.headers.get('Content-Type', b'')
     cenc = response.headers.get('Content-Encoding', b'').lower()
-    return (_is_gzipped(ctype) or
-            (_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')))
+    return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')
 
 
 def gzip_magic_number(response):
diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py
index 2fac3da1f..363b015a8 100644
--- a/tests/test_utils_http.py
+++ b/tests/test_utils_http.py
@@ -13,7 +13,7 @@ class ChunkedTest(unittest.TestCase):
         chunked_body += "8\r\n" + "sequence\r\n"
         chunked_body += "0\r\n\r\n"
         body = decode_chunked_transfer(chunked_body)
-        self.assertEqual(body,
-                         "This is the data in the first chunk\r\n" +
-                         "and this is the second one\r\n" +
-                         "consequence")
+        self.assertEqual(
+            body,
+            "This is the data in the first chunk\r\nand this is the second one\r\nconsequence"
+        )

From 418b9b5f5222e05bec497a83af8c7dfcca30c6cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 6 May 2020 11:15:02 +0200
Subject: [PATCH 082/243] Travis CI: do not run security and Flake8 on multiple
 jobs

---
 .travis.yml | 11 +++++++----
 tox.ini     |  2 +-
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 6bde973f4..75d3c5a98 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,14 +15,17 @@ matrix:
       python: 3.7  # Keep in sync with .readthedocs.yml
 
     - env: TOXENV=pypy3
-    - python: 3.5
+    - env: TOXENV=py
+      python: 3.5
     - env: TOXENV=pinned
       python: 3.5
     - env: TOXENV=asyncio
       python: 3.5.2
-    - python: 3.6
-    - python: 3.7
-    - env: PYPI_RELEASE_JOB=true
+    - env: TOXENV=py
+      python: 3.6
+    - env: TOXENV=py
+      python: 3.7
+    - env: TOXENV=py PYPI_RELEASE_JOB=true
       python: 3.8
     - env: TOXENV=extra-deps
       python: 3.8
diff --git a/tox.ini b/tox.ini
index 697328ebd..2102fc602 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = security,flake8,py3
+envlist = security,flake8,py
 minversion = 1.7.0
 
 [testenv]

From 8d1e3ee0dd8d48428f719a0fffda964d30ab956d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 09:24:32 -0300
Subject: [PATCH 083/243] Remove deprecated BaseItem from the docs

---
 docs/topics/items.rst | 2 --
 1 file changed, 2 deletions(-)

diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 78612f524..0941a8a1b 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -257,6 +257,4 @@ Field objects
 Other classes related to Item
 =============================
 
-.. autoclass:: BaseItem
-
 .. autoclass:: ItemMeta

From 49e8a337f78ec5e30eacfcd201b66d68deeecb56 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 09:37:01 -0300
Subject: [PATCH 084/243] Flake8: remove E127 (continuation line over-indented
 for visual indent)

---
 pytest.ini                             | 10 +++++-----
 scrapy/core/downloader/handlers/ftp.py | 11 ++++++-----
 scrapy/core/engine.py                  |  3 +--
 scrapy/utils/deprecate.py              | 21 +++++++++++----------
 scrapy/utils/request.py                |  3 +--
 5 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 4f3494e0e..fa65a0da2 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -41,13 +41,13 @@ flake8-ignore =
     scrapy/commands/runspider.py E501
     scrapy/commands/settings.py E128
     scrapy/commands/shell.py E128 E501
-    scrapy/commands/startproject.py E127 E501 E128
+    scrapy/commands/startproject.py E501 E128
     scrapy/commands/version.py E501 E128
     # scrapy/contracts
     scrapy/contracts/__init__.py E501
     scrapy/contracts/default.py E128
     # scrapy/core
-    scrapy/core/engine.py E501 E128 E127
+    scrapy/core/engine.py E501 E128
     scrapy/core/scheduler.py E501
     scrapy/core/scraper.py E501 E128
     scrapy/core/spidermw.py E501 E126
@@ -57,7 +57,7 @@ flake8-ignore =
     scrapy/core/downloader/tls.py E501
     scrapy/core/downloader/webclient.py E501 E128 E126
     scrapy/core/downloader/handlers/__init__.py E501
-    scrapy/core/downloader/handlers/ftp.py E501 E128 E127
+    scrapy/core/downloader/handlers/ftp.py E501 E128
     scrapy/core/downloader/handlers/http10.py E501
     scrapy/core/downloader/handlers/http11.py E501
     scrapy/core/downloader/handlers/s3.py E501 E128 E126
@@ -124,7 +124,7 @@ flake8-ignore =
     scrapy/utils/datatypes.py E501
     scrapy/utils/decorators.py E501
     scrapy/utils/defer.py E501 E128
-    scrapy/utils/deprecate.py E128 E501 E127
+    scrapy/utils/deprecate.py E501
     scrapy/utils/gz.py E501
     scrapy/utils/http.py F403
     scrapy/utils/httpobj.py E501
@@ -137,7 +137,7 @@ flake8-ignore =
     scrapy/utils/python.py E501
     scrapy/utils/reactor.py E501
     scrapy/utils/reqser.py E501
-    scrapy/utils/request.py E127 E501
+    scrapy/utils/request.py E501
     scrapy/utils/response.py E501 E128
     scrapy/utils/signal.py E501 E128
     scrapy/utils/sitemap.py E501
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 432cb1831..94b55c347 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -94,11 +94,12 @@ class FTPDownloadHandler:
     def gotClient(self, client, request, filepath):
         self.client = client
         protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename"))
-        return client.retrieveFile(filepath, protocol)\
-                .addCallbacks(callback=self._build_response,
-                        callbackArgs=(request, protocol),
-                        errback=self._failed,
-                        errbackArgs=(request,))
+        return client.retrieveFile(filepath, protocol).addCallbacks(
+            callback=self._build_response,
+            callbackArgs=(request, protocol),
+            errback=self._failed,
+            errbackArgs=(request,),
+        )
 
     def _build_response(self, result, request, protocol):
         self.result = result
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 77d71846e..324d21716 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -230,8 +230,7 @@ class ExecutionEngine:
 
     def _downloaded(self, response, slot, request, spider):
         slot.remove_request(request)
-        return self.download(response, spider) \
-                if isinstance(response, Request) else response
+        return self.download(response, spider) if isinstance(response, Request) else response
 
     def _download(self, request, spider):
         slot = self.slot
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index 36001d982..3dbea5fee 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -15,16 +15,17 @@ def attribute(obj, oldattr, newattr, version='0.12'):
         stacklevel=3)
 
 
-def create_deprecated_class(name, new_class, clsdict=None,
-                            warn_category=ScrapyDeprecationWarning,
-                            warn_once=True,
-                            old_class_path=None,
-                            new_class_path=None,
-                            subclass_warn_message="{cls} inherits from "
-                                    "deprecated class {old}, please inherit "
-                                    "from {new}.",
-                            instance_warn_message="{cls} is deprecated, "
-                                    "instantiate {new} instead."):
+def create_deprecated_class(
+    name,
+    new_class,
+    clsdict=None,
+    warn_category=ScrapyDeprecationWarning,
+    warn_once=True,
+    old_class_path=None,
+    new_class_path=None,
+    subclass_warn_message="{cls} inherits from deprecated class {old}, please inherit from {new}.",
+    instance_warn_message="{cls} is deprecated, instantiate {new} instead."
+):
     """
     Return a "deprecated" class that causes its subclasses to issue a warning.
     Subclasses of ``new_class`` are considered subclasses of this class.
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index b8c140a7e..12c03d78e 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -50,8 +50,7 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False):
 
     """
     if include_headers:
-        include_headers = tuple(to_bytes(h.lower())
-                                 for h in sorted(include_headers))
+        include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
     cache = _fingerprint_cache.setdefault(request, {})
     cache_key = (include_headers, keep_fragments)
     if cache_key not in cache:

From fe0c582ee083ad8085a33443af0ffbc67b44fc16 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 09:49:10 -0300
Subject: [PATCH 085/243] Flake8: remove E127 in tests (continuation line
 over-indented for visual indent)

---
 pytest.ini                                    |  18 +--
 tests/spiders.py                              |   3 +-
 tests/test_closespider.py                     |   3 +-
 tests/test_downloader_handlers.py             |   9 +-
 ...test_downloadermiddleware_decompression.py |   4 +-
 tests/test_downloadermiddleware_redirect.py   |   3 +-
 tests/test_http_request.py                    |   6 +-
 tests/test_selector.py                        |   3 +-
 tests/test_spidermiddleware_httperror.py      |   6 +-
 tests/test_utils_url.py                       | 120 ++++++++++--------
 10 files changed, 90 insertions(+), 85 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index fa65a0da2..3eefe70f1 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -171,8 +171,8 @@ flake8-ignore =
     tests/__init__.py E402 E501
     tests/mockserver.py E401 E501 E126 E123
     tests/pipelines.py F841
-    tests/spiders.py E501 E127
-    tests/test_closespider.py E501 E127
+    tests/spiders.py E501
+    tests/test_closespider.py E501
     tests/test_command_fetch.py E501
     tests/test_command_parse.py E501 E128
     tests/test_command_shell.py E501 E128
@@ -181,17 +181,17 @@ flake8-ignore =
     tests/test_crawl.py E501 E741
     tests/test_crawler.py F841 E501
     tests/test_dependencies.py F841 E501
-    tests/test_downloader_handlers.py E124 E127 E128 E501 E126 E123
+    tests/test_downloader_handlers.py E124 E128 E501 E126 E123
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
     tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126
-    tests/test_downloadermiddleware_decompression.py E127
     tests/test_downloadermiddleware_defaultheaders.py E501
     tests/test_downloadermiddleware_downloadtimeout.py E501
     tests/test_downloadermiddleware_httpcache.py E501
     tests/test_downloadermiddleware_httpcompression.py E501 E126 E123
+    tests/test_downloadermiddleware_decompression.py E501
     tests/test_downloadermiddleware_httpproxy.py E501 E128
-    tests/test_downloadermiddleware_redirect.py E501 E128 E127
+    tests/test_downloadermiddleware_redirect.py E501 E128
     tests/test_downloadermiddleware_retry.py E501 E128 E126
     tests/test_downloadermiddleware_robotstxt.py E501
     tests/test_downloadermiddleware_stats.py E501
@@ -202,7 +202,7 @@ flake8-ignore =
     tests/test_feedexport.py E501 F841
     tests/test_http_cookies.py E501
     tests/test_http_headers.py E501
-    tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
+    tests/test_http_request.py E402 E501 E128 E128 E126 E123
     tests/test_http_response.py E501 E128
     tests/test_item.py E128 F841
     tests/test_link.py E501
@@ -220,10 +220,10 @@ flake8-ignore =
     tests/test_responsetypes.py E501
     tests/test_robotstxt_interface.py E501 E501
     tests/test_scheduler.py E501 E126 E123
-    tests/test_selector.py E501 E127
+    tests/test_selector.py E501
     tests/test_spider.py E501
     tests/test_spidermiddleware.py E501
-    tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
+    tests/test_spidermiddleware_httperror.py E128 E501 E121
     tests/test_spidermiddleware_offsite.py E501 E128 E111
     tests/test_spidermiddleware_output_chain.py E501
     tests/test_spidermiddleware_referer.py E501 F841 E125 E124 E501 E121
@@ -243,7 +243,7 @@ flake8-ignore =
     tests/test_utils_response.py E501
     tests/test_utils_signal.py E741 F841
     tests/test_utils_sitemap.py E128 E501 E124
-    tests/test_utils_url.py E501 E127 E125 E501 E126 E123
+    tests/test_utils_url.py E501 E125 E501 E126 E123
     tests/test_webclient.py E501 E128 E122 E402 E123 E126
     tests/test_cmdline/__init__.py E501
     tests/test_settings/__init__.py E501 E128
diff --git a/tests/spiders.py b/tests/spiders.py
index 284c77829..33d5d02e1 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -184,8 +184,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
             if self.fail_yielding:
                 2 / 0
 
-        assert self.seedsseen, \
-                'All start requests consumed before any download happened'
+        assert self.seedsseen, 'All start requests consumed before any download happened'
 
     def parse(self, response):
         self.seedsseen.append(response.meta.get('seed'))
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index 4a56425b7..5ec5e2989 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -41,8 +41,7 @@ class TestCloseSpider(TestCase):
         yield crawler.crawl(total=1000000, mockserver=self.mockserver)
         reason = crawler.spider.meta['close_reason']
         self.assertEqual(reason, 'closespider_errorcount')
-        key = 'spider_exceptions/{name}'\
-                .format(name=crawler.spider.exception_cls.__name__)
+        key = 'spider_exceptions/{name}'.format(name=crawler.spider.exception_cls.__name__)
         errorcount = crawler.stats.get_value(key)
         self.assertTrue(errorcount >= close_on)
 
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 29d06bab4..24ef560c1 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1090,8 +1090,7 @@ class DataURITestCase(unittest.TestCase):
     def test_default_mediatype_encoding(self):
         def _test(response):
             self.assertEqual(response.text, 'A brief note')
-            self.assertEqual(type(response),
-                              responsetypes.from_mimetype("text/plain"))
+            self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
             self.assertEqual(response.encoding, "US-ASCII")
 
         request = Request("data:,A%20brief%20note")
@@ -1100,8 +1099,7 @@ class DataURITestCase(unittest.TestCase):
     def test_default_mediatype(self):
         def _test(response):
             self.assertEqual(response.text, u'\u038e\u03a3\u038e')
-            self.assertEqual(type(response),
-                              responsetypes.from_mimetype("text/plain"))
+            self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
             self.assertEqual(response.encoding, "iso-8859-7")
 
         request = Request("data:;charset=iso-8859-7,%be%d3%be")
@@ -1119,8 +1117,7 @@ class DataURITestCase(unittest.TestCase):
     def test_mediatype_parameters(self):
         def _test(response):
             self.assertEqual(response.text, u'\u038e\u03a3\u038e')
-            self.assertEqual(type(response),
-                              responsetypes.from_mimetype("text/plain"))
+            self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
             self.assertEqual(response.encoding, "utf-8")
 
         request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;'
diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py
index 77b35a8c3..dbae4d3ae 100644
--- a/tests/test_downloadermiddleware_decompression.py
+++ b/tests/test_downloadermiddleware_decompression.py
@@ -28,8 +28,8 @@ class DecompressionMiddlewareTest(TestCase):
         for fmt in self.test_formats:
             rsp = self.test_responses[fmt]
             new = self.mw.process_response(None, rsp, self.spider)
-            assert isinstance(new, XmlResponse), \
-                    'Failed %s, response type %s' % (fmt, type(new).__name__)
+            error_msg = 'Failed %s, response type %s' % (fmt, type(new).__name__)
+            assert isinstance(new, XmlResponse), error_msg
             assert_samelines(self, new.body, self.uncompressed_body, fmt)
 
     def test_plain_response(self):
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 053e26fc3..551e124ab 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -181,8 +181,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
             rsp = Response(url, headers={'Location': url2}, status=301, request=req)
             r = self.mw.process_response(req, rsp, self.spider)
             self.assertIs(r, rsp)
-        _test_passthrough(Request(url, meta={'handle_httpstatus_list':
-                                                           [404, 301, 302]}))
+        _test_passthrough(Request(url, meta={'handle_httpstatus_list': [404, 301, 302]}))
         _test_passthrough(Request(url, meta={'handle_httpstatus_all': True}))
 
     def test_latin1_location(self):
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index cc2cddda4..b12841ba2 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -399,8 +399,7 @@ class FormRequestTest(RequestTest):
 
     def test_custom_encoding_bytes(self):
         data = {b'\xb5 one': b'two', b'price': b'\xa3 100'}
-        r2 = self.request_class("http://www.example.com", formdata=data,
-                                    encoding='latin1')
+        r2 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
         self.assertEqual(r2.method, 'POST')
         self.assertEqual(r2.encoding, 'latin1')
         self.assertQueryEqual(r2.body, b'price=%A3+100&%B5+one=two')
@@ -408,8 +407,7 @@ class FormRequestTest(RequestTest):
 
     def test_custom_encoding_textual_data(self):
         data = {'price': u'£ 100'}
-        r3 = self.request_class("http://www.example.com", formdata=data,
-                                    encoding='latin1')
+        r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
         self.assertEqual(r3.encoding, 'latin1')
         self.assertEqual(r3.body, b'price=%A3+100')
 
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 09c2546fb..65b0f5860 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -67,8 +67,7 @@ class SelectorTestCase(unittest.TestCase):
         headers = {'Content-Type': ['text/html; charset=utf-8']}
         response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
         x = Selector(response)
-        self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(),
-                          [u'\xa3'])
+        self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3'])
 
     def test_badly_encoded_body(self):
         # \xe9 alone isn't valid utf8 sequence
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index dacd0147f..6b61df56f 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -111,8 +111,7 @@ class TestHttpErrorMiddlewareSettings(TestCase):
                 self.mw.process_spider_input(self.res402, self.spider))
 
     def test_meta_overrides_settings(self):
-        request = Request('http://scrapytest.org',
-                              meta={'handle_httpstatus_list': [404]})
+        request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
         res404 = self.res404.copy()
         res404.request = request
         res402 = self.res402.copy()
@@ -146,8 +145,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
                 self.mw.process_spider_input(self.res404, self.spider))
 
     def test_meta_overrides_settings(self):
-        request = Request('http://scrapytest.org',
-                              meta={'handle_httpstatus_list': [404]})
+        request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
         res404 = self.res404.copy()
         res404.request = request
         res402 = self.res402.copy()
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 72a16e9b1..3bb6d40db 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -77,108 +77,124 @@ class UrlUtilsTest(unittest.TestCase):
 class AddHttpIfNoScheme(unittest.TestCase):
 
     def test_add_scheme(self):
-        self.assertEqual(add_http_if_no_scheme('www.example.com'),
-                                               'http://www.example.com')
+        self.assertEqual(add_http_if_no_scheme('www.example.com'), 'http://www.example.com')
 
     def test_without_subdomain(self):
-        self.assertEqual(add_http_if_no_scheme('example.com'),
-                                               'http://example.com')
+        self.assertEqual(add_http_if_no_scheme('example.com'), 'http://example.com')
 
     def test_path(self):
-        self.assertEqual(add_http_if_no_scheme('www.example.com/some/page.html'),
-                                               'http://www.example.com/some/page.html')
+        self.assertEqual(
+            add_http_if_no_scheme('www.example.com/some/page.html'),
+            'http://www.example.com/some/page.html')
 
     def test_port(self):
-        self.assertEqual(add_http_if_no_scheme('www.example.com:80'),
-                                               'http://www.example.com:80')
+        self.assertEqual(
+            add_http_if_no_scheme('www.example.com:80'),
+            'http://www.example.com:80')
 
     def test_fragment(self):
-        self.assertEqual(add_http_if_no_scheme('www.example.com/some/page#frag'),
-                                               'http://www.example.com/some/page#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('www.example.com/some/page#frag'),
+            'http://www.example.com/some/page#frag')
 
     def test_query(self):
-        self.assertEqual(add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
-                                               'http://www.example.com/do?a=1&b=2&c=3')
+        self.assertEqual(
+            add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
+            'http://www.example.com/do?a=1&b=2&c=3')
 
     def test_username_password(self):
-        self.assertEqual(add_http_if_no_scheme('username:password@www.example.com'),
-                                               'http://username:password@www.example.com')
+        self.assertEqual(
+            add_http_if_no_scheme('username:password@www.example.com'),
+            'http://username:password@www.example.com')
 
     def test_complete_url(self):
-        self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
-                                               'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
+            'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
 
     def test_preserve_http(self):
-        self.assertEqual(add_http_if_no_scheme('http://www.example.com'),
-                                               'http://www.example.com')
+        self.assertEqual(add_http_if_no_scheme('http://www.example.com'), 'http://www.example.com')
 
     def test_preserve_http_without_subdomain(self):
-        self.assertEqual(add_http_if_no_scheme('http://example.com'),
-                                               'http://example.com')
+        self.assertEqual(
+            add_http_if_no_scheme('http://example.com'),
+            'http://example.com')
 
     def test_preserve_http_path(self):
-        self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page.html'),
-                                               'http://www.example.com/some/page.html')
+        self.assertEqual(
+            add_http_if_no_scheme('http://www.example.com/some/page.html'),
+            'http://www.example.com/some/page.html')
 
     def test_preserve_http_port(self):
-        self.assertEqual(add_http_if_no_scheme('http://www.example.com:80'),
-                                               'http://www.example.com:80')
+        self.assertEqual(
+            add_http_if_no_scheme('http://www.example.com:80'),
+            'http://www.example.com:80')
 
     def test_preserve_http_fragment(self):
-        self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page#frag'),
-                                               'http://www.example.com/some/page#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('http://www.example.com/some/page#frag'),
+            'http://www.example.com/some/page#frag')
 
     def test_preserve_http_query(self):
-        self.assertEqual(add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
-                                               'http://www.example.com/do?a=1&b=2&c=3')
+        self.assertEqual(
+            add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
+            'http://www.example.com/do?a=1&b=2&c=3')
 
     def test_preserve_http_username_password(self):
-        self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com'),
-                                               'http://username:password@www.example.com')
+        self.assertEqual(
+            add_http_if_no_scheme('http://username:password@www.example.com'),
+            'http://username:password@www.example.com')
 
     def test_preserve_http_complete_url(self):
-        self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
-                                               'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
+            'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
 
     def test_protocol_relative(self):
-        self.assertEqual(add_http_if_no_scheme('//www.example.com'),
-                                               'http://www.example.com')
+        self.assertEqual(
+            add_http_if_no_scheme('//www.example.com'), 'http://www.example.com')
 
     def test_protocol_relative_without_subdomain(self):
-        self.assertEqual(add_http_if_no_scheme('//example.com'),
-                                               'http://example.com')
+        self.assertEqual(
+            add_http_if_no_scheme('//example.com'), 'http://example.com')
 
     def test_protocol_relative_path(self):
-        self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page.html'),
-                                               'http://www.example.com/some/page.html')
+        self.assertEqual(
+            add_http_if_no_scheme('//www.example.com/some/page.html'),
+            'http://www.example.com/some/page.html')
 
     def test_protocol_relative_port(self):
-        self.assertEqual(add_http_if_no_scheme('//www.example.com:80'),
-                                               'http://www.example.com:80')
+        self.assertEqual(
+            add_http_if_no_scheme('//www.example.com:80'),
+            'http://www.example.com:80')
 
     def test_protocol_relative_fragment(self):
-        self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page#frag'),
-                                               'http://www.example.com/some/page#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('//www.example.com/some/page#frag'),
+            'http://www.example.com/some/page#frag')
 
     def test_protocol_relative_query(self):
-        self.assertEqual(add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),
-                                               'http://www.example.com/do?a=1&b=2&c=3')
+        self.assertEqual(
+            add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),
+            'http://www.example.com/do?a=1&b=2&c=3')
 
     def test_protocol_relative_username_password(self):
-        self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com'),
-                                               'http://username:password@www.example.com')
+        self.assertEqual(
+            add_http_if_no_scheme('//username:password@www.example.com'),
+            'http://username:password@www.example.com')
 
     def test_protocol_relative_complete_url(self):
-        self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
-                                               'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
+            'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
 
     def test_preserve_https(self):
-        self.assertEqual(add_http_if_no_scheme('https://www.example.com'),
-                                               'https://www.example.com')
+        self.assertEqual(
+            add_http_if_no_scheme('https://www.example.com'),
+            'https://www.example.com')
 
     def test_preserve_ftp(self):
-        self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'),
-                                               'ftp://www.example.com')
+        self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'), 'ftp://www.example.com')
 
 
 class GuessSchemeTest(unittest.TestCase):

From 63600243e08cb7e783798bd6c59fb97595488e9e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 10:21:01 -0300
Subject: [PATCH 086/243] Flake8: remove E125 (Continuation line with same
 indent as next logical line)

Also remove E401 from pytest.ini - no occurrences in the codebase
---
 pytest.ini                             | 12 ++++----
 scrapy/pipelines/media.py              | 10 ++++---
 tests/test_spidermiddleware_referer.py | 40 +++++++++++++-------------
 tests/test_utils_url.py                | 14 ++++-----
 4 files changed, 39 insertions(+), 37 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 3eefe70f1..8ed1ad0cf 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -35,7 +35,7 @@ flake8-ignore =
     scrapy/commands/check.py E501
     scrapy/commands/crawl.py E501
     scrapy/commands/edit.py E501
-    scrapy/commands/fetch.py E401 E501 E128
+    scrapy/commands/fetch.py E501 E128
     scrapy/commands/genspider.py E128 E501
     scrapy/commands/parse.py E128 E501
     scrapy/commands/runspider.py E501
@@ -99,7 +99,7 @@ flake8-ignore =
     scrapy/pipelines/__init__.py E501
     scrapy/pipelines/files.py E116 E501
     scrapy/pipelines/images.py E501
-    scrapy/pipelines/media.py E125 E501
+    scrapy/pipelines/media.py E501
     # scrapy/selector
     scrapy/selector/__init__.py F403
     scrapy/selector/unified.py E501 E111
@@ -169,7 +169,7 @@ flake8-ignore =
     scrapy/statscollectors.py E501
     # tests
     tests/__init__.py E402 E501
-    tests/mockserver.py E401 E501 E126 E123
+    tests/mockserver.py E501 E126 E123
     tests/pipelines.py F841
     tests/spiders.py E501
     tests/test_closespider.py E501
@@ -196,7 +196,7 @@ flake8-ignore =
     tests/test_downloadermiddleware_robotstxt.py E501
     tests/test_downloadermiddleware_stats.py E501
     tests/test_dupefilters.py E501 E741 E128 E124
-    tests/test_engine.py E401 E501 E128
+    tests/test_engine.py E501 E128
     tests/test_exporters.py E501 E128 E124
     tests/test_extension_telnet.py F841
     tests/test_feedexport.py E501 F841
@@ -226,7 +226,7 @@ flake8-ignore =
     tests/test_spidermiddleware_httperror.py E128 E501 E121
     tests/test_spidermiddleware_offsite.py E501 E128 E111
     tests/test_spidermiddleware_output_chain.py E501
-    tests/test_spidermiddleware_referer.py E501 F841 E125 E124 E501 E121
+    tests/test_spidermiddleware_referer.py E501 F841 E124 E501 E121
     tests/test_squeues.py E501 E741
     tests/test_utils_asyncio.py E501
     tests/test_utils_conf.py E501 E128
@@ -243,7 +243,7 @@ flake8-ignore =
     tests/test_utils_response.py E501
     tests/test_utils_signal.py E741 F841
     tests/test_utils_sitemap.py E128 E501 E124
-    tests/test_utils_url.py E501 E125 E501 E126 E123
+    tests/test_utils_url.py E501 E501 E126 E123
     tests/test_webclient.py E501 E128 E122 E402 E123 E126
     tests/test_cmdline/__init__.py E501
     tests/test_settings/__init__.py E501 E128
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 8a0636264..aa65f4f0e 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -43,8 +43,7 @@ class MediaPipeline:
         if allow_redirects:
             self.handle_httpstatus_list = SequenceExclude(range(300, 400))
 
-    def _key_for_pipe(self, key, base_class_name=None,
-                      settings=None):
+    def _key_for_pipe(self, key, base_class_name=None, settings=None):
         """
         >>> MediaPipeline()._key_for_pipe("IMAGES")
         'IMAGES'
@@ -55,8 +54,11 @@ class MediaPipeline:
         """
         class_name = self.__class__.__name__
         formatted_key = "{}_{}".format(class_name.upper(), key)
-        if class_name == base_class_name or not base_class_name \
-            or (settings and not settings.get(formatted_key)):
+        if (
+            not base_class_name
+            or class_name == base_class_name
+            or settings and not settings.get(formatted_key)
+        ):
             return key
         return formatted_key
 
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 742adc64f..41589177a 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -478,32 +478,32 @@ class TestSettingsPolicyByName(TestCase):
 
     def test_valid_name(self):
         for s, p in [
-                (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
-                (POLICY_NO_REFERRER, NoReferrerPolicy),
-                (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
-                (POLICY_SAME_ORIGIN, SameOriginPolicy),
-                (POLICY_ORIGIN, OriginPolicy),
-                (POLICY_STRICT_ORIGIN, StrictOriginPolicy),
-                (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
-                (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
-                (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
-            ]:
+            (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
+            (POLICY_NO_REFERRER, NoReferrerPolicy),
+            (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
+            (POLICY_SAME_ORIGIN, SameOriginPolicy),
+            (POLICY_ORIGIN, OriginPolicy),
+            (POLICY_STRICT_ORIGIN, StrictOriginPolicy),
+            (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
+            (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
+            (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
+        ]:
             settings = Settings({'REFERRER_POLICY': s})
             mw = RefererMiddleware(settings)
             self.assertEqual(mw.default_policy, p)
 
     def test_valid_name_casevariants(self):
         for s, p in [
-                (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
-                (POLICY_NO_REFERRER, NoReferrerPolicy),
-                (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
-                (POLICY_SAME_ORIGIN, SameOriginPolicy),
-                (POLICY_ORIGIN, OriginPolicy),
-                (POLICY_STRICT_ORIGIN, StrictOriginPolicy),
-                (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
-                (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
-                (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
-            ]:
+            (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
+            (POLICY_NO_REFERRER, NoReferrerPolicy),
+            (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
+            (POLICY_SAME_ORIGIN, SameOriginPolicy),
+            (POLICY_ORIGIN, OriginPolicy),
+            (POLICY_STRICT_ORIGIN, StrictOriginPolicy),
+            (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
+            (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
+            (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
+        ]:
             settings = Settings({'REFERRER_POLICY': s.upper()})
             mw = RefererMiddleware(settings)
             self.assertEqual(mw.default_policy, p)
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 3bb6d40db..bed1a5634 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -288,7 +288,7 @@ class StripUrl(unittest.TestCase):
             ('http://www.example.com',
              True,
              'http://www.example.com/'),
-            ]:
+        ]:
             self.assertEqual(strip_url(input_url, origin_only=origin), output_url)
 
     def test_credentials(self):
@@ -301,7 +301,7 @@ class StripUrl(unittest.TestCase):
 
             ('ftp://username:password@www.example.com/index.html?somekey=somevalue#section',
              'ftp://www.example.com/index.html?somekey=somevalue'),
-            ]:
+        ]:
             self.assertEqual(strip_url(i, strip_credentials=True), o)
 
     def test_credentials_encoded_delims(self):
@@ -320,7 +320,7 @@ class StripUrl(unittest.TestCase):
             # password: "user@domain.com"
             ('ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section',
              'ftp://www.example.com/index.html?somekey=somevalue'),
-            ]:
+        ]:
             self.assertEqual(strip_url(i, strip_credentials=True), o)
 
     def test_default_ports_creds_off(self):
@@ -348,7 +348,7 @@ class StripUrl(unittest.TestCase):
 
             ('ftp://username:password@www.example.com:221/file.txt',
              'ftp://www.example.com:221/file.txt'),
-            ]:
+        ]:
             self.assertEqual(strip_url(i), o)
 
     def test_default_ports(self):
@@ -376,7 +376,7 @@ class StripUrl(unittest.TestCase):
 
             ('ftp://username:password@www.example.com:221/file.txt',
              'ftp://username:password@www.example.com:221/file.txt'),
-            ]:
+        ]:
             self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o)
 
     def test_default_ports_keep(self):
@@ -404,7 +404,7 @@ class StripUrl(unittest.TestCase):
 
             ('ftp://username:password@www.example.com:221/file.txt',
              'ftp://username:password@www.example.com:221/file.txt'),
-            ]:
+        ]:
             self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o)
 
     def test_origin_only(self):
@@ -420,7 +420,7 @@ class StripUrl(unittest.TestCase):
 
             ('https://username:password@www.example.com:443/index.html',
              'https://www.example.com/'),
-            ]:
+        ]:
             self.assertEqual(strip_url(i, origin_only=True), o)
 
 

From 628c4a531914b6803ae0ec4991363aad52069ca1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Panek?= <michal@panek.cloud>
Date: Wed, 6 May 2020 17:09:20 +0200
Subject: [PATCH 087/243] Add a warning/error in case of incorrect gcs
 permissions (#4508)

---
 scrapy/pipelines/files.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index ae365db5b..a9066986b 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -230,6 +230,20 @@ class GCSFilesStore:
         bucket, prefix = uri[5:].split('/', 1)
         self.bucket = client.bucket(bucket)
         self.prefix = prefix
+        permissions = self.bucket.test_iam_permissions(
+            ['storage.objects.get', 'storage.objects.create']
+        )
+        if 'storage.objects.get' not in permissions:
+            logger.warning(
+                "No 'storage.objects.get' permission for GSC bucket %(bucket)s. "
+                "Checking if files are up to date will be impossible. Files will be downloaded every time.",
+                {'bucket': bucket}
+            )
+        if 'storage.objects.create' not in permissions:
+            logger.error(
+                "No 'storage.objects.create' permission for GSC bucket %(bucket)s. Saving files will be impossible!",
+                {'bucket': bucket}
+            )
 
     def stat_file(self, path, info):
         def _onsuccess(blob):

From 8643e8d3557449393989b15b9b8f2ec813f3e6ad Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 12:26:04 -0300
Subject: [PATCH 088/243] Flake8: remove E123 (Closing bracket does not match
 indentation of opening bracket's line)

---
 pytest.ini                                    | 18 +++---
 scrapy/extensions/closespider.py              |  2 +-
 scrapy/http/request/form.py                   | 11 ++--
 tests/test_downloader_handlers.py             | 19 +++---
 ...st_downloadermiddleware_httpcompression.py | 24 +++----
 tests/test_http_request.py                    |  2 +-
 tests/test_scheduler.py                       | 30 +++++----
 tests/test_utils_url.py                       | 64 +++++++++++--------
 tests/test_webclient.py                       |  6 +-
 9 files changed, 95 insertions(+), 81 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 8ed1ad0cf..1a73b41be 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -73,7 +73,7 @@ flake8-ignore =
     scrapy/downloadermiddlewares/robotstxt.py E501
     scrapy/downloadermiddlewares/stats.py E501
     # scrapy/extensions
-    scrapy/extensions/closespider.py E501 E128 E123
+    scrapy/extensions/closespider.py E501 E128
     scrapy/extensions/corestats.py E501
     scrapy/extensions/feedexport.py E128 E501
     scrapy/extensions/httpcache.py E128 E501
@@ -85,7 +85,7 @@ flake8-ignore =
     scrapy/http/common.py E501
     scrapy/http/cookies.py E501
     scrapy/http/request/__init__.py E501
-    scrapy/http/request/form.py E501 E123
+    scrapy/http/request/form.py E501
     scrapy/http/request/json_request.py E501
     scrapy/http/response/__init__.py E501 E128
     scrapy/http/response/text.py E501 E128 E124
@@ -169,7 +169,7 @@ flake8-ignore =
     scrapy/statscollectors.py E501
     # tests
     tests/__init__.py E402 E501
-    tests/mockserver.py E501 E126 E123
+    tests/mockserver.py E501 E126
     tests/pipelines.py F841
     tests/spiders.py E501
     tests/test_closespider.py E501
@@ -181,14 +181,14 @@ flake8-ignore =
     tests/test_crawl.py E501 E741
     tests/test_crawler.py F841 E501
     tests/test_dependencies.py F841 E501
-    tests/test_downloader_handlers.py E124 E128 E501 E126 E123
+    tests/test_downloader_handlers.py E124 E128 E501 E126
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
     tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126
     tests/test_downloadermiddleware_defaultheaders.py E501
     tests/test_downloadermiddleware_downloadtimeout.py E501
     tests/test_downloadermiddleware_httpcache.py E501
-    tests/test_downloadermiddleware_httpcompression.py E501 E126 E123
+    tests/test_downloadermiddleware_httpcompression.py E501 E126
     tests/test_downloadermiddleware_decompression.py E501
     tests/test_downloadermiddleware_httpproxy.py E501 E128
     tests/test_downloadermiddleware_redirect.py E501 E128
@@ -202,7 +202,7 @@ flake8-ignore =
     tests/test_feedexport.py E501 F841
     tests/test_http_cookies.py E501
     tests/test_http_headers.py E501
-    tests/test_http_request.py E402 E501 E128 E128 E126 E123
+    tests/test_http_request.py E402 E501 E128 E128 E126
     tests/test_http_response.py E501 E128
     tests/test_item.py E128 F841
     tests/test_link.py E501
@@ -219,7 +219,7 @@ flake8-ignore =
     tests/test_request_cb_kwargs.py E501
     tests/test_responsetypes.py E501
     tests/test_robotstxt_interface.py E501 E501
-    tests/test_scheduler.py E501 E126 E123
+    tests/test_scheduler.py E501 E126
     tests/test_selector.py E501
     tests/test_spider.py E501
     tests/test_spidermiddleware.py E501
@@ -243,8 +243,8 @@ flake8-ignore =
     tests/test_utils_response.py E501
     tests/test_utils_signal.py E741 F841
     tests/test_utils_sitemap.py E128 E501 E124
-    tests/test_utils_url.py E501 E501 E126 E123
-    tests/test_webclient.py E501 E128 E122 E402 E123 E126
+    tests/test_utils_url.py E501 E501 E126
+    tests/test_webclient.py E501 E128 E122 E402 E126
     tests/test_cmdline/__init__.py E501
     tests/test_settings/__init__.py E501 E128
     tests/test_spiderloader/__init__.py E128 E501
diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py
index e3f212bef..812844c0a 100644
--- a/scrapy/extensions/closespider.py
+++ b/scrapy/extensions/closespider.py
@@ -20,7 +20,7 @@ class CloseSpider:
             'itemcount': crawler.settings.getint('CLOSESPIDER_ITEMCOUNT'),
             'pagecount': crawler.settings.getint('CLOSESPIDER_PAGECOUNT'),
             'errorcount': crawler.settings.getint('CLOSESPIDER_ERRORCOUNT'),
-            }
+        }
 
         if not any(self.close_on.values()):
             raise NotConfigured
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index af02c8484..cd4e3373f 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -178,12 +178,11 @@ def _get_clickable(clickdata, form):
     if the latter is given. If not, it returns the first
     clickable element found
     """
-    clickables = [
-        el for el in form.xpath(
-            'descendant::input[re:test(@type, "^(submit|image)$", "i")]'
-            '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]',
-            namespaces={"re": "http://exslt.org/regular-expressions"})
-        ]
+    clickables = list(form.xpath(
+        'descendant::input[re:test(@type, "^(submit|image)$", "i")]'
+        '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]',
+        namespaces={"re": "http://exslt.org/regular-expressions"}
+    ))
     if not clickables:
         return
 
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 24ef560c1..f93bce8ef 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -822,11 +822,15 @@ class S3TestCase(unittest.TestCase):
     def test_request_signing2(self):
         # puts an object into the johnsmith bucket.
         date = 'Tue, 27 Mar 2007 21:15:45 +0000'
-        req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={
-            'Content-Type': 'image/jpeg',
-            'Date': date,
-            'Content-Length': '94328',
-            })
+        req = Request(
+            's3://johnsmith/photos/puppy.jpg',
+            method='PUT',
+            headers={
+                'Content-Type': 'image/jpeg',
+                'Date': date,
+                'Content-Length': '94328',
+            },
+        )
         with self._mocked_date(date):
             httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'],
@@ -906,11 +910,10 @@ class S3TestCase(unittest.TestCase):
         # ensure that spaces are quoted properly before signing
         date = 'Tue, 27 Mar 2007 19:42:41 +0000'
         req = Request(
-            ("s3://johnsmith/photos/my puppy.jpg"
-             "?response-content-disposition=my puppy.jpg"),
+            "s3://johnsmith/photos/my puppy.jpg?response-content-disposition=my puppy.jpg",
             method='GET',
             headers={'Date': date},
-            )
+        )
         with self._mocked_date(date):
             httpreq = self.download_request(req, self.spider)
         self.assertEqual(
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 106ca3360..e86568bfb 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -16,12 +16,12 @@ from w3lib.encoding import resolve_encoding
 SAMPLEDIR = join(tests_datadir, 'compressed')
 
 FORMAT = {
-        'gzip': ('html-gzip.bin', 'gzip'),
-        'x-gzip': ('html-gzip.bin', 'gzip'),
-        'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
-        'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
-        'br': ('html-br.bin', 'br')
-        }
+    'gzip': ('html-gzip.bin', 'gzip'),
+    'x-gzip': ('html-gzip.bin', 'gzip'),
+    'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
+    'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
+    'br': ('html-br.bin', 'br'),
+}
 
 
 class HttpCompressionTest(TestCase):
@@ -40,12 +40,12 @@ class HttpCompressionTest(TestCase):
             body = sample.read()
 
         headers = {
-                'Server': 'Yaws/1.49 Yet Another Web Server',
-                'Date': 'Sun, 08 Mar 2009 00:41:03 GMT',
-                'Content-Length': len(body),
-                'Content-Type': 'text/html',
-                'Content-Encoding': contentencoding,
-                }
+            'Server': 'Yaws/1.49 Yet Another Web Server',
+            'Date': 'Sun, 08 Mar 2009 00:41:03 GMT',
+            'Content-Length': len(body),
+            'Content-Type': 'text/html',
+            'Content-Encoding': contentencoding,
+        }
 
         response = Response('http://scrapytest.org/', body=body, headers=headers)
         response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'})
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index b12841ba2..3b6d119a9 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -467,7 +467,7 @@ class FormRequestTest(RequestTest):
             </form>""",
             url="http://www.example.com/this/list.html",
             encoding='latin1',
-            )
+        )
         req = self.request_class.from_response(response,
                 formdata={'one': ['two', 'three'], 'six': 'seven'})
 
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 00568aee9..930a5dd99 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -46,13 +46,13 @@ class MockCrawler(Crawler):
     def __init__(self, priority_queue_cls, jobdir):
 
         settings = dict(
-                SCHEDULER_DEBUG=False,
-                SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue',
-                SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue',
-                SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
-                JOBDIR=jobdir,
-                DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter'
-                )
+            SCHEDULER_DEBUG=False,
+            SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue',
+            SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue',
+            SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
+            JOBDIR=jobdir,
+            DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter',
+        )
         super(MockCrawler, self).__init__(Spider, settings)
         self.engine = MockEngine(downloader=MockDownloader())
 
@@ -305,10 +305,12 @@ class StartUrlsSpider(Spider):
 class TestIntegrationWithDownloaderAwareInMemory(TestCase):
     def setUp(self):
         self.crawler = get_crawler(
-                    StartUrlsSpider,
-                    {'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue',
-                     'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter'}
-                    )
+            spidercls=StartUrlsSpider,
+            settings_dict={
+                'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue',
+                'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter',
+            },
+        )
 
     @defer.inlineCallbacks
     def tearDown(self):
@@ -329,9 +331,9 @@ class TestIncompatibility(unittest.TestCase):
 
     def _incompatible(self):
         settings = dict(
-                SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',
-                CONCURRENT_REQUESTS_PER_IP=1
-                )
+            SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',
+            CONCURRENT_REQUESTS_PER_IP=1,
+        )
         crawler = Crawler(Spider, settings)
         scheduler = Scheduler.from_crawler(crawler)
         spider = Spider(name='spider')
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index bed1a5634..1f8388957 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -218,41 +218,49 @@ def create_skipped_scheme_t(args):
     return do_expected
 
 
-for k, args in enumerate([
-            ('/index', 'file://'),
-            ('/index.html', 'file://'),
-            ('./index.html', 'file://'),
-            ('../index.html', 'file://'),
-            ('../../index.html', 'file://'),
-            ('./data/index.html', 'file://'),
-            ('.hidden/data/index.html', 'file://'),
-            ('/home/user/www/index.html', 'file://'),
-            ('//home/user/www/index.html', 'file://'),
-            ('file:///home/user/www/index.html', 'file://'),
+for k, args in enumerate(
+    [
+        ('/index', 'file://'),
+        ('/index.html', 'file://'),
+        ('./index.html', 'file://'),
+        ('../index.html', 'file://'),
+        ('../../index.html', 'file://'),
+        ('./data/index.html', 'file://'),
+        ('.hidden/data/index.html', 'file://'),
+        ('/home/user/www/index.html', 'file://'),
+        ('//home/user/www/index.html', 'file://'),
+        ('file:///home/user/www/index.html', 'file://'),
 
-            ('index.html', 'http://'),
-            ('example.com', 'http://'),
-            ('www.example.com', 'http://'),
-            ('www.example.com/index.html', 'http://'),
-            ('http://example.com', 'http://'),
-            ('http://example.com/index.html', 'http://'),
-            ('localhost', 'http://'),
-            ('localhost/index.html', 'http://'),
+        ('index.html', 'http://'),
+        ('example.com', 'http://'),
+        ('www.example.com', 'http://'),
+        ('www.example.com/index.html', 'http://'),
+        ('http://example.com', 'http://'),
+        ('http://example.com/index.html', 'http://'),
+        ('localhost', 'http://'),
+        ('localhost/index.html', 'http://'),
 
-            # some corner cases (default to http://)
-            ('/', 'http://'),
-            ('.../test', 'http://'),
-
-        ], start=1):
+        # some corner cases (default to http://)
+        ('/', 'http://'),
+        ('.../test', 'http://'),
+    ],
+    start=1,
+):
     t_method = create_guess_scheme_t(args)
     t_method.__name__ = 'test_uri_%03d' % k
     setattr(GuessSchemeTest, t_method.__name__, t_method)
 
 # TODO: the following tests do not pass with current implementation
-for k, args in enumerate([
-            (r'C:\absolute\path\to\a\file.html', 'file://',
-             'Windows filepath are not supported for scrapy shell'),
-        ], start=1):
+for k, args in enumerate(
+    [
+        (
+            r'C:\absolute\path\to\a\file.html',
+            'file://',
+            'Windows filepath are not supported for scrapy shell',
+        ),
+    ],
+    start=1,
+):
     t_method = create_skipped_scheme_t(args)
     t_method.__name__ = 'test_uri_skipped_%03d' % k
     setattr(GuessSchemeTest, t_method.__name__, t_method)
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index d4abebbfb..de61e2125 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -149,7 +149,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             headers={
                 'X-Meta-Single': 'single',
                 'X-Meta-Multivalued': ['value1', 'value2'],
-                }))
+            },
+        ))
 
         self._test(factory,
             b"GET /bar HTTP/1.0\r\n"
@@ -165,7 +166,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             headers=Headers({
                 'X-Meta-Single': 'single',
                 'X-Meta-Multivalued': ['value1', 'value2'],
-                })))
+            }),
+        ))
 
         self._test(factory,
             b"GET /bar HTTP/1.0\r\n"

From d0bb04f08936435202488404d08c0b82f25aa1e5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 13:37:23 -0300
Subject: [PATCH 089/243] Switch to pickle protocol 4

---
 scrapy/exporters.py              | 2 +-
 scrapy/extensions/httpcache.py   | 4 ++--
 scrapy/extensions/spiderstate.py | 2 +-
 scrapy/squeues.py                | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 0cb6cef98..349a9586b 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -250,7 +250,7 @@ class CsvItemExporter(BaseItemExporter):
 
 class PickleItemExporter(BaseItemExporter):
 
-    def __init__(self, file, protocol=2, **kwargs):
+    def __init__(self, file, protocol=4, **kwargs):
         super().__init__(**kwargs)
         self.file = file
         self.protocol = protocol
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 8546628a8..7972b58b1 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -250,7 +250,7 @@ class DbmCacheStorage:
             'headers': dict(response.headers),
             'body': response.body,
         }
-        self.db['%s_data' % key] = pickle.dumps(data, protocol=2)
+        self.db['%s_data' % key] = pickle.dumps(data, protocol=4)
         self.db['%s_time' % key] = str(time())
 
     def _read_data(self, spider, request):
@@ -317,7 +317,7 @@ class FilesystemCacheStorage:
         with self._open(os.path.join(rpath, 'meta'), 'wb') as f:
             f.write(to_bytes(repr(metadata)))
         with self._open(os.path.join(rpath, 'pickled_meta'), 'wb') as f:
-            pickle.dump(metadata, f, protocol=2)
+            pickle.dump(metadata, f, protocol=4)
         with self._open(os.path.join(rpath, 'response_headers'), 'wb') as f:
             f.write(headers_dict_to_raw(response.headers))
         with self._open(os.path.join(rpath, 'response_body'), 'wb') as f:
diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py
index 2e5ff569f..bea00596e 100644
--- a/scrapy/extensions/spiderstate.py
+++ b/scrapy/extensions/spiderstate.py
@@ -26,7 +26,7 @@ class SpiderState:
     def spider_closed(self, spider):
         if self.jobdir:
             with open(self.statefn, 'wb') as f:
-                pickle.dump(spider.state, f, protocol=2)
+                pickle.dump(spider.state, f, protocol=4)
 
     def spider_opened(self, spider):
         if self.jobdir and os.path.exists(self.statefn):
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index d0686dac3..8d05bd0d0 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -81,7 +81,7 @@ def _scrapy_non_serialization_queue(queue_class):
 
 def _pickle_serialize(obj):
     try:
-        return pickle.dumps(obj, protocol=2)
+        return pickle.dumps(obj, protocol=4)
     # Python <= 3.4 raises pickle.PicklingError here while
     # 3.5 <= Python < 3.6 raises AttributeError and
     # Python >= 3.6 raises TypeError

From b1ddd7bd7b84d8d8417228aa7392d418463c9728 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 13:44:02 -0300
Subject: [PATCH 090/243] Refactor test_squeues.py

---
 tests/test_squeues.py | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index 5ad8035f7..7e997a25e 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -47,12 +47,7 @@ def nonserializable_object_test(self):
     self.assertRaises(ValueError, q.push, sel)
 
 
-class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
-
-    chunksize = 100000
-
-    def queue(self):
-        return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize)
+class FifoDiskQueueTestMixin:
 
     def test_serialize(self):
         q = self.queue()
@@ -66,6 +61,13 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
     test_nonserializable_object = nonserializable_object_test
 
 
+class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
+    chunksize = 100000
+
+    def queue(self):
+        return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize)
+
+
 class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
     chunksize = 1
 
@@ -82,7 +84,7 @@ class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
     chunksize = 4
 
 
-class PickleFifoDiskQueueTest(MarshalFifoDiskQueueTest):
+class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
 
     chunksize = 100000
 
@@ -133,10 +135,7 @@ class ChunkSize4PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
     chunksize = 4
 
 
-class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest):
-
-    def queue(self):
-        return MarshalLifoDiskQueue(self.qpath)
+class LifoDiskQueueTestMixin:
 
     def test_serialize(self):
         q = self.queue()
@@ -150,7 +149,13 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest):
     test_nonserializable_object = nonserializable_object_test
 
 
-class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest):
+class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
+
+    def queue(self):
+        return MarshalLifoDiskQueue(self.qpath)
+
+
+class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
 
     def queue(self):
         return PickleLifoDiskQueue(self.qpath)

From 93436f9d3a67cd8abe3b321321c2d36d94f75b8b Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 14:05:27 -0300
Subject: [PATCH 091/243] Chain pickling exception, test_squeues.py updates

---
 scrapy/squeues.py     |  7 +++----
 tests/test_squeues.py | 28 ++++++++++++++--------------
 2 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 8d05bd0d0..c7ad4d53d 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -82,11 +82,10 @@ def _scrapy_non_serialization_queue(queue_class):
 def _pickle_serialize(obj):
     try:
         return pickle.dumps(obj, protocol=4)
-    # Python <= 3.4 raises pickle.PicklingError here while
-    # 3.5 <= Python < 3.6 raises AttributeError and
-    # Python >= 3.6 raises TypeError
+    # Both pickle.PicklingError and AttributeError can be raised by pickle.dump(s)
+    # TypeError is raised from parsel.Selector
     except (pickle.PicklingError, AttributeError, TypeError) as e:
-        raise ValueError(str(e))
+        raise ValueError(str(e)) from e
 
 
 PickleFifoDiskQueueNonRequest = _serializable_queue(
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index 7e997a25e..a20d242f4 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -28,20 +28,7 @@ class TestLoader(ItemLoader):
 
 def nonserializable_object_test(self):
     q = self.queue()
-    try:
-        pickle.dumps(lambda x: x)
-    except Exception:
-        # Trigger Twisted bug #7989
-        import twisted.persisted.styles  # NOQA
-        self.assertRaises(ValueError, q.push, lambda x: x)
-    else:
-        # Use a different unpickleable object
-        class A:
-            pass
-
-        a = A()
-        a.__reduce__ = a.__reduce_ex__ = None
-        self.assertRaises(ValueError, q.push, a)
+    self.assertRaises(ValueError, q.push, lambda x: x)
     # Selectors should fail (lxml.html.HtmlElement objects can't be pickled)
     sel = Selector(text='<html><body><p>some text</p></body></html>')
     self.assertRaises(ValueError, q.push, sel)
@@ -118,6 +105,19 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
         self.assertEqual(r.url, r2.url)
         assert r2.meta['request'] is r2
 
+    def test_non_pickable_object(self):
+        q = self.queue()
+        try:
+            q.push(lambda x: x)
+        except ValueError as exc:
+            self.assertIsInstance(exc.__context__, AttributeError)
+
+        sel = Selector(text='<html><body><p>some text</p></body></html>')
+        try:
+            q.push(sel)
+        except ValueError as exc:
+            self.assertIsInstance(exc.__context__, TypeError)
+
 
 class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
     chunksize = 1

From 0e382c816024baffca05b0da29def95f723d27fd Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 14:09:10 -0300
Subject: [PATCH 092/243] Remove unused import

---
 tests/test_squeues.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index a20d242f4..51c0c028a 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -1,5 +1,3 @@
-import pickle
-
 from queuelib.tests import test_queue as t
 from scrapy.squeues import (
     MarshalFifoDiskQueueNonRequest as MarshalFifoDiskQueue,

From d71804ef29a00fb526ac496930356a47006c639d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 15:23:36 -0300
Subject: [PATCH 093/243] Flake8: Remove E122

---
 pytest.ini              |  4 ++--
 tests/test_webclient.py | 40 ++++++++++++++++++++--------------------
 2 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 1a73b41be..f8c4ce19a 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -208,7 +208,7 @@ flake8-ignore =
     tests/test_link.py E501
     tests/test_linkextractors.py E501 E128 E124
     tests/test_loader.py E501 E741 E128 E117
-    tests/test_logformatter.py E128 E501 E122
+    tests/test_logformatter.py E128 E501
     tests/test_mail.py E128 E501
     tests/test_middleware.py E501 E128
     tests/test_pipeline_crawl.py E501 E128 E126
@@ -244,7 +244,7 @@ flake8-ignore =
     tests/test_utils_signal.py E741 F841
     tests/test_utils_sitemap.py E128 E501 E124
     tests/test_utils_url.py E501 E501 E126
-    tests/test_webclient.py E501 E128 E122 E402 E126
+    tests/test_webclient.py E501 E128 E402 E126
     tests/test_cmdline/__init__.py E501
     tests/test_settings/__init__.py E501 E128
     tests/test_spiderloader/__init__.py E128 E501
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index de61e2125..b657c7ab6 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -53,29 +53,29 @@ class ParseUrlTestCase(unittest.TestCase):
     def testParse(self):
         lip = '127.0.0.1'
         tests = (
-    ("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
-    ("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
-    ("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
-    ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
-    ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
-    ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')),
+            ("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
+            ("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
+            ("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
+            ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
+            ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
+            ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')),
 
-    ("http://127.0.0.1", ('http', lip, lip, 80, '/')),
-    ("http://127.0.0.1/", ('http', lip, lip, 80, '/')),
-    ("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')),
-    ("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')),
-    ("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')),
-    ("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')),
-    ("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')),
-    ("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),
+            ("http://127.0.0.1", ('http', lip, lip, 80, '/')),
+            ("http://127.0.0.1/", ('http', lip, lip, 80, '/')),
+            ("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')),
+            ("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')),
+            ("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')),
+            ("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')),
+            ("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')),
+            ("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),
 
-    ("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')),
-    ("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')),
-    ("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')),
+            ("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')),
+            ("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')),
+            ("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')),
 
-    ("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')),
-    ("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')),
-    )
+            ("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')),
+            ("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')),
+        )
 
         for url, test in tests:
             test = tuple(

From cc23d1cb580795f6fde6e27f92f39ce4e3b8b558 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 15:40:37 -0300
Subject: [PATCH 094/243] Flake8: Remove E124

---
 pytest.ini                   | 14 +++----
 tests/test_dupefilters.py    |  3 +-
 tests/test_exporters.py      | 76 +++++++++++++++++++-----------------
 tests/test_linkextractors.py | 34 +++++++++-------
 tests/test_utils_sitemap.py  | 51 ++++++++++++++++--------
 5 files changed, 103 insertions(+), 75 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index f8c4ce19a..998633d54 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -88,7 +88,7 @@ flake8-ignore =
     scrapy/http/request/form.py E501
     scrapy/http/request/json_request.py E501
     scrapy/http/response/__init__.py E501 E128
-    scrapy/http/response/text.py E501 E128 E124
+    scrapy/http/response/text.py E501 E128
     # scrapy/linkextractors
     scrapy/linkextractors/__init__.py E501 E402
     scrapy/linkextractors/lxmlhtml.py E501
@@ -181,7 +181,7 @@ flake8-ignore =
     tests/test_crawl.py E501 E741
     tests/test_crawler.py F841 E501
     tests/test_dependencies.py F841 E501
-    tests/test_downloader_handlers.py E124 E128 E501 E126
+    tests/test_downloader_handlers.py E128 E501 E126
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
     tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126
@@ -195,9 +195,9 @@ flake8-ignore =
     tests/test_downloadermiddleware_retry.py E501 E128 E126
     tests/test_downloadermiddleware_robotstxt.py E501
     tests/test_downloadermiddleware_stats.py E501
-    tests/test_dupefilters.py E501 E741 E128 E124
+    tests/test_dupefilters.py E501 E741 E128
     tests/test_engine.py E501 E128
-    tests/test_exporters.py E501 E128 E124
+    tests/test_exporters.py E501 E128
     tests/test_extension_telnet.py F841
     tests/test_feedexport.py E501 F841
     tests/test_http_cookies.py E501
@@ -206,7 +206,7 @@ flake8-ignore =
     tests/test_http_response.py E501 E128
     tests/test_item.py E128 F841
     tests/test_link.py E501
-    tests/test_linkextractors.py E501 E128 E124
+    tests/test_linkextractors.py E501 E128
     tests/test_loader.py E501 E741 E128 E117
     tests/test_logformatter.py E128 E501
     tests/test_mail.py E128 E501
@@ -226,7 +226,7 @@ flake8-ignore =
     tests/test_spidermiddleware_httperror.py E128 E501 E121
     tests/test_spidermiddleware_offsite.py E501 E128 E111
     tests/test_spidermiddleware_output_chain.py E501
-    tests/test_spidermiddleware_referer.py E501 F841 E124 E501 E121
+    tests/test_spidermiddleware_referer.py E501 F841 E501 E121
     tests/test_squeues.py E501 E741
     tests/test_utils_asyncio.py E501
     tests/test_utils_conf.py E501 E128
@@ -242,7 +242,7 @@ flake8-ignore =
     tests/test_utils_request.py E501 E128
     tests/test_utils_response.py E501
     tests/test_utils_signal.py E741 F841
-    tests/test_utils_sitemap.py E128 E501 E124
+    tests/test_utils_sitemap.py E128 E501
     tests/test_utils_url.py E501 E501 E126
     tests/test_webclient.py E501 E128 E402 E126
     tests/test_cmdline/__init__.py E501
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index ea0e664be..7426107c1 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -197,8 +197,7 @@ class RFPDupeFilterTest(unittest.TestCase):
 
             r1 = Request('http://scrapytest.org/index.html')
             r2 = Request('http://scrapytest.org/index.html',
-                headers={'Referer': 'http://scrapytest.org/INDEX.html'}
-            )
+                         headers={'Referer': 'http://scrapytest.org/INDEX.html'})
 
             dupefilter.log(r1, spider)
             dupefilter.log(r2, spider)
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 160912847..0f9dafcaa 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -342,20 +342,22 @@ class XmlItemExporterTest(BaseItemExporterTest):
         i2 = dict(name=u'bar', age=i1)
         i3 = TestItem(name=u'buz', age=i2)
 
-        self.assertExportResult(i3,
-            b'<?xml version="1.0" encoding="utf-8"?>\n'
-            b'<items>'
-                b'<item>'
-                    b'<age>'
-                        b'<age>'
-                            b'<age>22</age>'
-                            b'<name>foo\xc2\xa3hoo</name>'
-                        b'</age>'
-                        b'<name>bar</name>'
-                    b'</age>'
-                    b'<name>buz</name>'
-                b'</item>'
-            b'</items>'
+        self.assertExportResult(
+            i3,
+            b"""<?xml version="1.0" encoding="utf-8"?>\n
+                <items>
+                    <item>
+                        <age>
+                            <age>
+                                <age>22</age>
+                                <name>foo\xc2\xa3hoo</name>
+                            </age>
+                            <name>bar</name>
+                        </age>
+                        <name>buz</name>
+                    </item>
+                </items>
+            """
         )
 
     def test_nested_list_item(self):
@@ -363,31 +365,35 @@ class XmlItemExporterTest(BaseItemExporterTest):
         i2 = dict(name=u'bar', v2={"egg": ["spam"]})
         i3 = TestItem(name=u'buz', age=[i1, i2])
 
-        self.assertExportResult(i3,
-            b'<?xml version="1.0" encoding="utf-8"?>\n'
-            b'<items>'
-                b'<item>'
-                    b'<age>'
-                        b'<value><name>foo</name></value>'
-                        b'<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>'
-                    b'</age>'
-                    b'<name>buz</name>'
-                b'</item>'
-            b'</items>'
+        self.assertExportResult(
+            i3,
+            b"""<?xml version="1.0" encoding="utf-8"?>\n
+                <items>
+                    <item>
+                        <age>
+                            <value><name>foo</name></value>
+                            <value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>
+                        </age>
+                        <name>buz</name>
+                    </item>
+                </items>
+            """
         )
 
     def test_nonstring_types_item(self):
         item = self._get_nonstring_types_item()
-        self.assertExportResult(item,
-            b'<?xml version="1.0" encoding="utf-8"?>\n'
-            b'<items>'
-               b'<item>'
-                   b'<float>3.14</float>'
-                   b'<boolean>False</boolean>'
-                   b'<number>22</number>'
-                   b'<time>2015-01-01 01:01:01</time>'
-               b'</item>'
-            b'</items>'
+        self.assertExportResult(
+            item,
+            b"""<?xml version="1.0" encoding="utf-8"?>\n
+                <items>
+                   <item>
+                       <float>3.14</float>
+                       <boolean>False</boolean>
+                       <number>22</number>
+                       <time>2015-01-01 01:01:01</time>
+                   </item>
+                </items>
+            """
         )
 
 
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 53968e60e..68e8514ba 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -413,24 +413,30 @@ class Base:
             response = HtmlResponse("http://example.com/index.xhtml", body=xhtml)
 
             lx = self.extractor_cls()
-            self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
-                              Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
-                              Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
-                              Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
-                              Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
-                            )
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
+                    Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
+                    Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
+                    Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
+                    Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+                ]
+            )
 
             response = XmlResponse("http://example.com/index.xhtml", body=xhtml)
 
             lx = self.extractor_cls()
-            self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
-                              Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
-                              Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
-                              Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
-                              Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
-                            )
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
+                    Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
+                    Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
+                    Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
+                    Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+                ]
+            )
 
         def test_link_wrong_href(self):
             html = b"""
diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py
index db323ab31..08b215434 100644
--- a/tests/test_utils_sitemap.py
+++ b/tests/test_utils_sitemap.py
@@ -58,10 +58,13 @@ class SitemapTest(unittest.TestCase):
   </url>
 </urlset>
 """)
-        self.assertEqual(list(s),
-            [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
-             {'loc': 'http://www.example.com/2', 'lastmod': ''},
-            ])
+        self.assertEqual(
+            list(s),
+            [
+                {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
+                {'loc': 'http://www.example.com/2', 'lastmod': ''},
+            ]
+        )
 
     def test_sitemap_wrong_ns(self):
         """We have seen sitemaps with wrongs ns. Presumably, Google still works
@@ -80,10 +83,13 @@ class SitemapTest(unittest.TestCase):
   </url>
 </urlset>
 """)
-        self.assertEqual(list(s),
-            [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
-             {'loc': 'http://www.example.com/2', 'lastmod': ''},
-            ])
+        self.assertEqual(
+            list(s),
+            [
+                {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
+                {'loc': 'http://www.example.com/2', 'lastmod': ''},
+            ]
+        )
 
     def test_sitemap_wrong_ns2(self):
         """We have seen sitemaps with wrongs ns. Presumably, Google still works
@@ -103,10 +109,13 @@ class SitemapTest(unittest.TestCase):
 </urlset>
 """)
         assert s.type == 'urlset'
-        self.assertEqual(list(s),
-            [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
-             {'loc': 'http://www.example.com/2', 'lastmod': ''},
-            ])
+        self.assertEqual(
+            list(s),
+            [
+                {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
+                {'loc': 'http://www.example.com/2', 'lastmod': ''},
+            ]
+        )
 
     def test_sitemap_urls_from_robots(self):
         robots = """User-agent: *
@@ -195,11 +204,19 @@ Disallow: /forum/active/
         </url>
     </urlset>""")
 
-        self.assertEqual(list(s), [
-            {'loc': 'http://www.example.com/english/',
-             'alternate': ['http://www.example.com/deutsch/', 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/english/']
-            }
-        ])
+        self.assertEqual(
+            list(s),
+            [
+                {
+                    'loc': 'http://www.example.com/english/',
+                    'alternate': [
+                        'http://www.example.com/deutsch/',
+                        'http://www.example.com/schweiz-deutsch/',
+                        'http://www.example.com/english/',
+                    ],
+                }
+            ]
+        )
 
     def test_xml_entity_expansion(self):
         s = Sitemap(b"""<?xml version="1.0" encoding="utf-8"?>

From 4c12a234ae65d49678a9840708ff5e7b9d6dcecc Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 16:10:21 -0300
Subject: [PATCH 095/243] Flake8: Remove E126

---
 pytest.ini                                 | 34 +++++++++++-----------
 scrapy/core/downloader/contextfactory.py   | 10 +++----
 scrapy/core/downloader/handlers/s3.py      | 13 +++++----
 scrapy/core/downloader/webclient.py        |  4 +--
 scrapy/downloadermiddlewares/retry.py      | 12 ++++++--
 scrapy/spiderloader.py                     | 25 +++++++++-------
 tests/test_downloadermiddleware_cookies.py | 10 ++++---
 tests/test_downloadermiddleware_retry.py   | 12 ++++++--
 tests/test_http_request.py                 | 32 +++++++++++---------
 tests/test_pipeline_crawl.py               |  4 +--
 tests/test_webclient.py                    | 13 +++++----
 11 files changed, 96 insertions(+), 73 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 998633d54..1570a3a75 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -50,26 +50,26 @@ flake8-ignore =
     scrapy/core/engine.py E501 E128
     scrapy/core/scheduler.py E501
     scrapy/core/scraper.py E501 E128
-    scrapy/core/spidermw.py E501 E126
+    scrapy/core/spidermw.py E501
     scrapy/core/downloader/__init__.py E501
-    scrapy/core/downloader/contextfactory.py E501 E128 E126
+    scrapy/core/downloader/contextfactory.py E501 E128
     scrapy/core/downloader/middleware.py E501
     scrapy/core/downloader/tls.py E501
-    scrapy/core/downloader/webclient.py E501 E128 E126
+    scrapy/core/downloader/webclient.py E501 E128
     scrapy/core/downloader/handlers/__init__.py E501
     scrapy/core/downloader/handlers/ftp.py E501 E128
     scrapy/core/downloader/handlers/http10.py E501
     scrapy/core/downloader/handlers/http11.py E501
-    scrapy/core/downloader/handlers/s3.py E501 E128 E126
+    scrapy/core/downloader/handlers/s3.py E501 E128
     # scrapy/downloadermiddlewares
     scrapy/downloadermiddlewares/ajaxcrawl.py E501
     scrapy/downloadermiddlewares/decompression.py E501
     scrapy/downloadermiddlewares/defaultheaders.py E501
-    scrapy/downloadermiddlewares/httpcache.py E501 E126
+    scrapy/downloadermiddlewares/httpcache.py E501
     scrapy/downloadermiddlewares/httpcompression.py E501 E128
     scrapy/downloadermiddlewares/httpproxy.py E501
     scrapy/downloadermiddlewares/redirect.py E501
-    scrapy/downloadermiddlewares/retry.py E501 E126
+    scrapy/downloadermiddlewares/retry.py E501
     scrapy/downloadermiddlewares/robotstxt.py E501
     scrapy/downloadermiddlewares/stats.py E501
     # scrapy/extensions
@@ -164,12 +164,12 @@ flake8-ignore =
     scrapy/robotstxt.py E501
     scrapy/shell.py E501
     scrapy/signalmanager.py E501
-    scrapy/spiderloader.py F841 E501 E126
+    scrapy/spiderloader.py F841 E501
     scrapy/squeues.py E128
     scrapy/statscollectors.py E501
     # tests
     tests/__init__.py E402 E501
-    tests/mockserver.py E501 E126
+    tests/mockserver.py E501
     tests/pipelines.py F841
     tests/spiders.py E501
     tests/test_closespider.py E501
@@ -181,18 +181,18 @@ flake8-ignore =
     tests/test_crawl.py E501 E741
     tests/test_crawler.py F841 E501
     tests/test_dependencies.py F841 E501
-    tests/test_downloader_handlers.py E128 E501 E126
+    tests/test_downloader_handlers.py E128 E501
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
-    tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126
+    tests/test_downloadermiddleware_cookies.py E741 E501 E128
     tests/test_downloadermiddleware_defaultheaders.py E501
     tests/test_downloadermiddleware_downloadtimeout.py E501
     tests/test_downloadermiddleware_httpcache.py E501
-    tests/test_downloadermiddleware_httpcompression.py E501 E126
+    tests/test_downloadermiddleware_httpcompression.py E501
     tests/test_downloadermiddleware_decompression.py E501
     tests/test_downloadermiddleware_httpproxy.py E501 E128
     tests/test_downloadermiddleware_redirect.py E501 E128
-    tests/test_downloadermiddleware_retry.py E501 E128 E126
+    tests/test_downloadermiddleware_retry.py E501 E128
     tests/test_downloadermiddleware_robotstxt.py E501
     tests/test_downloadermiddleware_stats.py E501
     tests/test_dupefilters.py E501 E741 E128
@@ -202,7 +202,7 @@ flake8-ignore =
     tests/test_feedexport.py E501 F841
     tests/test_http_cookies.py E501
     tests/test_http_headers.py E501
-    tests/test_http_request.py E402 E501 E128 E128 E126
+    tests/test_http_request.py E402 E501 E128 E128
     tests/test_http_response.py E501 E128
     tests/test_item.py E128 F841
     tests/test_link.py E501
@@ -211,7 +211,7 @@ flake8-ignore =
     tests/test_logformatter.py E128 E501
     tests/test_mail.py E128 E501
     tests/test_middleware.py E501 E128
-    tests/test_pipeline_crawl.py E501 E128 E126
+    tests/test_pipeline_crawl.py E501 E128
     tests/test_pipeline_files.py E501
     tests/test_pipeline_images.py F841 E501
     tests/test_pipeline_media.py E501 E741 E128
@@ -219,7 +219,7 @@ flake8-ignore =
     tests/test_request_cb_kwargs.py E501
     tests/test_responsetypes.py E501
     tests/test_robotstxt_interface.py E501 E501
-    tests/test_scheduler.py E501 E126
+    tests/test_scheduler.py E501
     tests/test_selector.py E501
     tests/test_spider.py E501
     tests/test_spidermiddleware.py E501
@@ -243,8 +243,8 @@ flake8-ignore =
     tests/test_utils_response.py E501
     tests/test_utils_signal.py E741 F841
     tests/test_utils_sitemap.py E128 E501
-    tests/test_utils_url.py E501 E501 E126
-    tests/test_webclient.py E501 E128 E402 E126
+    tests/test_utils_url.py E501 E501
+    tests/test_webclient.py E501 E128 E402
     tests/test_cmdline/__init__.py E501
     tests/test_settings/__init__.py E501 E128
     tests/test_spiderloader/__init__.py E128 E501
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 6e023ebcc..ab73e12c8 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -86,8 +86,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
         #
         # This means that a website like https://www.cacert.org will be rejected
         # by default, since CAcert.org CA certificate is seldom shipped.
-        return optionsForClientTLS(hostname.decode("ascii"),
-                                   trustRoot=platformTrust(),
-                                   extraCertificateOptions={
-                                        'method': self._ssl_method,
-                                   })
+        return optionsForClientTLS(
+            hostname=hostname.decode("ascii"),
+            trustRoot=platformTrust(),
+            extraCertificateOptions={'method': self._ssl_method},
+        )
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 40a1fa48e..8f63ad974 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -100,11 +100,12 @@ class S3DownloadHandler:
                 url=url, headers=awsrequest.headers.items())
         else:
             signed_headers = self.conn.make_request(
-                    method=request.method,
-                    bucket=bucket,
-                    key=unquote(p.path),
-                    query_args=unquote(p.query),
-                    headers=request.headers,
-                    data=request.body)
+                method=request.method,
+                bucket=bucket,
+                key=unquote(p.path),
+                query_args=unquote(p.query),
+                headers=request.headers,
+                data=request.body,
+            )
             request = request.replace(url=url, headers=signed_headers)
         return self._download_http(request, spider)
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index a90a77b2b..355045d74 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -88,8 +88,8 @@ class ScrapyHTTPPageGetter(HTTPClient):
             self.transport.stopProducing()
 
         self.factory.noPage(
-                defer.TimeoutError("Getting %s took longer than %s seconds." %
-                                   (self.factory.url, self.factory.timeout)))
+            defer.TimeoutError("Getting %s took longer than %s seconds."
+                               % (self.factory.url, self.factory.timeout)))
 
 
 class ScrapyHTTPClientFactory(HTTPClientFactory):
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index bbf5fca05..6d11af5b2 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -12,9 +12,15 @@ once the spider has finished crawling all regular (non failed) pages.
 import logging
 
 from twisted.internet import defer
-from twisted.internet.error import TimeoutError, DNSLookupError, \
-        ConnectionRefusedError, ConnectionDone, ConnectError, \
-        ConnectionLost, TCPTimedOutError
+from twisted.internet.error import (
+    ConnectError,
+    ConnectionDone,
+    ConnectionLost,
+    ConnectionRefusedError,
+    DNSLookupError,
+    TCPTimedOutError,
+    TimeoutError,
+)
 from twisted.web.client import ResponseFailed
 
 from scrapy.exceptions import NotConfigured
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 3be5aaec5..8dc89c2e9 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -24,15 +24,17 @@ class SpiderLoader:
         self._load_all_spiders()
 
     def _check_name_duplicates(self):
-        dupes = ["\n".join("  {cls} named {name!r} (in {module})".format(
-                                module=mod, cls=cls, name=name)
-                           for (mod, cls) in locations)
-                 for name, locations in self._found.items()
-                 if len(locations) > 1]
+        dupes = []
+        for name, locations in self._found.items():
+            dupes.extend([
+                "  {cls} named {name!r} (in {module})".format(module=mod, cls=cls, name=name)
+                for mod, cls in locations
+            ])
+
         if dupes:
+            dupes_string = "\n\n".join(dupes)
             msg = ("There are several spiders with the same name:\n\n"
-                   "{}\n\n  This can cause unexpected behavior.".format(
-                        "\n\n".join(dupes)))
+                   "{}\n\n  This can cause unexpected behavior.".format(dupes_string))
             warnings.warn(msg, UserWarning)
 
     def _load_spiders(self, module):
@@ -45,11 +47,12 @@ class SpiderLoader:
             try:
                 for module in walk_modules(name):
                     self._load_spiders(module)
-            except ImportError as e:
+            except ImportError:
                 if self.warn_only:
-                    msg = ("\n{tb}Could not load spiders from module '{modname}'. "
-                           "See above traceback for details.".format(
-                                modname=name, tb=traceback.format_exc()))
+                    msg = (
+                        "\n{tb}Could not load spiders from module '{modname}'. "
+                        "See above traceback for details.".format(modname=name, tb=traceback.format_exc())
+                    )
                     warnings.warn(msg, RuntimeWarning)
                 else:
                     raise
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index b686a14d6..f86c50f50 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -139,10 +139,12 @@ class CookiesMiddlewareTest(TestCase):
 
     def test_complex_cookies(self):
         # merge some cookies into jar
-        cookies = [{'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'},
-                {'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'},
-                {'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'},
-                {'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'}]
+        cookies = [
+            {'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'},
+            {'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'},
+            {'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'},
+            {'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'},
+        ]
 
         req = Request('http://scrapytest.org/', cookies=cookies)
         self.mw.process_request(req, self.spider)
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 9c989977e..e118750e3 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -1,8 +1,14 @@
 import unittest
 from twisted.internet import defer
-from twisted.internet.error import TimeoutError, DNSLookupError, \
-        ConnectionRefusedError, ConnectionDone, ConnectError, \
-        ConnectionLost, TCPTimedOutError
+from twisted.internet.error import (
+    ConnectError,
+    ConnectionDone,
+    ConnectionLost,
+    ConnectionRefusedError,
+    DNSLookupError,
+    TCPTimedOutError,
+    TimeoutError,
+)
 from twisted.web.client import ResponseFailed
 
 from scrapy.downloadermiddlewares.retry import RetryMiddleware
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 3b6d119a9..77da15ce6 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -502,11 +502,13 @@ class FormRequestTest(RequestTest):
 
     def test_from_response_duplicate_form_key(self):
         response = _buildresponse(
-                '<form></form>',
-                url='http://www.example.com')
-        req = self.request_class.from_response(response,
-                method='GET',
-                formdata=(('foo', 'bar'), ('foo', 'baz')))
+            '<form></form>',
+            url='http://www.example.com')
+        req = self.request_class.from_response(
+            response=response,
+            method='GET',
+            formdata=(('foo', 'bar'), ('foo', 'baz')),
+        )
         self.assertEqual(urlparse(req.url).hostname, 'www.example.com')
         self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz')
 
@@ -530,9 +532,11 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test" value="val2">
             <input type="hidden" name="test2" value="xxx">
             </form>""")
-        req = self.request_class.from_response(response,
-                formdata={'one': ['two', 'three'], 'six': 'seven'},
-                headers={"Accept-Encoding": "gzip,deflate"})
+        req = self.request_class.from_response(
+            response=response,
+            formdata={'one': ['two', 'three'], 'six': 'seven'},
+            headers={"Accept-Encoding": "gzip,deflate"},
+        )
         self.assertEqual(req.method, 'POST')
         self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded')
         self.assertEqual(req.headers['Accept-Encoding'], b'gzip,deflate')
@@ -580,9 +584,9 @@ class FormRequestTest(RequestTest):
 
     def test_from_response_override_method(self):
         response = _buildresponse(
-                '''<html><body>
-                <form action="/app"></form>
-                </body></html>''')
+            '''<html><body>
+            <form action="/app"></form>
+            </body></html>''')
         request = FormRequest.from_response(response)
         self.assertEqual(request.method, 'GET')
         request = FormRequest.from_response(response, method='POST')
@@ -590,9 +594,9 @@ class FormRequestTest(RequestTest):
 
     def test_from_response_override_url(self):
         response = _buildresponse(
-                '''<html><body>
-                <form action="/app"></form>
-                </body></html>''')
+            '''<html><body>
+            <form action="/app"></form>
+            </body></html>''')
         request = FormRequest.from_response(response)
         self.assertEqual(request.url, 'http://example.com/app')
         request = FormRequest.from_response(response, url='http://foo.bar/absolute')
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 962c33144..24c516473 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -44,9 +44,7 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider):
     name = 'redirectedmedia'
 
     def _process_url(self, url):
-        return add_or_replace_parameter(
-                    self.mockserver.url('/redirect-to'),
-                    'goto', url)
+        return add_or_replace_parameter(self.mockserver.url('/redirect-to'), 'goto', url)
 
 
 class FileDownloadCrawlTestCase(TestCase):
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index b657c7ab6..307fadb5c 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -18,6 +18,14 @@ except ImportError:
 from twisted.python.filepath import FilePath
 from twisted.protocols.policies import WrappingFactory
 from twisted.internet.defer import inlineCallbacks
+from twisted.web.test.test_webclient import (
+    ForeverTakingResource,
+    ErrorResource,
+    NoLengthResource,
+    HostHeaderResource,
+    PayloadResource,
+    BrokenDownloadResource,
+)
 
 from scrapy.core.downloader import webclient as client
 from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
@@ -202,11 +210,6 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             Headers({'Hello': ['World'], 'Foo': ['Bar']}))
 
 
-from twisted.web.test.test_webclient import ForeverTakingResource, \
-        ErrorResource, NoLengthResource, HostHeaderResource, \
-        PayloadResource, BrokenDownloadResource
-
-
 class EncodingResource(resource.Resource):
     out_encoding = 'cp1251'
 

From 88efc988473fb0db8ca8fb512b2aab834c5aa7af Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 16:42:47 -0300
Subject: [PATCH 096/243] Flake8: remove E129

---
 pytest.ini                    | 4 ++--
 tests/test_utils_iterators.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 1a73b41be..ff0bb010f 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -110,7 +110,7 @@ flake8-ignore =
     # scrapy/spidermiddlewares
     scrapy/spidermiddlewares/httperror.py E501
     scrapy/spidermiddlewares/offsite.py E501
-    scrapy/spidermiddlewares/referer.py E501 E129
+    scrapy/spidermiddlewares/referer.py E501
     scrapy/spidermiddlewares/urllength.py E501
     # scrapy/spiders
     scrapy/spiders/__init__.py E501 E402
@@ -235,7 +235,7 @@ flake8-ignore =
     tests/test_utils_defer.py E501 F841
     tests/test_utils_deprecate.py F841 E501
     tests/test_utils_http.py E501 E128
-    tests/test_utils_iterators.py E501 E128 E129
+    tests/test_utils_iterators.py E501 E128
     tests/test_utils_log.py E741
     tests/test_utils_python.py E501
     tests/test_utils_reqser.py E501 E128
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index a85087619..46aaaecbc 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -93,8 +93,8 @@ class XmliterTestCase(unittest.TestCase):
             # with bytes
             XmlResponse(url="http://example.com", body=body.encode('utf-8')),
             # Unicode body needs encoding information
-            XmlResponse(url="http://example.com", body=body, encoding='utf-8')):
-
+            XmlResponse(url="http://example.com", body=body, encoding='utf-8'),
+        ):
             attrs = []
             for x in self.xmliter(r, u'þingflokkur'):
                 attrs.append((x.attrib['id'],

From 2af34873cc385e47cc962903ff3d6a13c9847615 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 17:10:10 -0300
Subject: [PATCH 097/243] Flake8: Remove E128 (commands module)

---
 pytest.ini                      | 16 ++++++++--------
 scrapy/commands/__init__.py     | 12 ++++++------
 scrapy/commands/fetch.py        |  4 ++--
 scrapy/commands/genspider.py    | 10 +++++-----
 scrapy/commands/parse.py        | 24 ++++++++++++------------
 scrapy/commands/settings.py     | 10 +++++-----
 scrapy/commands/shell.py        |  8 ++++----
 scrapy/commands/startproject.py |  6 ++----
 scrapy/commands/version.py      |  2 +-
 9 files changed, 45 insertions(+), 47 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 1570a3a75..d96ac9684 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -31,18 +31,18 @@ flake8-ignore =
     extras/qps-bench-server.py E501
     extras/qpsclient.py E501 E501
     # scrapy/commands
-    scrapy/commands/__init__.py E128 E501
+    scrapy/commands/__init__.py E501
     scrapy/commands/check.py E501
     scrapy/commands/crawl.py E501
     scrapy/commands/edit.py E501
-    scrapy/commands/fetch.py E501 E128
-    scrapy/commands/genspider.py E128 E501
-    scrapy/commands/parse.py E128 E501
+    scrapy/commands/fetch.py E501
+    scrapy/commands/genspider.py E501
+    scrapy/commands/parse.py E501
     scrapy/commands/runspider.py E501
-    scrapy/commands/settings.py E128
-    scrapy/commands/shell.py E128 E501
-    scrapy/commands/startproject.py E501 E128
-    scrapy/commands/version.py E501 E128
+    scrapy/commands/settings.py E501
+    scrapy/commands/shell.py E501
+    scrapy/commands/startproject.py E501
+    scrapy/commands/version.py E501
     # scrapy/contracts
     scrapy/contracts/__init__.py E501
     scrapy/contracts/default.py E128
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 9f8e6986a..97f818cb2 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -59,17 +59,17 @@ class ScrapyCommand:
         """
         group = OptionGroup(parser, "Global Options")
         group.add_option("--logfile", metavar="FILE",
-            help="log file. if omitted stderr will be used")
+                         help="log file. if omitted stderr will be used")
         group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
-            help="log level (default: %s)" % self.settings['LOG_LEVEL'])
+                         help="log level (default: %s)" % self.settings['LOG_LEVEL'])
         group.add_option("--nolog", action="store_true",
-            help="disable logging completely")
+                         help="disable logging completely")
         group.add_option("--profile", metavar="FILE", default=None,
-            help="write python cProfile stats to FILE")
+                         help="write python cProfile stats to FILE")
         group.add_option("--pidfile", metavar="FILE",
-            help="write process ID to FILE")
+                         help="write process ID to FILE")
         group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
-            help="set/override setting (may be repeated)")
+                         help="set/override setting (may be repeated)")
         group.add_option("--pdb", action="store_true", help="enable pdb on failure")
 
         parser.add_option_group(group)
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 506d1f1b7..063195f50 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -27,8 +27,8 @@ class Command(ScrapyCommand):
         parser.add_option("--spider", dest="spider", help="use this spider")
         parser.add_option("--headers", dest="headers", action="store_true",
                           help="print response HTTP headers instead of body")
-        parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
-                          default=False, help="do not handle HTTP 3xx status codes and print response as-is")
+        parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
+                          help="do not handle HTTP 3xx status codes and print response as-is")
 
     def _print_headers(self, headers, prefix):
         for key, values in headers.items():
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 2e837abed..abf3b7a5c 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -36,15 +36,15 @@ class Command(ScrapyCommand):
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
         parser.add_option("-l", "--list", dest="list", action="store_true",
-            help="List available templates")
+                          help="List available templates")
         parser.add_option("-e", "--edit", dest="edit", action="store_true",
-            help="Edit spider after creating it")
+                          help="Edit spider after creating it")
         parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",
-            help="Dump template to standard output")
+                          help="Dump template to standard output")
         parser.add_option("-t", "--template", dest="template", default="basic",
-            help="Uses a custom template.")
+                          help="Uses a custom template.")
         parser.add_option("--force", dest="force", action="store_true",
-            help="If the spider already exists, overwrite it with the template")
+                          help="If the spider already exists, overwrite it with the template")
 
     def run(self, args, opts):
         if opts.list:
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 1cefed106..d9ab2126a 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -33,29 +33,29 @@ class Command(ScrapyCommand):
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
         parser.add_option("--spider", dest="spider", default=None,
-            help="use this spider without looking for one")
+                          help="use this spider without looking for one")
         parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
-            help="set spider argument (may be repeated)")
+                          help="set spider argument (may be repeated)")
         parser.add_option("--pipelines", action="store_true",
-            help="process items through pipelines")
+                          help="process items through pipelines")
         parser.add_option("--nolinks", dest="nolinks", action="store_true",
-            help="don't show links to follow (extracted requests)")
+                          help="don't show links to follow (extracted requests)")
         parser.add_option("--noitems", dest="noitems", action="store_true",
-            help="don't show scraped items")
+                          help="don't show scraped items")
         parser.add_option("--nocolour", dest="nocolour", action="store_true",
-            help="avoid using pygments to colorize the output")
+                          help="avoid using pygments to colorize the output")
         parser.add_option("-r", "--rules", dest="rules", action="store_true",
-            help="use CrawlSpider rules to discover the callback")
+                          help="use CrawlSpider rules to discover the callback")
         parser.add_option("-c", "--callback", dest="callback",
-            help="use this callback for parsing, instead looking for a callback")
+                          help="use this callback for parsing, instead looking for a callback")
         parser.add_option("-m", "--meta", dest="meta",
-            help="inject extra meta into the Request, it must be a valid raw json string")
+                          help="inject extra meta into the Request, it must be a valid raw json string")
         parser.add_option("--cbkwargs", dest="cbkwargs",
-            help="inject extra callback kwargs into the Request, it must be a valid raw json string")
+                          help="inject extra callback kwargs into the Request, it must be a valid raw json string")
         parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
-            help="maximum depth for parsing requests [default: %default]")
+                          help="maximum depth for parsing requests [default: %default]")
         parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
-            help="print each depth level one by one")
+                          help="print each depth level one by one")
 
     @property
     def max_level(self):
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index 603bafb9f..8d49e440f 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -19,15 +19,15 @@ class Command(ScrapyCommand):
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
         parser.add_option("--get", dest="get", metavar="SETTING",
-            help="print raw setting value")
+                          help="print raw setting value")
         parser.add_option("--getbool", dest="getbool", metavar="SETTING",
-            help="print setting value, interpreted as a boolean")
+                          help="print setting value, interpreted as a boolean")
         parser.add_option("--getint", dest="getint", metavar="SETTING",
-            help="print setting value, interpreted as an integer")
+                          help="print setting value, interpreted as an integer")
         parser.add_option("--getfloat", dest="getfloat", metavar="SETTING",
-            help="print setting value, interpreted as a float")
+                          help="print setting value, interpreted as a float")
         parser.add_option("--getlist", dest="getlist", metavar="SETTING",
-            help="print setting value, interpreted as a list")
+                          help="print setting value, interpreted as a list")
 
     def run(self, args, opts):
         settings = self.crawler_process.settings
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 5946f21e8..d1944df3d 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -34,11 +34,11 @@ class Command(ScrapyCommand):
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
         parser.add_option("-c", dest="code",
-            help="evaluate the code in the shell, print the result and exit")
+                          help="evaluate the code in the shell, print the result and exit")
         parser.add_option("--spider", dest="spider",
-            help="use this spider")
-        parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
-            default=False, help="do not handle HTTP 3xx status codes and print response as-is")
+                          help="use this spider")
+        parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
+                          help="do not handle HTTP 3xx status codes and print response as-is")
 
     def update_vars(self, vars):
         """You can use this function to update the Scrapy objects that will be
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index b123e5c84..ebe3a9c2c 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -102,10 +102,8 @@ class Command(ScrapyCommand):
         move(join(project_dir, 'module'), join(project_dir, project_name))
         for paths in TEMPLATES_TO_RENDER:
             path = join(*paths)
-            tplfile = join(project_dir,
-                string.Template(path).substitute(project_name=project_name))
-            render_templatefile(tplfile, project_name=project_name,
-                ProjectName=string_camelcase(project_name))
+            tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name))
+            render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name))
         print("New Scrapy project '%s', using template directory '%s', "
               "created in:" % (project_name, self.templates_dir))
         print("    %s\n" % abspath(project_dir))
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 1516c5997..d0ea72a67 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -17,7 +17,7 @@ class Command(ScrapyCommand):
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
         parser.add_option("--verbose", "-v", dest="verbose", action="store_true",
-            help="also display twisted/python/platform info (useful for bug reports)")
+                          help="also display twisted/python/platform info (useful for bug reports)")
 
     def run(self, args, opts):
         if opts.verbose:

From 7383b2b428b08e84d6e3e40e6229c892e5802f7a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 17:27:44 -0300
Subject: [PATCH 098/243] Flake8: Remove E128 (core module)

---
 pytest.ini                               | 12 ++++++------
 scrapy/core/downloader/contextfactory.py | 11 ++++++-----
 scrapy/core/downloader/handlers/ftp.py   |  7 +++----
 scrapy/core/engine.py                    | 13 +++++--------
 4 files changed, 20 insertions(+), 23 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index d96ac9684..7c6ee07d6 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -47,20 +47,20 @@ flake8-ignore =
     scrapy/contracts/__init__.py E501
     scrapy/contracts/default.py E128
     # scrapy/core
-    scrapy/core/engine.py E501 E128
+    scrapy/core/engine.py E501
     scrapy/core/scheduler.py E501
-    scrapy/core/scraper.py E501 E128
+    scrapy/core/scraper.py E501
     scrapy/core/spidermw.py E501
     scrapy/core/downloader/__init__.py E501
-    scrapy/core/downloader/contextfactory.py E501 E128
+    scrapy/core/downloader/contextfactory.py E501
     scrapy/core/downloader/middleware.py E501
     scrapy/core/downloader/tls.py E501
-    scrapy/core/downloader/webclient.py E501 E128
+    scrapy/core/downloader/webclient.py E501
     scrapy/core/downloader/handlers/__init__.py E501
-    scrapy/core/downloader/handlers/ftp.py E501 E128
+    scrapy/core/downloader/handlers/ftp.py E501
     scrapy/core/downloader/handlers/http10.py E501
     scrapy/core/downloader/handlers/http11.py E501
-    scrapy/core/downloader/handlers/s3.py E501 E128
+    scrapy/core/downloader/handlers/s3.py E501
     # scrapy/downloadermiddlewares
     scrapy/downloadermiddlewares/ajaxcrawl.py E501
     scrapy/downloadermiddlewares/decompression.py E501
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index ab73e12c8..452242d47 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -46,11 +46,12 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
         #
         # * getattr() for `_ssl_method` attribute for context factories
         #   not calling super(..., self).__init__
-        return CertificateOptions(verify=False,
-                    method=getattr(self, 'method',
-                                   getattr(self, '_ssl_method', None)),
-                    fixBrokenPeers=True,
-                    acceptableCiphers=self.tls_ciphers)
+        return CertificateOptions(
+            verify=False,
+            method=getattr(self, 'method', getattr(self, '_ssl_method', None)),
+            fixBrokenPeers=True,
+            acceptableCiphers=self.tls_ciphers,
+        )
 
     # kept for old-style HTTP/1.0 downloader context twisted calls,
     # e.g. connectSSL()
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 94b55c347..3ef129587 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -86,10 +86,9 @@ class FTPDownloadHandler:
         password = request.meta.get("ftp_password", self.default_password)
         passive_mode = 1 if bool(request.meta.get("ftp_passive",
                                                   self.passive_mode)) else 0
-        creator = ClientCreator(reactor, FTPClient, user, password,
-            passive=passive_mode)
-        return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient,
-                                request, unquote(parsed_url.path))
+        creator = ClientCreator(reactor, FTPClient, user, password, passive=passive_mode)
+        dfd = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21)
+        return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path))
 
     def gotClient(self, client, request, filepath):
         self.client = client
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 324d21716..de0da4b70 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -217,11 +217,9 @@ class ExecutionEngine:
         self.slot.nextcall.schedule()
 
     def schedule(self, request, spider):
-        self.signals.send_catch_log(signal=signals.request_scheduled,
-                request=request, spider=spider)
+        self.signals.send_catch_log(signals.request_scheduled, request=request, spider=spider)
         if not self.slot.scheduler.enqueue_request(request):
-            self.signals.send_catch_log(signal=signals.request_dropped,
-                                        request=request, spider=spider)
+            self.signals.send_catch_log(signals.request_dropped, request=request, spider=spider)
 
     def download(self, request, spider):
         d = self._download(request, spider)
@@ -247,8 +245,8 @@ class ExecutionEngine:
                 logkws = self.logformatter.crawled(request, response, spider)
                 if logkws is not None:
                     logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
-                self.signals.send_catch_log(signal=signals.response_received,
-                    response=response, request=request, spider=spider)
+                self.signals.send_catch_log(signals.response_received,
+                                            response=response, request=request, spider=spider)
             return response
 
         def _on_complete(_):
@@ -286,8 +284,7 @@ class ExecutionEngine:
         next loop and this function is guaranteed to be called (at least) once
         again for this spider.
         """
-        res = self.signals.send_catch_log(signal=signals.spider_idle,
-            spider=spider, dont_log=DontCloseSpider)
+        res = self.signals.send_catch_log(signals.spider_idle, spider=spider, dont_log=DontCloseSpider)
         if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res):
             return
 

From a72f5aadcf0b7ee8ef77f43602aecf34ffea7fd6 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 17:38:39 -0300
Subject: [PATCH 099/243] Flake8: Remove E128 (utils module)

---
 pytest.ini               | 10 +++++-----
 scrapy/utils/defer.py    |  7 +++++--
 scrapy/utils/log.py      | 10 ++++++----
 scrapy/utils/response.py |  7 +++----
 scrapy/utils/signal.py   |  6 ++----
 scrapy/utils/url.py      |  3 +--
 6 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 7c6ee07d6..63a3051cf 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -123,13 +123,13 @@ flake8-ignore =
     scrapy/utils/conf.py E402 E501
     scrapy/utils/datatypes.py E501
     scrapy/utils/decorators.py E501
-    scrapy/utils/defer.py E501 E128
+    scrapy/utils/defer.py E501
     scrapy/utils/deprecate.py E501
     scrapy/utils/gz.py E501
     scrapy/utils/http.py F403
     scrapy/utils/httpobj.py E501
     scrapy/utils/iterators.py E501
-    scrapy/utils/log.py E128 E501
+    scrapy/utils/log.py E501
     scrapy/utils/markup.py F403
     scrapy/utils/misc.py E501
     scrapy/utils/multipart.py F403
@@ -138,13 +138,13 @@ flake8-ignore =
     scrapy/utils/reactor.py E501
     scrapy/utils/reqser.py E501
     scrapy/utils/request.py E501
-    scrapy/utils/response.py E501 E128
-    scrapy/utils/signal.py E501 E128
+    scrapy/utils/response.py E501
+    scrapy/utils/signal.py E501
     scrapy/utils/sitemap.py E501
     scrapy/utils/spider.py E501
     scrapy/utils/ssl.py E501
     scrapy/utils/test.py E501
-    scrapy/utils/url.py E501 F403 E128 F405
+    scrapy/utils/url.py E501 F403 F405
     # scrapy
     scrapy/__init__.py E402 E501
     scrapy/cmdline.py E501
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 34b8d9774..a3950db75 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -88,8 +88,11 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw):
     """Return a Deferred built by chaining the given callbacks and errbacks"""
     d = defer.Deferred()
     for cb, eb in zip(callbacks, errbacks):
-        d.addCallbacks(cb, eb, callbackArgs=a, callbackKeywords=kw,
-            errbackArgs=a, errbackKeywords=kw)
+        d.addCallbacks(
+            callback=cb, errback=eb,
+            callbackArgs=a, callbackKeywords=kw,
+            errbackArgs=a, errbackKeywords=kw,
+        )
     if isinstance(input, failure.Failure):
         d.errback(input)
     else:
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 5998dc33b..83c359bd4 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -144,10 +144,12 @@ def _get_handler(settings):
 def log_scrapy_info(settings):
     logger.info("Scrapy %(version)s started (bot: %(bot)s)",
                 {'version': scrapy.__version__, 'bot': settings['BOT_NAME']})
-    logger.info("Versions: %(versions)s",
-                {'versions': ", ".join("%s %s" % (name, version)
-                    for name, version in scrapy_components_versions()
-                    if name != "Scrapy")})
+    versions = [
+        "%s %s" % (name, version)
+        for name, version in scrapy_components_versions()
+        if name != "Scrapy"
+    ]
+    logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)})
     from twisted.internet import reactor
     logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
 
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 29fdaaf2c..edbc0db25 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -19,8 +19,7 @@ def get_base_url(response):
     """Return the base url of the given response, joined with the response url"""
     if response not in _baseurl_cache:
         text = response.text[0:4096]
-        _baseurl_cache[response] = html.get_base_url(text, response.url,
-            response.encoding)
+        _baseurl_cache[response] = html.get_base_url(text, response.url, response.encoding)
     return _baseurl_cache[response]
 
 
@@ -31,8 +30,8 @@ def get_meta_refresh(response, ignore_tags=('script', 'noscript')):
     """Parse the http-equiv refrsh parameter from the given response"""
     if response not in _metaref_cache:
         text = response.text[0:4096]
-        _metaref_cache[response] = html.get_meta_refresh(text, response.url,
-            response.encoding, ignore_tags=ignore_tags)
+        _metaref_cache[response] = html.get_meta_refresh(
+            text, response.url, response.encoding, ignore_tags=ignore_tags)
     return _metaref_cache[response]
 
 
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index 60c561da6..a311e9257 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -28,8 +28,7 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
     responses = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
         try:
-            response = robustApply(receiver, signal=signal, sender=sender,
-                *arguments, **named)
+            response = robustApply(receiver, signal=signal, sender=sender, *arguments, **named)
             if isinstance(response, Deferred):
                 logger.error("Cannot return deferreds from signal handler: %(receiver)s",
                              {'receiver': receiver}, extra={'spider': spider})
@@ -63,8 +62,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
     spider = named.get('spider', None)
     dfds = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
-        d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender,
-                *arguments, **named)
+        d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender, *arguments, **named)
         d.addErrback(logerror, receiver)
         d.addBoth(lambda result: (receiver, result))
         dfds.append(d)
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index c9abb12d5..955b63d4b 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -27,8 +27,7 @@ def url_is_from_any_domain(url, domains):
 
 def url_is_from_spider(url, spider):
     """Return True if the url belongs to the given spider"""
-    return url_is_from_any_domain(url,
-        [spider.name] + list(getattr(spider, 'allowed_domains', [])))
+    return url_is_from_any_domain(url, [spider.name] + list(getattr(spider, 'allowed_domains', [])))
 
 
 def url_has_any_extension(url, extensions):

From 4171b3f663aea5706ba6acf15f64e92be0e34f2a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 18:28:12 -0300
Subject: [PATCH 100/243] Flake8: Remove E128 (extensions module)

---
 pytest.ini                     | 6 +++---
 scrapy/extensions/httpcache.py | 5 +++--
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 63a3051cf..1af3ea0d1 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -73,10 +73,10 @@ flake8-ignore =
     scrapy/downloadermiddlewares/robotstxt.py E501
     scrapy/downloadermiddlewares/stats.py E501
     # scrapy/extensions
-    scrapy/extensions/closespider.py E501 E128
+    scrapy/extensions/closespider.py E501
     scrapy/extensions/corestats.py E501
-    scrapy/extensions/feedexport.py E128 E501
-    scrapy/extensions/httpcache.py E128 E501
+    scrapy/extensions/feedexport.py E501
+    scrapy/extensions/httpcache.py E501
     scrapy/extensions/memdebug.py E501
     scrapy/extensions/spiderstate.py E501
     scrapy/extensions/telnet.py E501
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 8546628a8..6289efec0 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -46,9 +46,10 @@ class RFC2616Policy:
     def __init__(self, settings):
         self.always_store = settings.getbool('HTTPCACHE_ALWAYS_STORE')
         self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
-        self.ignore_response_cache_controls = [to_bytes(cc) for cc in
-            settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')]
         self._cc_parsed = WeakKeyDictionary()
+        self.ignore_response_cache_controls = [
+            to_bytes(cc) for cc in settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')
+        ]
 
     def _parse_cachecontrol(self, r):
         if r not in self._cc_parsed:

From 2fbbca56fb1e3bba98bf6c4c84591121c4c1da57 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 18:34:40 -0300
Subject: [PATCH 101/243] pytest.ini: remove unnecessary E128 lines

---
 pytest.ini | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 1af3ea0d1..c0d74f7d9 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -45,7 +45,6 @@ flake8-ignore =
     scrapy/commands/version.py E501
     # scrapy/contracts
     scrapy/contracts/__init__.py E501
-    scrapy/contracts/default.py E128
     # scrapy/core
     scrapy/core/engine.py E501
     scrapy/core/scheduler.py E501
@@ -66,7 +65,7 @@ flake8-ignore =
     scrapy/downloadermiddlewares/decompression.py E501
     scrapy/downloadermiddlewares/defaultheaders.py E501
     scrapy/downloadermiddlewares/httpcache.py E501
-    scrapy/downloadermiddlewares/httpcompression.py E501 E128
+    scrapy/downloadermiddlewares/httpcompression.py E501
     scrapy/downloadermiddlewares/httpproxy.py E501
     scrapy/downloadermiddlewares/redirect.py E501
     scrapy/downloadermiddlewares/retry.py E501
@@ -87,13 +86,13 @@ flake8-ignore =
     scrapy/http/request/__init__.py E501
     scrapy/http/request/form.py E501
     scrapy/http/request/json_request.py E501
-    scrapy/http/response/__init__.py E501 E128
-    scrapy/http/response/text.py E501 E128
+    scrapy/http/response/__init__.py E501
+    scrapy/http/response/text.py E501
     # scrapy/linkextractors
     scrapy/linkextractors/__init__.py E501 E402
     scrapy/linkextractors/lxmlhtml.py E501
     # scrapy/loader
-    scrapy/loader/__init__.py E501 E128
+    scrapy/loader/__init__.py E501
     scrapy/loader/processors.py E501
     # scrapy/pipelines
     scrapy/pipelines/__init__.py E501
@@ -157,7 +156,7 @@ flake8-ignore =
     scrapy/link.py E501
     scrapy/logformatter.py E501
     scrapy/mail.py E402 E128 E501
-    scrapy/middleware.py E128 E501
+    scrapy/middleware.py E501
     scrapy/pqueues.py E501
     scrapy/resolver.py E501
     scrapy/responsetypes.py E128 E501
@@ -165,7 +164,6 @@ flake8-ignore =
     scrapy/shell.py E501
     scrapy/signalmanager.py E501
     scrapy/spiderloader.py F841 E501
-    scrapy/squeues.py E128
     scrapy/statscollectors.py E501
     # tests
     tests/__init__.py E402 E501

From 2851f641f2c105350948a34be55aa00dfed9933a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 18:35:50 -0300
Subject: [PATCH 102/243] Flake8: Remove E128 (item, mail, responsetypes)

---
 pytest.ini              |  6 +++---
 scrapy/item.py          |  6 ++----
 scrapy/mail.py          | 32 ++++++++++++++++++++++----------
 scrapy/responsetypes.py |  6 +++---
 4 files changed, 30 insertions(+), 20 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index c0d74f7d9..0a5a172b3 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -152,14 +152,14 @@ flake8-ignore =
     scrapy/exceptions.py E501
     scrapy/exporters.py E501
     scrapy/interfaces.py E501
-    scrapy/item.py E501 E128
+    scrapy/item.py E501
     scrapy/link.py E501
     scrapy/logformatter.py E501
-    scrapy/mail.py E402 E128 E501
+    scrapy/mail.py E402 E501
     scrapy/middleware.py E501
     scrapy/pqueues.py E501
     scrapy/resolver.py E501
-    scrapy/responsetypes.py E128 E501
+    scrapy/responsetypes.py E501
     scrapy/robotstxt.py E501
     scrapy/shell.py E501
     scrapy/signalmanager.py E501
diff --git a/scrapy/item.py b/scrapy/item.py
index 748368932..b75d04404 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -86,8 +86,7 @@ class DictItem(MutableMapping, BaseItem):
         if key in self.fields:
             self._values[key] = value
         else:
-            raise KeyError("%s does not support field: %s" %
-                (self.__class__.__name__, key))
+            raise KeyError("%s does not support field: %s" % (self.__class__.__name__, key))
 
     def __delitem__(self, key):
         del self._values[key]
@@ -99,8 +98,7 @@ class DictItem(MutableMapping, BaseItem):
 
     def __setattr__(self, name, value):
         if not name.startswith('_'):
-            raise AttributeError("Use item[%r] = %r to set field value" %
-                (name, value))
+            raise AttributeError("Use item[%r] = %r to set field value" % (name, value))
         super(DictItem, self).__setattr__(name, value)
 
     def __len__(self):
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 9d7896ef6..7d7a2c435 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -28,8 +28,10 @@ def _to_bytes_or_none(text):
 
 
 class MailSender:
-    def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost',
-            smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False):
+    def __init__(
+        self, smtphost='localhost', mailfrom='scrapy@localhost', smtpuser=None,
+        smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False
+    ):
         self.smtphost = smtphost
         self.smtpport = smtpport
         self.smtpuser = _to_bytes_or_none(smtpuser)
@@ -41,9 +43,15 @@ class MailSender:
 
     @classmethod
     def from_settings(cls, settings):
-        return cls(settings['MAIL_HOST'], settings['MAIL_FROM'], settings['MAIL_USER'],
-            settings['MAIL_PASS'], settings.getint('MAIL_PORT'),
-            settings.getbool('MAIL_TLS'), settings.getbool('MAIL_SSL'))
+        return cls(
+            smtphost=settings['MAIL_HOST'],
+            mailfrom=settings['MAIL_FROM'],
+            smtpuser=settings['MAIL_USER'],
+            smtppass=settings['MAIL_PASS'],
+            smtpport=settings.getint('MAIL_PORT'),
+            smtptls=settings.getbool('MAIL_TLS'),
+            smtpssl=settings.getbool('MAIL_SSL'),
+        )
 
     def send(self, to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None, _callback=None):
         from twisted.internet import reactor
@@ -89,9 +97,12 @@ class MailSender:
             return
 
         dfd = self._sendmail(rcpts, msg.as_string().encode(charset or 'utf-8'))
-        dfd.addCallbacks(self._sent_ok, self._sent_failed,
+        dfd.addCallbacks(
+            callback=self._sent_ok,
+            errback=self._sent_failed,
             callbackArgs=[to, cc, subject, len(attachs)],
-            errbackArgs=[to, cc, subject, len(attachs)])
+            errbackArgs=[to, cc, subject, len(attachs)],
+        )
         reactor.addSystemEventTrigger('before', 'shutdown', lambda: dfd)
         return dfd
 
@@ -115,9 +126,10 @@ class MailSender:
         from twisted.mail.smtp import ESMTPSenderFactory
         msg = BytesIO(msg)
         d = defer.Deferred()
-        factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom,
-            to_addrs, msg, d, heloFallback=True, requireAuthentication=False,
-            requireTransportSecurity=self.smtptls)
+        factory = ESMTPSenderFactory(
+            self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d,
+            heloFallback=True, requireAuthentication=False, requireTransportSecurity=self.smtptls,
+        )
         factory.noisy = False
 
         if self.smtpssl:
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 7c5eeac21..d207088e6 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -58,9 +58,9 @@ class ResponseTypes:
 
     def from_content_disposition(self, content_disposition):
         try:
-            filename = to_unicode(content_disposition,
-                encoding='latin-1', errors='replace').split(';')[1].split('=')[1]
-            filename = filename.strip('"\'')
+            filename = to_unicode(
+                content_disposition, encoding='latin-1', errors='replace'
+            ).split(';')[1].split('=')[1].strip('"\'')
             return self.from_filename(filename)
         except IndexError:
             return Response

From 9c6d13075a2ab06c0d670e19457f9d9f219ceeee Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 18:56:14 -0300
Subject: [PATCH 103/243] Flake8: Remove E128 (tests, part 1)

---
 pytest.ini                                   | 28 +++++++++---------
 tests/test_command_parse.py                  |  4 +--
 tests/test_command_shell.py                  |  6 ++--
 tests/test_contracts.py                      |  3 +-
 tests/test_downloader_handlers.py            |  8 ++---
 tests/test_downloadermiddleware_httpproxy.py |  7 +++--
 tests/test_downloadermiddleware_redirect.py  |  2 +-
 tests/test_downloadermiddleware_retry.py     | 14 +++++++--
 tests/test_dupefilters.py                    | 31 +++++++++++++-------
 tests/test_http_request.py                   | 18 ++++--------
 tests/test_http_response.py                  | 20 ++++++++-----
 11 files changed, 80 insertions(+), 61 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 0a5a172b3..d8eefea7e 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -172,37 +172,37 @@ flake8-ignore =
     tests/spiders.py E501
     tests/test_closespider.py E501
     tests/test_command_fetch.py E501
-    tests/test_command_parse.py E501 E128
-    tests/test_command_shell.py E501 E128
-    tests/test_commands.py E128 E501
-    tests/test_contracts.py E501 E128
+    tests/test_command_parse.py E501
+    tests/test_command_shell.py E501
+    tests/test_commands.py E501
+    tests/test_contracts.py E501
     tests/test_crawl.py E501 E741
     tests/test_crawler.py F841 E501
     tests/test_dependencies.py F841 E501
-    tests/test_downloader_handlers.py E128 E501
+    tests/test_downloader_handlers.py E501
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
-    tests/test_downloadermiddleware_cookies.py E741 E501 E128
+    tests/test_downloadermiddleware_cookies.py E741 E501
     tests/test_downloadermiddleware_defaultheaders.py E501
     tests/test_downloadermiddleware_downloadtimeout.py E501
     tests/test_downloadermiddleware_httpcache.py E501
     tests/test_downloadermiddleware_httpcompression.py E501
     tests/test_downloadermiddleware_decompression.py E501
-    tests/test_downloadermiddleware_httpproxy.py E501 E128
-    tests/test_downloadermiddleware_redirect.py E501 E128
-    tests/test_downloadermiddleware_retry.py E501 E128
+    tests/test_downloadermiddleware_httpproxy.py E501
+    tests/test_downloadermiddleware_redirect.py E501
+    tests/test_downloadermiddleware_retry.py E501
     tests/test_downloadermiddleware_robotstxt.py E501
     tests/test_downloadermiddleware_stats.py E501
-    tests/test_dupefilters.py E501 E741 E128
-    tests/test_engine.py E501 E128
-    tests/test_exporters.py E501 E128
+    tests/test_dupefilters.py E501 E741
+    tests/test_engine.py E501
+    tests/test_exporters.py E501
     tests/test_extension_telnet.py F841
     tests/test_feedexport.py E501 F841
     tests/test_http_cookies.py E501
     tests/test_http_headers.py E501
-    tests/test_http_request.py E402 E501 E128 E128
-    tests/test_http_response.py E501 E128
     tests/test_item.py E128 F841
+    tests/test_http_request.py E402 E501
+    tests/test_http_response.py E501
     tests/test_link.py E501
     tests/test_linkextractors.py E501 E128
     tests/test_loader.py E501 E741 E128 E117
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 85a24d0bc..a09dcf072 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -142,8 +142,8 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
     @defer.inlineCallbacks
     def test_request_without_meta(self):
         _, _, stderr = yield self.execute(['--spider', self.spider_name,
-                                          '-c', 'parse_request_without_meta',
-                                          '--nolinks',
+                                           '-c', 'parse_request_without_meta',
+                                           '--nolinks',
                                            self.url('/html')])
         self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index d664b6ade..93fda2648 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -101,15 +101,13 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
     @defer.inlineCallbacks
     def test_local_nofile(self):
         filepath = 'file:///tests/sample_data/test_site/nothinghere.html'
-        errcode, out, err = yield self.execute([filepath, '-c', 'item'],
-                                       check_code=False)
+        errcode, out, err = yield self.execute([filepath, '-c', 'item'], check_code=False)
         self.assertEqual(errcode, 1, out or err)
         self.assertIn(b'No such file or directory', err)
 
     @defer.inlineCallbacks
     def test_dns_failures(self):
         url = 'www.somedomainthatdoesntexi.st'
-        errcode, out, err = yield self.execute([url, '-c', 'item'],
-                                       check_code=False)
+        errcode, out, err = yield self.execute([url, '-c', 'item'], check_code=False)
         self.assertEqual(errcode, 1, out or err)
         self.assertIn(b'DNS lookup failed', err)
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index d1ce80f9d..99120b128 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -232,7 +232,8 @@ class ContractsManagerTest(unittest.TestCase):
         # extract contracts correctly
         contracts = self.conman.extract_contracts(spider.returns_request)
         self.assertEqual(len(contracts), 2)
-        self.assertEqual(frozenset(type(x) for x in contracts),
+        self.assertEqual(
+            frozenset(type(x) for x in contracts),
             frozenset([UrlContract, ReturnsContract]))
 
         # returns request for valid method
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index f93bce8ef..1a05b679a 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -853,8 +853,7 @@ class S3TestCase(unittest.TestCase):
     def test_request_signing4(self):
         # fetches the access control policy sub-resource for the 'johnsmith' bucket.
         date = 'Tue, 27 Mar 2007 19:44:46 +0000'
-        req = Request('s3://johnsmith/?acl',
-            method='GET', headers={'Date': date})
+        req = Request('s3://johnsmith/?acl', method='GET', headers={'Date': date})
         with self._mocked_date(date):
             httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'],
@@ -879,8 +878,9 @@ class S3TestCase(unittest.TestCase):
         with self._mocked_date(date):
             httpreq = self.download_request(req, self.spider)
         # botocore does not override Date with x-amz-date
-        self.assertEqual(httpreq.headers['Authorization'],
-                b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
+        self.assertEqual(
+            httpreq.headers['Authorization'],
+            b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
 
     def test_request_signing6(self):
         # uploads an object to a CNAME style virtual hosted bucket with metadata.
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 36743b1de..9841d7a76 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -43,8 +43,11 @@ class TestHttpProxyMiddleware(TestCase):
         os.environ.pop('file_proxy', None)
         mw = HttpProxyMiddleware()
 
-        for url, proxy in [('http://e.com', http_proxy),
-                ('https://e.com', https_proxy), ('file://tmp/a', None)]:
+        for url, proxy in [
+            ('http://e.com', http_proxy),
+            ('https://e.com', https_proxy),
+            ('file://tmp/a', None),
+        ]:
             req = Request(url)
             assert mw.process_request(req, spider) is None
             self.assertEqual(req.url, url)
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 551e124ab..52c4d71a6 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -72,7 +72,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
         url = 'http://www.example.com/302'
         url2 = 'http://www.example.com/redirected2'
         req = Request(url, method='POST', body='test',
-            headers={'Content-Type': 'text/plain', 'Content-length': '4'})
+                      headers={'Content-Type': 'text/plain', 'Content-length': '4'})
         rsp = Response(url, headers={'Location': url2}, status=302)
 
         req2 = self.mw.process_response(req, rsp, self.spider)
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index e118750e3..29357ba94 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -81,9 +81,17 @@ class RetryTest(unittest.TestCase):
         assert self.crawler.stats.get_value('retry/count') == 2
 
     def test_twistederrors(self):
-        exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError,
-                DNSLookupError, ConnectionRefusedError, ConnectionDone,
-                ConnectError, ConnectionLost, ResponseFailed]
+        exceptions = [
+            ConnectError,
+            ConnectionDone,
+            ConnectionLost,
+            ConnectionRefusedError,
+            defer.TimeoutError,
+            DNSLookupError,
+            ResponseFailed,
+            TCPTimedOutError,
+            TimeoutError,
+        ]
 
         for exc in exceptions:
             req = Request('http://www.scrapytest.org/%s' % exc.__name__)
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 7426107c1..38b95b710 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -177,10 +177,13 @@ class RFPDupeFilterTest(unittest.TestCase):
             dupefilter.log(r2, spider)
 
             assert crawler.stats.get_value('dupefilter/filtered') == 2
-            l.check_present(('scrapy.dupefilters', 'DEBUG',
-                ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
-                ' - no more duplicates will be shown'
-                ' (see DUPEFILTER_DEBUG to show all duplicates)')))
+            l.check_present(
+                (
+                    'scrapy.dupefilters',
+                    'DEBUG',
+                    'Filtered duplicate request: <GET http://scrapytest.org/index.html> - no more duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)'
+                )
+            )
 
             dupefilter.close('finished')
 
@@ -203,11 +206,19 @@ class RFPDupeFilterTest(unittest.TestCase):
             dupefilter.log(r2, spider)
 
             assert crawler.stats.get_value('dupefilter/filtered') == 2
-            l.check_present(('scrapy.dupefilters', 'DEBUG',
-                ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
-                ' (referer: None)')))
-            l.check_present(('scrapy.dupefilters', 'DEBUG',
-                ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
-                ' (referer: http://scrapytest.org/INDEX.html)')))
+            l.check_present(
+                (
+                    'scrapy.dupefilters',
+                    'DEBUG',
+                    'Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)'
+                )
+            )
+            l.check_present(
+                (
+                    'scrapy.dupefilters',
+                    'DEBUG',
+                    'Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: http://scrapytest.org/INDEX.html)'
+                )
+            )
 
             dupefilter.close('finished')
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 77da15ce6..a672963f3 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -415,8 +415,7 @@ class FormRequestTest(RequestTest):
         # using multiples values for a single key
         data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']}
         r3 = self.request_class("http://www.example.com", formdata=data)
-        self.assertQueryEqual(r3.body,
-            b'colours=red&colours=blue&colours=green&price=%C2%A3+100')
+        self.assertQueryEqual(r3.body, b'colours=red&colours=blue&colours=green&price=%C2%A3+100')
 
     def test_from_response_post(self):
         response = _buildresponse(
@@ -426,8 +425,7 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test2" value="xxx">
             </form>""",
             url="http://www.example.com/this/list.html")
-        req = self.request_class.from_response(response,
-                formdata={'one': ['two', 'three'], 'six': 'seven'})
+        req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
 
         self.assertEqual(req.method, 'POST')
         self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
@@ -446,8 +444,7 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test2" value="xxx \xc2\xb5">
             </form>""",
             url="http://www.example.com/this/list.html")
-        req = self.request_class.from_response(response,
-                formdata={'one': ['two', 'three'], 'six': 'seven'})
+        req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
 
         self.assertEqual(req.method, 'POST')
         self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
@@ -468,8 +465,7 @@ class FormRequestTest(RequestTest):
             url="http://www.example.com/this/list.html",
             encoding='latin1',
         )
-        req = self.request_class.from_response(response,
-                formdata={'one': ['two', 'three'], 'six': 'seven'})
+        req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
 
         self.assertEqual(req.method, 'POST')
         self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
@@ -488,8 +484,7 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test2" value="xxx µ">
             </form>""",
             url="http://www.example.com/this/list.html")
-        req = self.request_class.from_response(response,
-                formdata={'one': ['two', 'three'], 'six': 'seven'})
+        req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
 
         self.assertEqual(req.method, 'POST')
         self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
@@ -549,8 +544,7 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test2" value="xxx">
             </form>""",
             url="http://www.example.com/this/list.html")
-        r1 = self.request_class.from_response(response,
-                formdata={'one': ['two', 'three'], 'six': 'seven'})
+        r1 = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
         self.assertEqual(r1.method, 'GET')
         self.assertEqual(urlparse(r1.url).hostname, "www.example.com")
         self.assertEqual(urlparse(r1.url).path, "/this/get.php")
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 522ec4875..04f58d305 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -485,8 +485,10 @@ class TextResponseTest(BaseResponseTest):
             response.xpath("normalize-space(//p[@class=\"content\"])").getall(),
         )
         self.assertEqual(
-            response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()",
-                pclass="content", pcount=1).getall(),
+            response.xpath(
+                "//title[count(following::p[@class=$pclass])=$pcount]/text()",
+                pclass="content", pcount=1,
+            ).getall(),
             response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").getall(),
         )
 
@@ -566,12 +568,14 @@ class TextResponseTest(BaseResponseTest):
             'http://example.com',
             body=b'''<html><body><a href=" foo\n">click me</a></body></html>'''
         )
-        self._assert_followed_url(resp.css('a')[0],
-                                 'http://example.com/foo',
-                                  response=resp)
-        self._assert_followed_url(resp.css('a::attr(href)')[0],
-                                 'http://example.com/foo',
-                                  response=resp)
+        self._assert_followed_url(
+            resp.css('a')[0],
+            'http://example.com/foo',
+            response=resp)
+        self._assert_followed_url(
+            resp.css('a::attr(href)')[0],
+            'http://example.com/foo',
+            response=resp)
 
     def test_follow_encoding(self):
         resp1 = self.response_class(

From bf56517abfc3d2287c08e1c88f07de7d3b1c499c Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 19:18:29 -0300
Subject: [PATCH 104/243] Flake8: Remove E128 (tests, part 2)

---
 pytest.ini                               | 20 ++++-----
 tests/test_item.py                       | 22 +++++-----
 tests/test_linkextractors.py             |  6 +--
 tests/test_logformatter.py               | 21 +++++----
 tests/test_mail.py                       |  2 +-
 tests/test_middleware.py                 |  9 ++--
 tests/test_pipeline_crawl.py             | 12 ++++--
 tests/test_pipeline_media.py             | 12 +++---
 tests/test_spidermiddleware_httperror.py | 55 ++++++++----------------
 tests/test_spidermiddleware_offsite.py   | 32 ++++++++------
 10 files changed, 94 insertions(+), 97 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index d8eefea7e..0b2766031 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -200,19 +200,19 @@ flake8-ignore =
     tests/test_feedexport.py E501 F841
     tests/test_http_cookies.py E501
     tests/test_http_headers.py E501
-    tests/test_item.py E128 F841
     tests/test_http_request.py E402 E501
     tests/test_http_response.py E501
+    tests/test_item.py E501 F841
     tests/test_link.py E501
-    tests/test_linkextractors.py E501 E128
-    tests/test_loader.py E501 E741 E128 E117
-    tests/test_logformatter.py E128 E501
-    tests/test_mail.py E128 E501
-    tests/test_middleware.py E501 E128
-    tests/test_pipeline_crawl.py E501 E128
+    tests/test_linkextractors.py E501
+    tests/test_loader.py E501 E741 E117
+    tests/test_logformatter.py E501
+    tests/test_mail.py E501
+    tests/test_middleware.py E501
+    tests/test_pipeline_crawl.py E501
     tests/test_pipeline_files.py E501
     tests/test_pipeline_images.py F841 E501
-    tests/test_pipeline_media.py E501 E741 E128
+    tests/test_pipeline_media.py E501 E741
     tests/test_proxy_connect.py E501 E741
     tests/test_request_cb_kwargs.py E501
     tests/test_responsetypes.py E501
@@ -221,8 +221,8 @@ flake8-ignore =
     tests/test_selector.py E501
     tests/test_spider.py E501
     tests/test_spidermiddleware.py E501
-    tests/test_spidermiddleware_httperror.py E128 E501 E121
-    tests/test_spidermiddleware_offsite.py E501 E128 E111
+    tests/test_spidermiddleware_httperror.py E501 E121
+    tests/test_spidermiddleware_offsite.py E501 E111
     tests/test_spidermiddleware_output_chain.py E501
     tests/test_spidermiddleware_referer.py E501 F841 E501 E121
     tests/test_squeues.py E501 E741
diff --git a/tests/test_item.py b/tests/test_item.py
index 4017f6e84..4fe247cda 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -162,8 +162,7 @@ class ItemTest(unittest.TestCase):
         item = D(save='X', load='Y')
         self.assertEqual(item['save'], 'X')
         self.assertEqual(item['load'], 'Y')
-        self.assertEqual(D.fields, {'load': {'default': 'A'},
-            'save': {'default': 'A'}})
+        self.assertEqual(D.fields, {'load': {'default': 'A'}, 'save': {'default': 'A'}})
 
         # D class inverted
         class E(C, B):
@@ -171,8 +170,7 @@ class ItemTest(unittest.TestCase):
 
         self.assertEqual(E(save='X')['save'], 'X')
         self.assertEqual(E(load='X')['load'], 'X')
-        self.assertEqual(E.fields, {'load': {'default': 'C'},
-            'save': {'default': 'C'}})
+        self.assertEqual(E.fields, {'load': {'default': 'C'}, 'save': {'default': 'C'}})
 
     def test_metaclass_multiple_inheritance_diamond(self):
         class A(Item):
@@ -193,8 +191,9 @@ class ItemTest(unittest.TestCase):
 
         self.assertEqual(D(save='X')['save'], 'X')
         self.assertEqual(D(load='X')['load'], 'X')
-        self.assertEqual(D.fields, {'save': {'default': 'C'},
-            'load': {'default': 'D'}, 'update': {'default': 'D'}})
+        self.assertEqual(
+            D.fields,
+            {'save': {'default': 'C'}, 'load': {'default': 'D'}, 'update': {'default': 'D'}})
 
         # D class inverted
         class E(C, B):
@@ -202,8 +201,9 @@ class ItemTest(unittest.TestCase):
 
         self.assertEqual(E(save='X')['save'], 'X')
         self.assertEqual(E(load='X')['load'], 'X')
-        self.assertEqual(E.fields, {'save': {'default': 'C'},
-            'load': {'default': 'E'}, 'update': {'default': 'C'}})
+        self.assertEqual(
+            E.fields,
+            {'save': {'default': 'C'}, 'load': {'default': 'E'}, 'update': {'default': 'C'}})
 
     def test_metaclass_multiple_inheritance_without_metaclass(self):
         class A(Item):
@@ -223,8 +223,7 @@ class ItemTest(unittest.TestCase):
 
         self.assertRaises(KeyError, D, not_allowed='value')
         self.assertEqual(D(save='X')['save'], 'X')
-        self.assertEqual(D.fields, {'save': {'default': 'A'},
-            'load': {'default': 'A'}})
+        self.assertEqual(D.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}})
 
         # D class inverted
         class E(C, B):
@@ -232,8 +231,7 @@ class ItemTest(unittest.TestCase):
 
         self.assertRaises(KeyError, E, not_allowed='value')
         self.assertEqual(E(save='X')['save'], 'X')
-        self.assertEqual(E.fields, {'save': {'default': 'A'},
-            'load': {'default': 'A'}})
+        self.assertEqual(E.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}})
 
     def test_to_dict(self):
         class TestItem(Item):
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 68e8514ba..7e1b62b7f 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -171,9 +171,9 @@ class Base:
             self.assertEqual(lx.matches(url1), False)
             self.assertEqual(lx.matches(url2), True)
 
-            lx = self.extractor_cls(allow=('blah1',), deny=('blah2',),
-                                   allow_domains=('blah1.com',),
-                                   deny_domains=('blah2.com',))
+            lx = self.extractor_cls(allow=['blah1'], deny=['blah2'],
+                                    allow_domains=['blah1.com'],
+                                    deny_domains=['blah2.com'])
             self.assertEqual(lx.matches('http://blah1.com/blah1'), True)
             self.assertEqual(lx.matches('http://blah1.com/blah2'), False)
             self.assertEqual(lx.matches('http://blah2.com/blah1'), False)
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index cd6cb8016..7064337ad 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -34,15 +34,15 @@ class LogFormatterTestCase(unittest.TestCase):
         res = Response("http://www.example.com")
         logkws = self.formatter.crawled(req, res, self.spider)
         logline = logkws['msg'] % logkws['args']
-        self.assertEqual(logline,
-            "Crawled (200) <GET http://www.example.com> (referer: None)")
+        self.assertEqual(logline, "Crawled (200) <GET http://www.example.com> (referer: None)")
 
     def test_crawled_without_referer(self):
         req = Request("http://www.example.com", headers={'referer': 'http://example.com'})
         res = Response("http://www.example.com", flags=['cached'])
         logkws = self.formatter.crawled(req, res, self.spider)
         logline = logkws['msg'] % logkws['args']
-        self.assertEqual(logline,
+        self.assertEqual(
+            logline,
             "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
 
     def test_flags_in_request(self):
@@ -50,8 +50,9 @@ class LogFormatterTestCase(unittest.TestCase):
         res = Response("http://www.example.com")
         logkws = self.formatter.crawled(req, res, self.spider)
         logline = logkws['msg'] % logkws['args']
-        self.assertEqual(logline,
-        "Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)")
+        self.assertEqual(
+            logline,
+            "Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)")
 
     def test_dropped(self):
         item = {}
@@ -140,7 +141,8 @@ class LogformatterSubclassTest(LogFormatterTestCase):
         res = Response("http://www.example.com")
         logkws = self.formatter.crawled(req, res, self.spider)
         logline = logkws['msg'] % logkws['args']
-        self.assertEqual(logline,
+        self.assertEqual(
+            logline,
             "Crawled (200) <GET http://www.example.com> (referer: None) []")
 
     def test_crawled_without_referer(self):
@@ -148,7 +150,8 @@ class LogformatterSubclassTest(LogFormatterTestCase):
         res = Response("http://www.example.com")
         logkws = self.formatter.crawled(req, res, self.spider)
         logline = logkws['msg'] % logkws['args']
-        self.assertEqual(logline,
+        self.assertEqual(
+            logline,
             "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
 
     def test_flags_in_request(self):
@@ -156,7 +159,9 @@ class LogformatterSubclassTest(LogFormatterTestCase):
         res = Response("http://www.example.com")
         logkws = self.formatter.crawled(req, res, self.spider)
         logline = logkws['msg'] % logkws['args']
-        self.assertEqual(logline, "Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']")
+        self.assertEqual(
+            logline,
+            "Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']")
 
 
 class SkipMessagesLogFormatter(LogFormatter):
diff --git a/tests/test_mail.py b/tests/test_mail.py
index f5cb81a8b..53dbc0686 100644
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@ -49,7 +49,7 @@ class MailSenderTest(unittest.TestCase):
 
         mailsender = MailSender(debug=True)
         mailsender.send(to=['test@scrapy.org'], subject='subject', body='body',
-                       attachs=attachs, _callback=self._catch_mail_sent)
+                        attachs=attachs, _callback=self._catch_mail_sent)
 
         assert self.catched_msg
         self.assertEqual(self.catched_msg['to'], ['test@scrapy.org'])
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index 3af514bb0..3364d2258 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -69,11 +69,14 @@ class MiddlewareManagerTest(unittest.TestCase):
 
     def test_methods(self):
         mwman = TestMiddlewareManager(M1(), M2(), M3())
-        self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']],
+        self.assertEqual(
+            [x.__self__.__class__ for x in mwman.methods['open_spider']],
             [M1, M2])
-        self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']],
+        self.assertEqual(
+            [x.__self__.__class__ for x in mwman.methods['close_spider']],
             [M2, M1])
-        self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']],
+        self.assertEqual(
+            [x.__self__.__class__ for x in mwman.methods['process']],
             [M1, M3])
 
     def test_enabled(self):
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 24c516473..188c5c3cf 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -132,7 +132,8 @@ class FileDownloadCrawlTestCase(TestCase):
     def test_download_media(self):
         crawler = self._create_crawler(MediaDownloadSpider)
         with LogCapture() as log:
-            yield crawler.crawl(self.mockserver.url("/files/images/"),
+            yield crawler.crawl(
+                self.mockserver.url("/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key)
         self._assert_files_downloaded(self.items, str(log))
@@ -141,7 +142,8 @@ class FileDownloadCrawlTestCase(TestCase):
     def test_download_media_wrong_urls(self):
         crawler = self._create_crawler(BrokenLinksMediaDownloadSpider)
         with LogCapture() as log:
-            yield crawler.crawl(self.mockserver.url("/files/images/"),
+            yield crawler.crawl(
+                self.mockserver.url("/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key)
         self._assert_files_download_failure(crawler, self.items, 404, str(log))
@@ -150,7 +152,8 @@ class FileDownloadCrawlTestCase(TestCase):
     def test_download_media_redirected_default_failure(self):
         crawler = self._create_crawler(RedirectedMediaDownloadSpider)
         with LogCapture() as log:
-            yield crawler.crawl(self.mockserver.url("/files/images/"),
+            yield crawler.crawl(
+                self.mockserver.url("/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key,
                 mockserver=self.mockserver)
@@ -164,7 +167,8 @@ class FileDownloadCrawlTestCase(TestCase):
 
         crawler = self._create_crawler(RedirectedMediaDownloadSpider)
         with LogCapture() as log:
-            yield crawler.crawl(self.mockserver.url("/files/images/"),
+            yield crawler.crawl(
+                self.mockserver.url("/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key,
                 mockserver=self.mockserver)
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 949f0dea1..b51f82422 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -214,9 +214,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
         item = dict(requests=req)
         new_item = yield self.pipe.process_item(item, self.spider)
         self.assertEqual(new_item['results'], [(True, rsp)])
-        self.assertEqual(self.pipe._mockcalled,
-                ['get_media_requests', 'media_to_download',
-                    'media_downloaded', 'request_callback', 'item_completed'])
+        self.assertEqual(
+            self.pipe._mockcalled,
+            ['get_media_requests', 'media_to_download', 'media_downloaded', 'request_callback', 'item_completed'])
 
     @inlineCallbacks
     def test_result_failure(self):
@@ -227,9 +227,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
         item = dict(requests=req)
         new_item = yield self.pipe.process_item(item, self.spider)
         self.assertEqual(new_item['results'], [(False, fail)])
-        self.assertEqual(self.pipe._mockcalled,
-                ['get_media_requests', 'media_to_download',
-                    'media_failed', 'request_errback', 'item_completed'])
+        self.assertEqual(
+            self.pipe._mockcalled,
+            ['get_media_requests', 'media_to_download', 'media_failed', 'request_errback', 'item_completed'])
 
     @inlineCallbacks
     def test_mix_of_success_and_failure(self):
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index 6b61df56f..714279ae0 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -68,29 +68,23 @@ class TestHttpErrorMiddleware(TestCase):
         self.res200, self.res404 = _responses(self.req, [200, 404])
 
     def test_process_spider_input(self):
-        self.assertEqual(None,
-                self.mw.process_spider_input(self.res200, self.spider))
-        self.assertRaises(HttpError,
-                self.mw.process_spider_input, self.res404, self.spider)
+        self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
+        self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)
 
     def test_process_spider_exception(self):
-        self.assertEqual([],
-                self.mw.process_spider_exception(self.res404,
-                        HttpError(self.res404), self.spider))
-        self.assertEqual(None,
-                self.mw.process_spider_exception(self.res404,
-                        Exception(), self.spider))
+        self.assertEqual(
+            [],
+            self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider))
+        self.assertIsNone(self.mw.process_spider_exception(self.res404, Exception(), self.spider))
 
     def test_handle_httpstatus_list(self):
         res = self.res404.copy()
         res.request = Request('http://scrapytest.org',
                               meta={'handle_httpstatus_list': [404]})
-        self.assertEqual(None,
-            self.mw.process_spider_input(res, self.spider))
+        self.assertIsNone(self.mw.process_spider_input(res, self.spider))
 
         self.spider.handle_httpstatus_list = [404]
-        self.assertEqual(None,
-            self.mw.process_spider_input(self.res404, self.spider))
+        self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
 
 
 class TestHttpErrorMiddlewareSettings(TestCase):
@@ -103,12 +97,9 @@ class TestHttpErrorMiddlewareSettings(TestCase):
         self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
 
     def test_process_spider_input(self):
-        self.assertEqual(None,
-                self.mw.process_spider_input(self.res200, self.spider))
-        self.assertRaises(HttpError,
-                self.mw.process_spider_input, self.res404, self.spider)
-        self.assertEqual(None,
-                self.mw.process_spider_input(self.res402, self.spider))
+        self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
+        self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)
+        self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider))
 
     def test_meta_overrides_settings(self):
         request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
@@ -117,17 +108,13 @@ class TestHttpErrorMiddlewareSettings(TestCase):
         res402 = self.res402.copy()
         res402.request = request
 
-        self.assertEqual(None,
-            self.mw.process_spider_input(res404, self.spider))
-        self.assertRaises(HttpError,
-                self.mw.process_spider_input, res402, self.spider)
+        self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
+        self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)
 
     def test_spider_override_settings(self):
         self.spider.handle_httpstatus_list = [404]
-        self.assertEqual(None,
-            self.mw.process_spider_input(self.res404, self.spider))
-        self.assertRaises(HttpError,
-                self.mw.process_spider_input, self.res402, self.spider)
+        self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
+        self.assertRaises(HttpError, self.mw.process_spider_input, self.res402, self.spider)
 
 
 class TestHttpErrorMiddlewareHandleAll(TestCase):
@@ -139,10 +126,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
         self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
 
     def test_process_spider_input(self):
-        self.assertEqual(None,
-                self.mw.process_spider_input(self.res200, self.spider))
-        self.assertEqual(None,
-                self.mw.process_spider_input(self.res404, self.spider))
+        self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
+        self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
 
     def test_meta_overrides_settings(self):
         request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
@@ -151,10 +136,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
         res402 = self.res402.copy()
         res402.request = request
 
-        self.assertEqual(None,
-            self.mw.process_spider_input(res404, self.spider))
-        self.assertRaises(HttpError,
-                self.mw.process_spider_input, res402, self.spider)
+        self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
+        self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)
 
 
 class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index b96807bc2..0f4b98a07 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -22,20 +22,24 @@ class TestOffsiteMiddleware(TestCase):
     def test_process_spider_output(self):
         res = Response('http://scrapytest.org')
 
-        onsite_reqs = [Request('http://scrapytest.org/1'),
-                       Request('http://scrapy.org/1'),
-                       Request('http://sub.scrapy.org/1'),
-                       Request('http://offsite.tld/letmepass', dont_filter=True),
-                       Request('http://scrapy.test.org/'),
-                       Request('http://scrapy.test.org:8000/')]
-        offsite_reqs = [Request('http://scrapy2.org'),
-                       Request('http://offsite.tld/'),
-                       Request('http://offsite.tld/scrapytest.org'),
-                       Request('http://offsite.tld/rogue.scrapytest.org'),
-                       Request('http://rogue.scrapytest.org.haha.com'),
-                       Request('http://roguescrapytest.org'),
-                       Request('http://test.org/'),
-                       Request('http://notscrapy.test.org/')]
+        onsite_reqs = [
+            Request('http://scrapytest.org/1'),
+            Request('http://scrapy.org/1'),
+            Request('http://sub.scrapy.org/1'),
+            Request('http://offsite.tld/letmepass', dont_filter=True),
+            Request('http://scrapy.test.org/'),
+            Request('http://scrapy.test.org:8000/'),
+        ]
+        offsite_reqs = [
+            Request('http://scrapy2.org'),
+            Request('http://offsite.tld/'),
+            Request('http://offsite.tld/scrapytest.org'),
+            Request('http://offsite.tld/rogue.scrapytest.org'),
+            Request('http://rogue.scrapytest.org.haha.com'),
+            Request('http://roguescrapytest.org'),
+            Request('http://test.org/'),
+            Request('http://notscrapy.test.org/'),
+        ]
         reqs = onsite_reqs + offsite_reqs
 
         out = list(self.mw.process_spider_output(res, reqs, self.spider))

From 881b4f417f645fa9719e6c54eb3788cc67d56053 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 19:33:22 -0300
Subject: [PATCH 105/243] Flake8: Remove E128 (tests, part 3)

---
 pytest.ini                          | 18 ++++++-------
 tests/test_settings/__init__.py     | 18 ++++++++-----
 tests/test_spiderloader/__init__.py | 21 ++++++++++-----
 tests/test_utils_conf.py            |  3 ++-
 tests/test_utils_iterators.py       | 23 +++++++++++------
 tests/test_utils_reqser.py          |  3 ++-
 tests/test_utils_request.py         |  5 ++--
 tests/test_utils_sitemap.py         |  9 +++++--
 tests/test_webclient.py             | 40 +++++++++++++++++------------
 9 files changed, 87 insertions(+), 53 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 0b2766031..7c3c9cc5b 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -227,23 +227,23 @@ flake8-ignore =
     tests/test_spidermiddleware_referer.py E501 F841 E501 E121
     tests/test_squeues.py E501 E741
     tests/test_utils_asyncio.py E501
-    tests/test_utils_conf.py E501 E128
+    tests/test_utils_conf.py E501
     tests/test_utils_curl.py E501
     tests/test_utils_datatypes.py E402 E501
     tests/test_utils_defer.py E501 F841
     tests/test_utils_deprecate.py F841 E501
-    tests/test_utils_http.py E501 E128
-    tests/test_utils_iterators.py E501 E128 E129
+    tests/test_utils_http.py E501
+    tests/test_utils_iterators.py E501 E129
     tests/test_utils_log.py E741
     tests/test_utils_python.py E501
-    tests/test_utils_reqser.py E501 E128
-    tests/test_utils_request.py E501 E128
+    tests/test_utils_reqser.py E501
+    tests/test_utils_request.py E501
     tests/test_utils_response.py E501
     tests/test_utils_signal.py E741 F841
-    tests/test_utils_sitemap.py E128 E501
+    tests/test_utils_sitemap.py E501
     tests/test_utils_url.py E501 E501
-    tests/test_webclient.py E501 E128 E402
+    tests/test_webclient.py E501 E402
     tests/test_cmdline/__init__.py E501
-    tests/test_settings/__init__.py E501 E128
-    tests/test_spiderloader/__init__.py E128 E501
+    tests/test_settings/__init__.py E501
+    tests/test_spiderloader/__init__.py E501
     tests/test_utils_misc/__init__.py E501
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index fda44653a..2da6aa4b5 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -314,13 +314,17 @@ class BaseSettingsTest(unittest.TestCase):
                           'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'),
                           'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
                           'HASNOBASE': BaseSettings({3: 3000}, 'default')})
-        self.assertDictEqual(s.copy_to_dict(),
-                            {'HASNOBASE': {3: 3000},
-                             'TEST': {1: 10, 3: 30},
-                             'TEST_BASE': {1: 1, 2: 2},
-                             'TEST_BOOLEAN': False,
-                             'TEST_LIST': [1, 2],
-                             'TEST_STRING': 'a string'})
+        self.assertDictEqual(
+            s.copy_to_dict(),
+            {
+                'HASNOBASE': {3: 3000},
+                'TEST': {1: 10, 3: 30},
+                'TEST_BASE': {1: 1, 2: 2},
+                'TEST_LIST': [1, 2],
+                'TEST_BOOLEAN': False,
+                'TEST_STRING': 'a string',
+            }
+        )
 
     def test_freeze(self):
         self.settings.freeze()
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index d8be6e277..b20e1b3f2 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -40,7 +40,8 @@ class SpiderLoaderTest(unittest.TestCase):
         verifyObject(ISpiderLoader, self.spider_loader)
 
     def test_list(self):
-        self.assertEqual(set(self.spider_loader.list()),
+        self.assertEqual(
+            set(self.spider_loader.list()),
             set(['spider1', 'spider2', 'spider3', 'spider4']))
 
     def test_load(self):
@@ -48,17 +49,23 @@ class SpiderLoaderTest(unittest.TestCase):
         self.assertEqual(spider1.__name__, 'Spider1')
 
     def test_find_by_request(self):
-        self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy1.org/test')),
+        self.assertEqual(
+            self.spider_loader.find_by_request(Request('http://scrapy1.org/test')),
             ['spider1'])
-        self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy2.org/test')),
+        self.assertEqual(
+            self.spider_loader.find_by_request(Request('http://scrapy2.org/test')),
             ['spider2'])
-        self.assertEqual(set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))),
+        self.assertEqual(
+            set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))),
             set(['spider1', 'spider2']))
-        self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy999.org/test')),
+        self.assertEqual(
+            self.spider_loader.find_by_request(Request('http://scrapy999.org/test')),
             [])
-        self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com')),
+        self.assertEqual(
+            self.spider_loader.find_by_request(Request('http://spider3.com')),
             [])
-        self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')),
+        self.assertEqual(
+            self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')),
             ['spider3'])
 
     def test_load_spider_module(self):
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index 332120021..e5d3ef582 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -93,7 +93,8 @@ class BuildComponentListTest(unittest.TestCase):
 class UtilsConfTestCase(unittest.TestCase):
 
     def test_arglist_to_dict(self):
-        self.assertEqual(arglist_to_dict(['arg1=val1', 'arg2=val2']),
+        self.assertEqual(
+            arglist_to_dict(['arg1=val1', 'arg2=val2']),
             {'arg1': 'val1', 'arg2': 'val2'})
 
 
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index a85087619..c677dabe5 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -47,8 +47,7 @@ class XmliterTestCase(unittest.TestCase):
             </root>
         """
         response = XmlResponse(url="http://example.com", body=body)
-        nodenames = [e.xpath('name()').getall()
-                 for e in self.xmliter(response, 'matchme...')]
+        nodenames = [e.xpath('name()').getall() for e in self.xmliter(response, 'matchme...')]
         self.assertEqual(nodenames, [['matchme...']])
 
     def test_xmliter_unicode(self):
@@ -359,15 +358,23 @@ class UtilsCsvTestCase(unittest.TestCase):
 
         response = TextResponse(url="http://example.com/", body=body1, encoding='latin1')
         csv = csviter(response)
-        self.assertEqual([row for row in csv],
-            [{u'id': u'1', u'name': u'latin1', u'value': u'test'},
-             {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'}])
+        self.assertEqual(
+            list(csv),
+            [
+                {u'id': u'1', u'name': u'latin1', u'value': u'test'},
+                {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'},
+            ]
+        )
 
         response = TextResponse(url="http://example.com/", body=body2, encoding='cp852')
         csv = csviter(response)
-        self.assertEqual([row for row in csv],
-            [{u'id': u'1', u'name': u'cp852', u'value': u'test'},
-             {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'}])
+        self.assertEqual(
+            list(csv),
+            [
+                {u'id': u'1', u'name': u'cp852', u'value': u'test'},
+                {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'},
+            ]
+        )
 
 
 class TestHelper(unittest.TestCase):
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 50b026d1c..450e4bdca 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -15,7 +15,8 @@ class RequestSerializationTest(unittest.TestCase):
         self._assert_serializes_ok(r)
 
     def test_all_attributes(self):
-        r = Request("http://www.example.com",
+        r = Request(
+            url="http://www.example.com",
             callback=self.spider.parse_item,
             errback=self.spider.handle_error,
             method="POST",
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 45f0f59e4..50efb63ca 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -36,8 +36,9 @@ class UtilsRequestTest(unittest.TestCase):
         self.assertEqual(request_fingerprint(r1),
                          request_fingerprint(r1, include_headers=['Accept-Language']))
 
-        self.assertNotEqual(request_fingerprint(r1),
-                         request_fingerprint(r2, include_headers=['Accept-Language']))
+        self.assertNotEqual(
+            request_fingerprint(r1),
+            request_fingerprint(r2, include_headers=['Accept-Language']))
 
         self.assertEqual(request_fingerprint(r3, include_headers=['accept-language', 'sessionid']),
                          request_fingerprint(r3, include_headers=['SESSIONID', 'Accept-Language']))
diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py
index 08b215434..d77978ff1 100644
--- a/tests/test_utils_sitemap.py
+++ b/tests/test_utils_sitemap.py
@@ -22,8 +22,13 @@ class SitemapTest(unittest.TestCase):
   </url>
 </urlset>""")
         assert s.type == 'urlset'
-        self.assertEqual(list(s),
-            [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'}])
+        self.assertEqual(
+            list(s),
+            [
+                {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
+                {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'},
+            ]
+        )
 
     def test_sitemap_index(self):
         s = Sitemap(b"""<?xml version="1.0" encoding="UTF-8"?>
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 307fadb5c..188e54602 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -47,8 +47,9 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
         return f
 
     from twisted.web.client import _makeGetterFactory
-    return _makeGetterFactory(to_bytes(url), _clientfactory,
-        contextFactory=contextFactory, *args, **kwargs).deferred
+    return _makeGetterFactory(
+        to_bytes(url), _clientfactory, contextFactory=contextFactory, *args, **kwargs
+    ).deferred
 
 
 class ParseUrlTestCase(unittest.TestCase):
@@ -105,7 +106,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
                 'Content-Length': '12981',
                 'Useful': 'value'}))
 
-        self._test(factory,
+        self._test(
+            factory,
             b"GET /bar HTTP/1.0\r\n"
             b"Content-Length: 9\r\n"
             b"Useful: value\r\n"
@@ -118,7 +120,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
 
         # test minimal sent headers
         factory = client.ScrapyHTTPClientFactory(Request('http://foo/bar'))
-        self._test(factory,
+        self._test(
+            factory,
             b"GET /bar HTTP/1.0\r\n"
             b"Host: foo\r\n"
             b"\r\n")
@@ -130,7 +133,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             body='name=value',
             headers={'Content-Type': 'application/x-www-form-urlencoded'}))
 
-        self._test(factory,
+        self._test(
+            factory,
             b"POST /bar HTTP/1.0\r\n"
             b"Host: foo\r\n"
             b"Connection: close\r\n"
@@ -145,7 +149,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             url='http://foo/bar'
         ))
 
-        self._test(factory,
+        self._test(
+            factory,
             b"POST /bar HTTP/1.0\r\n"
             b"Host: foo\r\n"
             b"Content-Length: 0\r\n"
@@ -160,7 +165,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             },
         ))
 
-        self._test(factory,
+        self._test(
+            factory,
             b"GET /bar HTTP/1.0\r\n"
             b"Host: foo\r\n"
             b"X-Meta-Multivalued: value1\r\n"
@@ -177,7 +183,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             }),
         ))
 
-        self._test(factory,
+        self._test(
+            factory,
             b"GET /bar HTTP/1.0\r\n"
             b"Host: foo\r\n"
             b"X-Meta-Multivalued: value1\r\n"
@@ -206,8 +213,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
         protocol.dataReceived(b"Hello: World\n")
         protocol.dataReceived(b"Foo: Bar\n")
         protocol.dataReceived(b"\n")
-        self.assertEqual(protocol.headers,
-            Headers({'Hello': ['World'], 'Foo': ['Bar']}))
+        self.assertEqual(protocol.headers, Headers({'Hello': ['World'], 'Foo': ['Bar']}))
 
 
 class EncodingResource(resource.Resource):
@@ -340,10 +346,11 @@ class WebClientTestCase(unittest.TestCase):
         return getPage(self.getURL("redirect")).addCallback(self._cbRedirect)
 
     def _cbRedirect(self, pageData):
-        self.assertEqual(pageData,
-                b'\n<html>\n    <head>\n        <meta http-equiv="refresh" content="0;URL=/file">\n'
-                b'    </head>\n    <body bgcolor="#FFFFFF" text="#000000">\n    '
-                b'<a href="/file">click here</a>\n    </body>\n</html>\n')
+        self.assertEqual(
+            pageData,
+            b'\n<html>\n    <head>\n        <meta http-equiv="refresh" content="0;URL=/file">\n'
+            b'    </head>\n    <body bgcolor="#FFFFFF" text="#000000">\n    '
+            b'<a href="/file">click here</a>\n    </body>\n</html>\n')
 
     def test_encoding(self):
         """ Test that non-standart body encoding matches
@@ -403,8 +410,9 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
         s = "0123456789" * 10
         settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': self.custom_ciphers})
         client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None)
-        return getPage(self.getURL("payload"), body=s,
-                       contextFactory=client_context_factory).addCallback(self.assertEqual, to_bytes(s))
+        return getPage(
+            self.getURL("payload"), body=s, contextFactory=client_context_factory
+        ).addCallback(self.assertEqual, to_bytes(s))
 
     def testPayloadDefaultCiphers(self):
         s = "0123456789" * 10

From d472402a0232781753515d9552b7a1997b43543a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 14:39:17 -0300
Subject: [PATCH 106/243] Fix pickle test for pypy

---
 pytest.ini            | 1 +
 tests/test_squeues.py | 9 +++++++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 4f3494e0e..d107c1fbe 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -166,6 +166,7 @@ flake8-ignore =
     scrapy/signalmanager.py E501
     scrapy/spiderloader.py F841 E501 E126
     scrapy/squeues.py E128
+    scrapy/squeues.py E501
     scrapy/statscollectors.py E501
     # tests
     tests/__init__.py E402 E501
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index 51c0c028a..d2cf9135f 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -1,3 +1,6 @@
+import pickle
+import sys
+
 from queuelib.tests import test_queue as t
 from scrapy.squeues import (
     MarshalFifoDiskQueueNonRequest as MarshalFifoDiskQueue,
@@ -108,8 +111,10 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
         try:
             q.push(lambda x: x)
         except ValueError as exc:
-            self.assertIsInstance(exc.__context__, AttributeError)
-
+            if hasattr(sys, "pypy_version_info"):
+                self.assertIsInstance(exc.__context__, pickle.PicklingError)
+            else:
+                self.assertIsInstance(exc.__context__, AttributeError)
         sel = Selector(text='<html><body><p>some text</p></body></html>')
         try:
             q.push(sel)

From b59dfb75fa72346f8268b83dedd2c1f9af460c71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 7 May 2020 14:14:59 +0200
Subject: [PATCH 107/243] Update disabled Pylint checks

---
 pylintrc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/pylintrc b/pylintrc
index c52a4c2d0..129c7bf7d 100644
--- a/pylintrc
+++ b/pylintrc
@@ -13,6 +13,7 @@ disable=abstract-method,
         bad-mcs-classmethod-argument,
         bad-super-call,
         bad-whitespace,
+        bare-except,
         blacklisted-name,
         broad-except,
         c-extension-no-member,
@@ -39,6 +40,8 @@ disable=abstract-method,
         inconsistent-return-statements,
         inherit-non-class,
         invalid-name,
+        invalid-overridden-method,
+        isinstance-second-argument-not-valid-type,
         keyword-arg-before-vararg,
         line-too-long,
         logging-format-interpolation,
@@ -77,6 +80,7 @@ disable=abstract-method,
         too-many-ancestors,
         too-many-arguments,
         too-many-branches,
+        too-many-format-args,
         too-many-function-args,
         too-many-instance-attributes,
         too-many-lines,
@@ -87,6 +91,7 @@ disable=abstract-method,
         trailing-whitespace,
         unbalanced-tuple-unpacking,
         undefined-variable,
+        undefined-loop-variable,
         unexpected-special-method-signature,
         ungrouped-imports,
         unidiomatic-typecheck,

From 422e6429b56e42b8344a0e46c45f4106d374d024 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 7 May 2020 09:22:14 -0300
Subject: [PATCH 108/243] Add mising len check in spiderloader

---
 scrapy/spiderloader.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 8dc89c2e9..92aed9b8e 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
-from collections import defaultdict
 import traceback
 import warnings
+from collections import defaultdict
 
 from zope.interface import implementer
 
@@ -16,6 +16,7 @@ class SpiderLoader:
     SpiderLoader is a class which locates and loads spiders
     in a Scrapy project.
     """
+
     def __init__(self, settings):
         self.spider_modules = settings.getlist('SPIDER_MODULES')
         self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY')
@@ -29,6 +30,7 @@ class SpiderLoader:
             dupes.extend([
                 "  {cls} named {name!r} (in {module})".format(module=mod, cls=cls, name=name)
                 for mod, cls in locations
+                if len(locations) > 1
             ])
 
         if dupes:
@@ -49,10 +51,9 @@ class SpiderLoader:
                     self._load_spiders(module)
             except ImportError:
                 if self.warn_only:
-                    msg = (
-                        "\n{tb}Could not load spiders from module '{modname}'. "
-                        "See above traceback for details.".format(modname=name, tb=traceback.format_exc())
-                    )
+                    msg = ("\n{tb}Could not load spiders from module '{modname}'. "
+                           "See above traceback for details.".format(
+                                modname=name, tb=traceback.format_exc()))
                     warnings.warn(msg, RuntimeWarning)
                 else:
                     raise
@@ -76,8 +77,10 @@ class SpiderLoader:
         """
         Return the list of spider names that can handle the given request.
         """
-        return [name for name, cls in self._spiders.items()
-                if cls.handles_request(request)]
+        return [
+            name for name, cls in self._spiders.items()
+            if cls.handles_request(request)
+        ]
 
     def list(self):
         """

From e0127a31230d4be13b1bd29e62d75c2954b47d9e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 7 May 2020 12:48:43 -0300
Subject: [PATCH 109/243] Refactor warnings in spiderloader

---
 scrapy/spiderloader.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 92aed9b8e..63da55718 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -35,9 +35,11 @@ class SpiderLoader:
 
         if dupes:
             dupes_string = "\n\n".join(dupes)
-            msg = ("There are several spiders with the same name:\n\n"
-                   "{}\n\n  This can cause unexpected behavior.".format(dupes_string))
-            warnings.warn(msg, UserWarning)
+            warnings.warn(
+                "There are several spiders with the same name:\n\n"
+                "{}\n\n  This can cause unexpected behavior.".format(dupes_string),
+                category=UserWarning,
+            )
 
     def _load_spiders(self, module):
         for spcls in iter_spider_classes(module):
@@ -51,10 +53,13 @@ class SpiderLoader:
                     self._load_spiders(module)
             except ImportError:
                 if self.warn_only:
-                    msg = ("\n{tb}Could not load spiders from module '{modname}'. "
-                           "See above traceback for details.".format(
-                                modname=name, tb=traceback.format_exc()))
-                    warnings.warn(msg, RuntimeWarning)
+                    warnings.warn(
+                        "\n{tb}Could not load spiders from module '{modname}'. "
+                        "See above traceback for details.".format(
+                            modname=name, tb=traceback.format_exc()
+                        ),
+                        category=RuntimeWarning,
+                    )
                 else:
                     raise
         self._check_name_duplicates()

From 5256eae60d3685de51c1f3891abe157e15d14def Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 7 May 2020 14:37:41 -0300
Subject: [PATCH 110/243] Meta class to handle isinstance checks for BaseItem

---
 pytest.ini                  |  2 +-
 scrapy/commands/parse.py    |  4 +--
 scrapy/contracts/default.py |  8 +++---
 scrapy/core/scraper.py      |  4 +--
 scrapy/exporters.py         |  4 +--
 scrapy/item.py              | 20 +++++++++++--
 scrapy/shell.py             |  5 ++--
 scrapy/utils/misc.py        |  4 +--
 scrapy/utils/serialize.py   |  4 +--
 tests/test_item.py          | 56 +++++++++++++++++++++++++++++++++----
 10 files changed, 85 insertions(+), 26 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 5a86ce2a7..292dbce41 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -204,7 +204,7 @@ flake8-ignore =
     tests/test_http_headers.py E501
     tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
     tests/test_http_response.py E501 E128
-    tests/test_item.py E128 F841
+    tests/test_item.py E128 F841 E501
     tests/test_link.py E501
     tests/test_linkextractors.py E501 E128 E124
     tests/test_loader.py E501 E741 E128 E117
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 1cefed106..098827ab9 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -5,7 +5,7 @@ from w3lib.url import is_url
 
 from scrapy.commands import ScrapyCommand
 from scrapy.http import Request
-from scrapy.item import BaseItem
+from scrapy.item import _BaseItem
 from scrapy.utils import display
 from scrapy.utils.conf import arglist_to_dict
 from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
@@ -117,7 +117,7 @@ class Command(ScrapyCommand):
         items, requests = [], []
 
         for x in iterate_spider_output(callback(response, **cb_kwargs)):
-            if isinstance(x, (BaseItem, dict)):
+            if isinstance(x, (_BaseItem, dict)):
                 items.append(x)
             elif isinstance(x, Request):
                 requests.append(x)
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index a1b0f8f22..cdc2bac15 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -1,6 +1,6 @@
 import json
 
-from scrapy.item import BaseItem
+from scrapy.item import _BaseItem
 from scrapy.http import Request
 from scrapy.exceptions import ContractFail
 
@@ -51,8 +51,8 @@ class ReturnsContract(Contract):
     objects = {
         'request': Request,
         'requests': Request,
-        'item': (BaseItem, dict),
-        'items': (BaseItem, dict),
+        'item': (_BaseItem, dict),
+        'items': (_BaseItem, dict),
     }
 
     def __init__(self, *args, **kwargs):
@@ -103,7 +103,7 @@ class ScrapesContract(Contract):
 
     def post_process(self, output):
         for x in output:
-            if isinstance(x, (BaseItem, dict)):
+            if isinstance(x, (_BaseItem, dict)):
                 missing = [arg for arg in self.args if arg not in x]
                 if missing:
                     raise ContractFail(
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index edbb4dd66..6785e103d 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -14,7 +14,7 @@ from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
 from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
 from scrapy import signals
 from scrapy.http import Request, Response
-from scrapy.item import BaseItem
+from scrapy.item import _BaseItem
 from scrapy.core.spidermw import SpiderMiddlewareManager
 
 
@@ -191,7 +191,7 @@ class Scraper:
         """
         if isinstance(output, Request):
             self.crawler.engine.crawl(request=output, spider=spider)
-        elif isinstance(output, (BaseItem, dict)):
+        elif isinstance(output, (_BaseItem, dict)):
             self.slot.itemproc_size += 1
             dfd = self.itemproc.process_item(output, spider)
             dfd.addBoth(self._itemproc_finished, output, response, spider)
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 0cb6cef98..4731b925a 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -12,7 +12,7 @@ from xml.sax.saxutils import XMLGenerator
 
 from scrapy.utils.serialize import ScrapyJSONEncoder
 from scrapy.utils.python import to_bytes, to_unicode, is_listlike
-from scrapy.item import BaseItem
+from scrapy.item import _BaseItem
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
@@ -312,7 +312,7 @@ class PythonItemExporter(BaseItemExporter):
         return serializer(value)
 
     def _serialize_value(self, value):
-        if isinstance(value, BaseItem):
+        if isinstance(value, _BaseItem):
             return self.export_item(value)
         if isinstance(value, dict):
             return dict(self._serialize_dict(value))
diff --git a/scrapy/item.py b/scrapy/item.py
index 46d20d017..f468ff86f 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -14,7 +14,23 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.trackref import object_ref
 
 
-class BaseItem(object_ref):
+class _BaseItem(object_ref):
+    """
+    Temporary class used internally to avoid the deprecation
+    warning raised by isinstance checks using BaseItem.
+    """
+    pass
+
+
+class _BaseItemMeta(ABCMeta):
+    def __instancecheck__(cls, instance):
+        if cls is BaseItem:
+            warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
+                 ScrapyDeprecationWarning, stacklevel=2)
+        return super().__instancecheck__(instance)
+
+
+class BaseItem(_BaseItem, metaclass=_BaseItemMeta):
     """
     Deprecated, please use :class:`scrapy.item.Item` instead
     """
@@ -30,7 +46,7 @@ class Field(dict):
     """Container of field metadata"""
 
 
-class ItemMeta(ABCMeta):
+class ItemMeta(_BaseItemMeta):
     """Metaclass_ of :class:`Item` that handles field definitions.
 
     .. _metaclass: https://realpython.com/python-metaclasses
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 08ce89481..83afb74c9 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -13,7 +13,7 @@ from w3lib.url import any_to_uri
 from scrapy.crawler import Crawler
 from scrapy.exceptions import IgnoreRequest
 from scrapy.http import Request, Response
-from scrapy.item import BaseItem
+from scrapy.item import _BaseItem
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
 from scrapy.utils.console import start_python_console
@@ -26,8 +26,7 @@ from scrapy.utils.console import DEFAULT_PYTHON_SHELLS
 
 class Shell:
 
-    relevant_classes = (Crawler, Spider, Request, Response, BaseItem,
-                        Settings)
+    relevant_classes = (Crawler, Spider, Request, Response, _BaseItem, Settings)
 
     def __init__(self, crawler, update_vars=None, code=None):
         self.crawler = crawler
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 52cfba208..bfe3ccd40 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -14,10 +14,10 @@ from w3lib.html import replace_entities
 
 from scrapy.utils.datatypes import LocalWeakReferencedCache
 from scrapy.utils.python import flatten, to_unicode
-from scrapy.item import BaseItem
+from scrapy.item import _BaseItem
 
 
-_ITERABLE_SINGLE_VALUES = dict, BaseItem, str, bytes
+_ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes
 
 
 def arg_to_iter(arg):
diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py
index 9dd72ea71..bf73dfa18 100644
--- a/scrapy/utils/serialize.py
+++ b/scrapy/utils/serialize.py
@@ -5,7 +5,7 @@ import decimal
 from twisted.internet import defer
 
 from scrapy.http import Request, Response
-from scrapy.item import BaseItem
+from scrapy.item import _BaseItem
 
 
 class ScrapyJSONEncoder(json.JSONEncoder):
@@ -26,7 +26,7 @@ class ScrapyJSONEncoder(json.JSONEncoder):
             return str(o)
         elif isinstance(o, defer.Deferred):
             return str(o)
-        elif isinstance(o, BaseItem):
+        elif isinstance(o, _BaseItem):
             return dict(o)
         elif isinstance(o, Request):
             return "<%s %s %s>" % (type(o).__name__, o.method, o.url)
diff --git a/tests/test_item.py b/tests/test_item.py
index f35a2b9f9..6fdd7e302 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -4,7 +4,7 @@ from unittest import mock
 from warnings import catch_warnings
 
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.item import ABCMeta, BaseItem, DictItem, Field, Item, ItemMeta
+from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta
 
 
 PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6)
@@ -334,29 +334,73 @@ class DictItemTest(unittest.TestCase):
 
 class BaseItemTest(unittest.TestCase):
 
+    def test_isinstance_check(self):
+
+        class SubclassedBaseItem(BaseItem):
+            pass
+
+        class SubclassedItem(Item):
+            pass
+
+        self.assertTrue(isinstance(BaseItem(), BaseItem))
+        self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem))
+        self.assertTrue(isinstance(Item(), BaseItem))
+        self.assertTrue(isinstance(SubclassedItem(), BaseItem))
+
+        # make sure internal checks using private _BaseItem class succeed
+        self.assertTrue(isinstance(BaseItem(), _BaseItem))
+        self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem))
+        self.assertTrue(isinstance(Item(), _BaseItem))
+        self.assertTrue(isinstance(SubclassedItem(), _BaseItem))
+
     def test_deprecation_warning(self):
+        """
+        Make sure deprecation warnings are logged whenever BaseItem is used,
+        either instantiated or in an isinstance check
+        """
         with catch_warnings(record=True) as warnings:
             BaseItem()
             self.assertEqual(len(warnings), 1)
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+
         with catch_warnings(record=True) as warnings:
+
             class SubclassedBaseItem(BaseItem):
                 pass
+
             SubclassedBaseItem()
             self.assertEqual(len(warnings), 1)
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
 
+        with catch_warnings(record=True) as warnings:
+            self.assertFalse(isinstance("foo", BaseItem))
+            self.assertEqual(len(warnings), 1)
+            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+
+        with catch_warnings(record=True) as warnings:
+            self.assertTrue(isinstance(BaseItem(), BaseItem))
+            self.assertEqual(len(warnings), 1)
+            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+
 
 class ItemNoDeprecationWarningTest(unittest.TestCase):
-
     def test_no_deprecation_warning(self):
+        """
+        Make sure deprecation warnings are NOT logged whenever BaseItem subclasses are used.
+        """
+        class SubclassedItem(Item):
+            pass
+
         with catch_warnings(record=True) as warnings:
             Item()
-            self.assertEqual(len(warnings), 0)
-        with catch_warnings(record=True) as warnings:
-            class SubclassedItem(Item):
-                pass
             SubclassedItem()
+            _BaseItem()
+            self.assertFalse(isinstance("foo", _BaseItem))
+            self.assertFalse(isinstance("foo", Item))
+            self.assertFalse(isinstance("foo", SubclassedItem))
+            self.assertTrue(isinstance(_BaseItem(), _BaseItem))
+            self.assertTrue(isinstance(Item(), Item))
+            self.assertTrue(isinstance(SubclassedItem(), SubclassedItem))
             self.assertEqual(len(warnings), 0)
 
 

From dcf7235f0e44c0199360cea9472b7a0da6bab1a7 Mon Sep 17 00:00:00 2001
From: Jay Rajput <jay24rajput@gmail.com>
Date: Fri, 8 May 2020 01:48:23 +0530
Subject: [PATCH 111/243] Code sharing between crawl and runspider command

---
 scrapy/commands/common_commands.py | 29 +++++++++++++++++++++++++++++
 scrapy/commands/crawl.py           | 27 ++++-----------------------
 scrapy/commands/runspider.py       | 24 ++----------------------
 3 files changed, 35 insertions(+), 45 deletions(-)
 create mode 100644 scrapy/commands/common_commands.py

diff --git a/scrapy/commands/common_commands.py b/scrapy/commands/common_commands.py
new file mode 100644
index 000000000..7da7494ac
--- /dev/null
+++ b/scrapy/commands/common_commands.py
@@ -0,0 +1,29 @@
+from scrapy.commands import ScrapyCommand
+from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
+from scrapy.exceptions import UsageError
+
+
+class CommonCommands(ScrapyCommand):
+
+    def add_options(self, parser):
+        ScrapyCommand.add_options(self, parser)
+        parser.add_option("-a", dest="spargs", action="append", default=[],
+                          metavar="NAME=VALUE",
+                          help="set spider argument (may be repeated)")
+        parser.add_option("-o", "--output", metavar="FILE", action="append",
+                          help="dump scraped items into FILE"
+                          + "(use - for stdout)")
+        parser.add_option("-t", "--output-format", metavar="FORMAT",
+                          help="format to use for dumping items with -o")
+
+    def process_options(self, args, opts):
+        ScrapyCommand.process_options(self, args, opts)
+        try:
+            opts.spargs = arglist_to_dict(opts.spargs)
+        except ValueError:
+            raise UsageError(
+                "Invalid -a value, use -a NAME=VALUE", print_help=False)
+        if opts.output:
+            feeds = feed_process_params_from_cli(
+                self.settings, opts.output, opts.output_format)
+            self.settings.set('FEEDS', feeds, priority='cmdline')
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index 4b2f9484b..b477d7c71 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -1,9 +1,8 @@
-from scrapy.commands import ScrapyCommand
-from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
 from scrapy.exceptions import UsageError
+from scrapy.commands.common_commands import CommonCommands
 
 
-class Command(ScrapyCommand):
+class Command(CommonCommands):
 
     requires_project = True
 
@@ -13,30 +12,12 @@ class Command(ScrapyCommand):
     def short_desc(self):
         return "Run a spider"
 
-    def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
-        parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
-                          help="set spider argument (may be repeated)")
-        parser.add_option("-o", "--output", metavar="FILE", action="append",
-                          help="dump scraped items into FILE (use - for stdout)")
-        parser.add_option("-t", "--output-format", metavar="FORMAT",
-                          help="format to use for dumping items with -o")
-
-    def process_options(self, args, opts):
-        ScrapyCommand.process_options(self, args, opts)
-        try:
-            opts.spargs = arglist_to_dict(opts.spargs)
-        except ValueError:
-            raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
-        if opts.output:
-            feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
-            self.settings.set('FEEDS', feeds, priority='cmdline')
-
     def run(self, args, opts):
         if len(args) < 1:
             raise UsageError()
         elif len(args) > 1:
-            raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported")
+            raise UsageError(
+                "running 'scrapy crawl' with more than one spider is no longer supported")
         spname = args[0]
 
         crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 62510609a..9959f6b0d 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -3,9 +3,8 @@ import os
 from importlib import import_module
 
 from scrapy.utils.spider import iter_spider_classes
-from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
-from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
+from scrapy.commands.common_commands import CommonCommands
 
 
 def _import_file(filepath):
@@ -24,7 +23,7 @@ def _import_file(filepath):
     return module
 
 
-class Command(ScrapyCommand):
+class Command(CommonCommands):
 
     requires_project = False
     default_settings = {'SPIDER_LOADER_WARN_ONLY': True}
@@ -38,25 +37,6 @@ class Command(ScrapyCommand):
     def long_desc(self):
         return "Run the spider defined in the given file"
 
-    def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
-        parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
-                          help="set spider argument (may be repeated)")
-        parser.add_option("-o", "--output", metavar="FILE", action="append",
-                          help="dump scraped items into FILE (use - for stdout)")
-        parser.add_option("-t", "--output-format", metavar="FORMAT",
-                          help="format to use for dumping items with -o")
-
-    def process_options(self, args, opts):
-        ScrapyCommand.process_options(self, args, opts)
-        try:
-            opts.spargs = arglist_to_dict(opts.spargs)
-        except ValueError:
-            raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
-        if opts.output:
-            feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
-            self.settings.set('FEEDS', feeds, priority='cmdline')
-
     def run(self, args, opts):
         if len(args) != 1:
             raise UsageError()

From cf09af787eafa6770bc5ab00bb0ee9759c75df23 Mon Sep 17 00:00:00 2001
From: Antonio Gordillo Toledo <antonio.gor@hotmail.com>
Date: Fri, 8 May 2020 06:45:19 -0700
Subject: [PATCH 112/243] Remove Python 2 encoding header from files (#4553)

---
 docs/conf.py                                        | 2 --
 scrapy/downloadermiddlewares/ajaxcrawl.py           | 1 -
 scrapy/spiderloader.py                              | 1 -
 scrapy/templates/project/module/items.py.tmpl       | 2 --
 scrapy/templates/project/module/middlewares.py.tmpl | 2 --
 scrapy/templates/project/module/pipelines.py.tmpl   | 2 --
 scrapy/templates/project/module/settings.py.tmpl    | 2 --
 scrapy/templates/spiders/basic.tmpl                 | 1 -
 scrapy/templates/spiders/crawl.tmpl                 | 1 -
 scrapy/templates/spiders/csvfeed.tmpl               | 1 -
 scrapy/templates/spiders/xmlfeed.tmpl               | 1 -
 scrapy/utils/log.py                                 | 2 --
 scrapy/utils/ssl.py                                 | 2 --
 tests/test_downloadermiddleware_redirect.py         | 2 --
 tests/test_downloadermiddleware_robotstxt.py        | 1 -
 tests/test_http_response.py                         | 1 -
 tests/test_pipeline_crawl.py                        | 1 -
 tests/test_responsetypes.py                         | 1 -
 tests/test_utils_deprecate.py                       | 1 -
 tests/test_utils_iterators.py                       | 1 -
 tests/test_utils_log.py                             | 1 -
 tests/test_utils_url.py                             | 1 -
 22 files changed, 30 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 813417bae..8ab38a090 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-#
 # Scrapy documentation build configuration file, created by
 # sphinx-quickstart on Mon Nov 24 12:02:52 2008.
 #
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index ad7a81e6b..4e12a5044 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import re
 import logging
 
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 63da55718..db4193430 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import traceback
 import warnings
 from collections import defaultdict
diff --git a/scrapy/templates/project/module/items.py.tmpl b/scrapy/templates/project/module/items.py.tmpl
index a12d08414..88a18331c 100644
--- a/scrapy/templates/project/module/items.py.tmpl
+++ b/scrapy/templates/project/module/items.py.tmpl
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 # Define here the models for your scraped items
 #
 # See documentation in:
diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl
index b3e58ff94..6490f52a7 100644
--- a/scrapy/templates/project/module/middlewares.py.tmpl
+++ b/scrapy/templates/project/module/middlewares.py.tmpl
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 # Define here the models for your spider middleware
 #
 # See documentation in:
diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl
index 4876526a9..ce0edd335 100644
--- a/scrapy/templates/project/module/pipelines.py.tmpl
+++ b/scrapy/templates/project/module/pipelines.py.tmpl
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 # Define your item pipelines here
 #
 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index cb220eafc..a414b5fde 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 # Scrapy settings for $project_name project
 #
 # For simplicity, this file contains only settings considered important or
diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl
index 1cfe9cc9d..e9112bc95 100644
--- a/scrapy/templates/spiders/basic.tmpl
+++ b/scrapy/templates/spiders/basic.tmpl
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import scrapy
 
 
diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl
index 878425125..356496487 100644
--- a/scrapy/templates/spiders/crawl.tmpl
+++ b/scrapy/templates/spiders/crawl.tmpl
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import scrapy
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import CrawlSpider, Rule
diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl
index c2e4bacfe..cbcbe9e2c 100644
--- a/scrapy/templates/spiders/csvfeed.tmpl
+++ b/scrapy/templates/spiders/csvfeed.tmpl
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from scrapy.spiders import CSVFeedSpider
 
 
diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl
index 863c9772f..5aa2aa8b0 100644
--- a/scrapy/templates/spiders/xmlfeed.tmpl
+++ b/scrapy/templates/spiders/xmlfeed.tmpl
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from scrapy.spiders import XMLFeedSpider
 
 
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 5998dc33b..203842fc8 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 import logging
 import sys
 import warnings
diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py
index 6e81b33ff..c3c5e329b 100644
--- a/scrapy/utils/ssl.py
+++ b/scrapy/utils/ssl.py
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 import OpenSSL
 import OpenSSL._util as pyOpenSSLutil
 
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 551e124ab..61c9eddbc 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 import unittest
 
 from scrapy.downloadermiddlewares.redirect import RedirectMiddleware, MetaRefreshMiddleware
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index a1645ed96..b9452a0e7 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from unittest import mock
 
 from twisted.internet import reactor, error
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 522ec4875..43d6d936a 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 
 from w3lib.encoding import resolve_encoding
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 24c516473..e2578a9c9 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import os
 import shutil
 
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 8cdf7a176..9e63ac924 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 from scrapy.responsetypes import responsetypes
 
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index b17e17f2f..adef66c1d 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import inspect
 import unittest
 from unittest import mock
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 46aaaecbc..69339256e 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import os
 
 from twisted.trial import unittest
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 21100aeb8..25cd904bc 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import sys
 import logging
 import unittest
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 1f8388957..16e7449c9 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 
 from scrapy.spiders import Spider

From b852fff6f82e24c535c0dd9b4ef7fa78e7946497 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 8 May 2020 15:19:22 -0300
Subject: [PATCH 113/243] Style changes in link extractor

---
 scrapy/linkextractors/lxmlhtml.py | 52 ++++++++++++++++++++++---------
 1 file changed, 37 insertions(+), 15 deletions(-)

diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index ceb37c5f1..9ebf8e7c7 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -28,8 +28,9 @@ def _nons(tag):
 
 
 class LxmlParserLinkExtractor:
-    def __init__(self, tag="a", attr="href", process=None, unique=False,
-                 strip=True, canonicalized=False):
+    def __init__(
+        self, tag="a", attr="href", process=None, unique=False, strip=True, canonicalized=False
+    ):
         self.scan_tag = tag if callable(tag) else lambda t: t == tag
         self.scan_attr = attr if callable(attr) else lambda a: a == attr
         self.process_attr = process if callable(process) else lambda v: v
@@ -93,10 +94,23 @@ class LxmlParserLinkExtractor:
 
 class LxmlLinkExtractor(FilteringLinkExtractor):
 
-    def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
-                 tags=('a', 'area'), attrs=('href',), canonicalize=False,
-                 unique=True, process_value=None, deny_extensions=None, restrict_css=(),
-                 strip=True, restrict_text=None):
+    def __init__(
+        self,
+        allow=(),
+        deny=(),
+        allow_domains=(),
+        deny_domains=(),
+        restrict_xpaths=(),
+        tags=('a', 'area'),
+        attrs=('href',),
+        canonicalize=False,
+        unique=True,
+        process_value=None,
+        deny_extensions=None,
+        restrict_css=(),
+        strip=True,
+        restrict_text=None,
+    ):
         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
         lx = LxmlParserLinkExtractor(
             tag=lambda x: x in tags,
@@ -106,12 +120,18 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
             strip=strip,
             canonicalized=canonicalize
         )
-
-        super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
-                                                allow_domains=allow_domains, deny_domains=deny_domains,
-                                                restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
-                                                canonicalize=canonicalize, deny_extensions=deny_extensions,
-                                                restrict_text=restrict_text)
+        super(LxmlLinkExtractor, self).__init__(
+            link_extractor=lx,
+            allow=allow,
+            deny=deny,
+            allow_domains=allow_domains,
+            deny_domains=deny_domains,
+            restrict_xpaths=restrict_xpaths,
+            restrict_css=restrict_css,
+            canonicalize=canonicalize,
+            deny_extensions=deny_extensions,
+            restrict_text=restrict_text,
+        )
 
     def extract_links(self, response):
         """Returns a list of :class:`~scrapy.link.Link` objects from the
@@ -124,9 +144,11 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
         """
         base_url = get_base_url(response)
         if self.restrict_xpaths:
-            docs = [subdoc
-                    for x in self.restrict_xpaths
-                    for subdoc in response.xpath(x)]
+            docs = [
+                subdoc
+                for x in self.restrict_xpaths
+                for subdoc in response.xpath(x)
+            ]
         else:
             docs = [response.selector]
         all_links = []

From 3ebf2a0d82b0ae5b9e701972ba2a9e1420c6d2c7 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 8 May 2020 15:17:33 -0300
Subject: [PATCH 114/243] Remove lambdas in link extractor

---
 scrapy/linkextractors/lxmlhtml.py | 28 +++++++++++++++++-----------
 tests/test_linkextractors.py      |  5 +++++
 2 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 9ebf8e7c7..1615d44d7 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -1,6 +1,8 @@
 """
 Link extractor based on lxml.html
 """
+import operator
+from functools import partial
 from urllib.parse import urljoin
 
 import lxml.etree as etree
@@ -8,10 +10,10 @@ from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url, safe_url_string
 
 from scrapy.link import Link
+from scrapy.linkextractors import FilteringLinkExtractor
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
 from scrapy.utils.python import unique as unique_list
 from scrapy.utils.response import get_base_url
-from scrapy.linkextractors import FilteringLinkExtractor
 
 
 # from lxml/src/lxml/html/__init__.py
@@ -27,20 +29,24 @@ def _nons(tag):
     return tag
 
 
+def _identity(x):
+    return x
+
+
+def _canonicalize_link_url(link):
+    return canonicalize_url(link.url, keep_fragments=True)
+
+
 class LxmlParserLinkExtractor:
     def __init__(
         self, tag="a", attr="href", process=None, unique=False, strip=True, canonicalized=False
     ):
-        self.scan_tag = tag if callable(tag) else lambda t: t == tag
-        self.scan_attr = attr if callable(attr) else lambda a: a == attr
-        self.process_attr = process if callable(process) else lambda v: v
+        self.scan_tag = tag if callable(tag) else partial(operator.eq, tag)
+        self.scan_attr = attr if callable(attr) else partial(operator.eq, attr)
+        self.process_attr = process if callable(process) else _identity
         self.unique = unique
         self.strip = strip
-        if canonicalized:
-            self.link_key = lambda link: link.url
-        else:
-            self.link_key = lambda link: canonicalize_url(link.url,
-                                                          keep_fragments=True)
+        self.link_key = operator.attrgetter("url") if canonicalized else _canonicalize_link_url
 
     def _iter_links(self, document):
         for el in document.iter(etree.Element):
@@ -113,8 +119,8 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
     ):
         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
         lx = LxmlParserLinkExtractor(
-            tag=lambda x: x in tags,
-            attr=lambda x: x in attrs,
+            tag=partial(operator.contains, tags),
+            attr=partial(operator.contains, attrs),
             unique=unique,
             process=process_value,
             strip=strip,
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 68e8514ba..46d8c13af 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -1,3 +1,4 @@
+import pickle
 import re
 import unittest
 from warnings import catch_warnings
@@ -462,6 +463,10 @@ class Base:
                 Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False),
             ])
 
+        def test_pickle_extractor(self):
+            lx = self.extractor_cls()
+            self.assertIsInstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls)
+
 
 class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
     extractor_cls = LxmlLinkExtractor

From 81d0b2f61ac119efab4d5970bd235dbc288496ef Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 8 May 2020 16:23:53 -0300
Subject: [PATCH 115/243] Flake8: remove E111

---
 pytest.ini                 | 4 ++--
 scrapy/selector/unified.py | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 604bbfe1d..20639baa3 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -102,7 +102,7 @@ flake8-ignore =
     scrapy/pipelines/media.py E501
     # scrapy/selector
     scrapy/selector/__init__.py F403
-    scrapy/selector/unified.py E501 E111
+    scrapy/selector/unified.py E501
     # scrapy/settings
     scrapy/settings/__init__.py E501
     scrapy/settings/default_settings.py E501 E114 E116
@@ -224,7 +224,7 @@ flake8-ignore =
     tests/test_spider.py E501
     tests/test_spidermiddleware.py E501
     tests/test_spidermiddleware_httperror.py E128 E501 E121
-    tests/test_spidermiddleware_offsite.py E501 E128 E111
+    tests/test_spidermiddleware_offsite.py E501 E128
     tests/test_spidermiddleware_output_chain.py E501
     tests/test_spidermiddleware_referer.py E501 F841 E501 E121
     tests/test_squeues.py E501 E741
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index a08955dc9..85a9bb526 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -65,9 +65,9 @@ class Selector(_ParselSelector, object_ref):
     selectorlist_cls = SelectorList
 
     def __init__(self, response=None, text=None, type=None, root=None, **kwargs):
-        if not(response is None or text is None):
-           raise ValueError('%s.__init__() received both response and text'
-                            % self.__class__.__name__)
+        if response is not None and text is not None:
+            raise ValueError('%s.__init__() received both response and text'
+                             % self.__class__.__name__)
 
         st = _st(response, type or self._default_type)
 

From 1a157f2e26274455405b7e272bab5aede0fa59fa Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 8 May 2020 16:27:21 -0300
Subject: [PATCH 116/243] Flake8: remove E116

---
 pytest.ini                | 4 ++--
 scrapy/pipelines/files.py | 3 +--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 20639baa3..295609435 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -97,7 +97,7 @@ flake8-ignore =
     scrapy/loader/processors.py E501
     # scrapy/pipelines
     scrapy/pipelines/__init__.py E501
-    scrapy/pipelines/files.py E116 E501
+    scrapy/pipelines/files.py E501
     scrapy/pipelines/images.py E501
     scrapy/pipelines/media.py E501
     # scrapy/selector
@@ -105,7 +105,7 @@ flake8-ignore =
     scrapy/selector/unified.py E501
     # scrapy/settings
     scrapy/settings/__init__.py E501
-    scrapy/settings/default_settings.py E501 E114 E116
+    scrapy/settings/default_settings.py E501 E114
     scrapy/settings/deprecated.py E501
     # scrapy/spidermiddlewares
     scrapy/spidermiddlewares/httperror.py E501
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index a9066986b..cd3e29057 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -83,8 +83,7 @@ class S3FilesStore:
     AWS_USE_SSL = None
     AWS_VERIFY = None
 
-    POLICY = 'private'  # Overriden from settings.FILES_STORE_S3_ACL in
-                        # FilesPipeline.from_settings.
+    POLICY = 'private'  # Overriden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings
     HEADERS = {
         'Cache-Control': 'max-age=172800',
     }

From 83ce82f400d9686f7fbaa526f21f635fca8491de Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 8 May 2020 16:28:26 -0300
Subject: [PATCH 117/243] Flake8: remove E114 and E117 (unused)

---
 pytest.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 295609435..59cce9ac4 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -105,7 +105,7 @@ flake8-ignore =
     scrapy/selector/unified.py E501
     # scrapy/settings
     scrapy/settings/__init__.py E501
-    scrapy/settings/default_settings.py E501 E114
+    scrapy/settings/default_settings.py E501
     scrapy/settings/deprecated.py E501
     # scrapy/spidermiddlewares
     scrapy/spidermiddlewares/httperror.py E501
@@ -207,7 +207,7 @@ flake8-ignore =
     tests/test_item.py E128 F841
     tests/test_link.py E501
     tests/test_linkextractors.py E501 E128
-    tests/test_loader.py E501 E741 E128 E117
+    tests/test_loader.py E501 E741 E128
     tests/test_logformatter.py E128 E501
     tests/test_mail.py E128 E501
     tests/test_middleware.py E501 E128

From c2c3054ac13838b89ad5062d6583cc52dd0ba317 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 8 May 2020 16:32:02 -0300
Subject: [PATCH 118/243] Flake8: remove E121

---
 pytest.ini                               | 4 ++--
 tests/test_spidermiddleware_httperror.py | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 59cce9ac4..61139f7fe 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -223,10 +223,10 @@ flake8-ignore =
     tests/test_selector.py E501
     tests/test_spider.py E501
     tests/test_spidermiddleware.py E501
-    tests/test_spidermiddleware_httperror.py E128 E501 E121
+    tests/test_spidermiddleware_httperror.py E128 E501
     tests/test_spidermiddleware_offsite.py E501 E128
     tests/test_spidermiddleware_output_chain.py E501
-    tests/test_spidermiddleware_referer.py E501 F841 E501 E121
+    tests/test_spidermiddleware_referer.py E501 F841 E501
     tests/test_squeues.py E501 E741
     tests/test_utils_asyncio.py E501
     tests/test_utils_conf.py E501 E128
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index 6b61df56f..29584f21b 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -21,10 +21,10 @@ class _HttpErrorSpider(MockServerSpider):
     def __init__(self, *args, **kwargs):
         super(_HttpErrorSpider, self).__init__(*args, **kwargs)
         self.start_urls = [
-           self.mockserver.url("/status?n=200"),
-           self.mockserver.url("/status?n=404"),
-           self.mockserver.url("/status?n=402"),
-           self.mockserver.url("/status?n=500"),
+            self.mockserver.url("/status?n=200"),
+            self.mockserver.url("/status?n=404"),
+            self.mockserver.url("/status?n=402"),
+            self.mockserver.url("/status?n=500"),
         ]
         self.failed = set()
         self.skipped = set()

From f689e917bf2caa2c48def7399317dd019264663d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 8 May 2020 19:44:05 -0300
Subject: [PATCH 119/243] Update Item docstring

---
 scrapy/item.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/scrapy/item.py b/scrapy/item.py
index f468ff86f..b9a0acd3d 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -79,8 +79,7 @@ class DictItem(MutableMapping, BaseItem):
 
     def __new__(cls, *args, **kwargs):
         if issubclass(cls, DictItem) and not issubclass(cls, Item):
-            warn('scrapy.item.DictItem is deprecated, please use '
-                 'scrapy.item.Item instead',
+            warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead',
                  ScrapyDeprecationWarning, stacklevel=2)
         return super(DictItem, cls).__new__(cls, *args, **kwargs)
 
@@ -147,6 +146,11 @@ class Item(DictItem, metaclass=ItemMeta):
     If you need instances of a custom class to be considered items by Scrapy,
     you must inherit from either :class:`Item` or :class:`dict`.
 
+    Items offer the ability to declare :class:`Field` attributes, which can be
+    used to define metadata and control the way data is processed internally.
+    Please refer to the :ref:`documentation about fields <topics-items-fields>`
+    for additional information.
+
     Unlike instances of :class:`dict`, instances of :class:`Item` may be
     :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
     """

From 92792cc3f7c66a81a9e7a0576be0848266950c7e Mon Sep 17 00:00:00 2001
From: Jay Rajput <jay24rajput@gmail.com>
Date: Sat, 9 May 2020 17:28:10 +0530
Subject: [PATCH 120/243] Moved common_commands.py to __init__.py

---
 scrapy/commands/__init__.py        | 52 ++++++++++++++++++++++++------
 scrapy/commands/common_commands.py | 29 -----------------
 scrapy/commands/crawl.py           |  4 +--
 scrapy/commands/runspider.py       |  4 +--
 4 files changed, 47 insertions(+), 42 deletions(-)
 delete mode 100644 scrapy/commands/common_commands.py

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 9f8e6986a..b95d395a1 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -5,7 +5,7 @@ import os
 from optparse import OptionGroup
 from twisted.python import failure
 
-from scrapy.utils.conf import arglist_to_dict
+from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
 from scrapy.exceptions import UsageError
 
 
@@ -59,18 +59,19 @@ class ScrapyCommand:
         """
         group = OptionGroup(parser, "Global Options")
         group.add_option("--logfile", metavar="FILE",
-            help="log file. if omitted stderr will be used")
+                         help="log file. if omitted stderr will be used")
         group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
-            help="log level (default: %s)" % self.settings['LOG_LEVEL'])
+                         help="log level (default: %s)" % self.settings['LOG_LEVEL'])
         group.add_option("--nolog", action="store_true",
-            help="disable logging completely")
+                         help="disable logging completely")
         group.add_option("--profile", metavar="FILE", default=None,
-            help="write python cProfile stats to FILE")
+                         help="write python cProfile stats to FILE")
         group.add_option("--pidfile", metavar="FILE",
-            help="write process ID to FILE")
+                         help="write process ID to FILE")
         group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
-            help="set/override setting (may be repeated)")
-        group.add_option("--pdb", action="store_true", help="enable pdb on failure")
+                         help="set/override setting (may be repeated)")
+        group.add_option("--pdb", action="store_true",
+                         help="enable pdb on failure")
 
         parser.add_option_group(group)
 
@@ -79,7 +80,8 @@ class ScrapyCommand:
             self.settings.setdict(arglist_to_dict(opts.set),
                                   priority='cmdline')
         except ValueError:
-            raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False)
+            raise UsageError(
+                "Invalid -s value, use -s NAME=VALUE", print_help=False)
 
         if opts.logfile:
             self.settings.set('LOG_ENABLED', True, priority='cmdline')
@@ -104,3 +106,35 @@ class ScrapyCommand:
         Entry point for running commands
         """
         raise NotImplementedError
+
+
+''''
+The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for
+performing common functionality between crawl.py and runspider.py
+'''
+
+
+class BaseRunSpiderCommands(ScrapyCommand):
+
+    def add_options(self, parser):
+        ScrapyCommand.add_options(self, parser)
+        parser.add_option("-a", dest="spargs", action="append", default=[],
+                          metavar="NAME=VALUE",
+                          help="set spider argument (may be repeated)")
+        parser.add_option("-o", "--output", metavar="FILE", action="append",
+                          help="dump scraped items into FILE"
+                          + "(use - for stdout)")
+        parser.add_option("-t", "--output-format", metavar="FORMAT",
+                          help="format to use for dumping items with -o")
+
+    def process_options(self, args, opts):
+        ScrapyCommand.process_options(self, args, opts)
+        try:
+            opts.spargs = arglist_to_dict(opts.spargs)
+        except ValueError:
+            raise UsageError(
+                "Invalid -a value, use -a NAME=VALUE", print_help=False)
+        if opts.output:
+            feeds = feed_process_params_from_cli(
+                self.settings, opts.output, opts.output_format)
+            self.settings.set('FEEDS', feeds, priority='cmdline')
diff --git a/scrapy/commands/common_commands.py b/scrapy/commands/common_commands.py
deleted file mode 100644
index 7da7494ac..000000000
--- a/scrapy/commands/common_commands.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from scrapy.commands import ScrapyCommand
-from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
-from scrapy.exceptions import UsageError
-
-
-class CommonCommands(ScrapyCommand):
-
-    def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
-        parser.add_option("-a", dest="spargs", action="append", default=[],
-                          metavar="NAME=VALUE",
-                          help="set spider argument (may be repeated)")
-        parser.add_option("-o", "--output", metavar="FILE", action="append",
-                          help="dump scraped items into FILE"
-                          + "(use - for stdout)")
-        parser.add_option("-t", "--output-format", metavar="FORMAT",
-                          help="format to use for dumping items with -o")
-
-    def process_options(self, args, opts):
-        ScrapyCommand.process_options(self, args, opts)
-        try:
-            opts.spargs = arglist_to_dict(opts.spargs)
-        except ValueError:
-            raise UsageError(
-                "Invalid -a value, use -a NAME=VALUE", print_help=False)
-        if opts.output:
-            feeds = feed_process_params_from_cli(
-                self.settings, opts.output, opts.output_format)
-            self.settings.set('FEEDS', feeds, priority='cmdline')
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index b477d7c71..ca19b1367 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -1,8 +1,8 @@
 from scrapy.exceptions import UsageError
-from scrapy.commands.common_commands import CommonCommands
+from scrapy.commands import BaseRunSpiderCommands
 
 
-class Command(CommonCommands):
+class Command(BaseRunSpiderCommands):
 
     requires_project = True
 
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 9959f6b0d..bd24a369e 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -4,7 +4,7 @@ from importlib import import_module
 
 from scrapy.utils.spider import iter_spider_classes
 from scrapy.exceptions import UsageError
-from scrapy.commands.common_commands import CommonCommands
+from scrapy.commands import BaseRunSpiderCommands
 
 
 def _import_file(filepath):
@@ -23,7 +23,7 @@ def _import_file(filepath):
     return module
 
 
-class Command(CommonCommands):
+class Command(BaseRunSpiderCommands):
 
     requires_project = False
     default_settings = {'SPIDER_LOADER_WARN_ONLY': True}

From ed4f4f84082f7023734d1200d88e8f1aec57904c Mon Sep 17 00:00:00 2001
From: Jay Rajput <jay24rajput@gmail.com>
Date: Sun, 10 May 2020 00:08:34 +0530
Subject: [PATCH 121/243] Applied suggested format changes

---
 scrapy/commands/__init__.py | 22 ++++++++--------------
 scrapy/commands/crawl.py    |  5 ++---
 2 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index b95d395a1..1dada1ceb 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -80,8 +80,7 @@ class ScrapyCommand:
             self.settings.setdict(arglist_to_dict(opts.set),
                                   priority='cmdline')
         except ValueError:
-            raise UsageError(
-                "Invalid -s value, use -s NAME=VALUE", print_help=False)
+            raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False)
 
         if opts.logfile:
             self.settings.set('LOG_ENABLED', True, priority='cmdline')
@@ -108,18 +107,15 @@ class ScrapyCommand:
         raise NotImplementedError
 
 
-''''
-The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for
-performing common functionality between crawl.py and runspider.py
-'''
-
-
 class BaseRunSpiderCommands(ScrapyCommand):
+    ''''
+    The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for
+    performing common functionality between crawl.py and runspider.py
+    '''
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_option("-a", dest="spargs", action="append", default=[],
-                          metavar="NAME=VALUE",
+        parser.add_option("-a", dest="spargs", action="append", default=[],metavar="NAME=VALUE",
                           help="set spider argument (may be repeated)")
         parser.add_option("-o", "--output", metavar="FILE", action="append",
                           help="dump scraped items into FILE"
@@ -132,9 +128,7 @@ class BaseRunSpiderCommands(ScrapyCommand):
         try:
             opts.spargs = arglist_to_dict(opts.spargs)
         except ValueError:
-            raise UsageError(
-                "Invalid -a value, use -a NAME=VALUE", print_help=False)
+            raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
         if opts.output:
-            feeds = feed_process_params_from_cli(
-                self.settings, opts.output, opts.output_format)
+            feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
             self.settings.set('FEEDS', feeds, priority='cmdline')
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index ca19b1367..c39c5a9ef 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -1,5 +1,5 @@
-from scrapy.exceptions import UsageError
 from scrapy.commands import BaseRunSpiderCommands
+from scrapy.exceptions import UsageError
 
 
 class Command(BaseRunSpiderCommands):
@@ -16,8 +16,7 @@ class Command(BaseRunSpiderCommands):
         if len(args) < 1:
             raise UsageError()
         elif len(args) > 1:
-            raise UsageError(
-                "running 'scrapy crawl' with more than one spider is no longer supported")
+            raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported")
         spname = args[0]
 
         crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)

From 02ac6664a5c3510104b67f7cfc9a71c831d5bd5f Mon Sep 17 00:00:00 2001
From: Jay Rajput <jay24rajput@gmail.com>
Date: Sun, 10 May 2020 00:26:48 +0530
Subject: [PATCH 122/243] Travis CI fixes

---
 scrapy/commands/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 1dada1ceb..38b1b77bd 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -115,7 +115,7 @@ class BaseRunSpiderCommands(ScrapyCommand):
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_option("-a", dest="spargs", action="append", default=[],metavar="NAME=VALUE",
+        parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
                           help="set spider argument (may be repeated)")
         parser.add_option("-o", "--output", metavar="FILE", action="append",
                           help="dump scraped items into FILE"

From 1bfbcc618090d4e71e06471b78aaac3071a51956 Mon Sep 17 00:00:00 2001
From: Will Beaufoy <will@willbeaufoy.net>
Date: Sun, 10 May 2020 13:48:09 +0100
Subject: [PATCH 123/243] Extend hoverxref_roles to custom crossrefs (#4495)

---
 docs/conf.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/conf.py b/docs/conf.py
index 8ab38a090..3ae709a04 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -300,3 +300,4 @@ hoverxref_role_types = {
     "mod": "tooltip",
     "ref": "tooltip",
 }
+hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']

From b5684909d1cb01ad138a389caa750485b51f79cf Mon Sep 17 00:00:00 2001
From: Jacty <jacty@users.noreply.github.com>
Date: Mon, 11 May 2020 11:18:25 +0800
Subject: [PATCH 124/243] Unnecessary update when value is None

When value is None, it is not necessary to invoke update and run other methods and conditions to make the code complicated there.
---
 scrapy/settings/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index b9a13c018..f28fbfaf9 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -83,7 +83,8 @@ class BaseSettings(MutableMapping):
     def __init__(self, values=None, priority='project'):
         self.frozen = False
         self.attributes = {}
-        self.update(values, priority)
+        if values is not None:
+            self.update(values, priority)
 
     def __getitem__(self, opt_name):
         if opt_name not in self:

From abfdc1b5425997f1aa69b29465c6ab2324f37fd0 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 11 May 2020 13:20:06 -0300
Subject: [PATCH 125/243] Update docstring for Item class

---
 scrapy/item.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/scrapy/item.py b/scrapy/item.py
index b9a0acd3d..97dfed976 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -138,18 +138,20 @@ class Item(DictItem, metaclass=ItemMeta):
     """
     Base class for scraped items.
 
-    In Scrapy, an object is considered an *item* if it is an instance of either
-    :class:`Item` or :class:`dict`. For example, when the output of a
+    In Scrapy, an object is considered an ``item`` if it is an instance of either
+    :class:`Item` or :class:`dict`, or any subclass. For example, when the output of a
     spider callback is evaluated, only instances of :class:`Item` or
     :class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
 
     If you need instances of a custom class to be considered items by Scrapy,
     you must inherit from either :class:`Item` or :class:`dict`.
 
-    Items offer the ability to declare :class:`Field` attributes, which can be
-    used to define metadata and control the way data is processed internally.
-    Please refer to the :ref:`documentation about fields <topics-items-fields>`
-    for additional information.
+    Items must declare :class:`Field` attributes, which are processed and stored
+    in the ``fields`` attribute. This restricts the set of allowed field names
+    and prevents typos, raising ``KeyError`` when referring to undefined fields.
+    Additionally, fields can be used to define metadata and control the way
+    data is processed internally. Please refer to the :ref:`documentation
+    about fields <topics-items-fields>` for additional information.
 
     Unlike instances of :class:`dict`, instances of :class:`Item` may be
     :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.

From 6f8758624c8d3df7472948ce9805601f6037548a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 11 May 2020 13:50:34 -0300
Subject: [PATCH 126/243] Flake8: remove F841

---
 pytest.ini                             | 22 ++++++++++------------
 tests/pipelines.py                     |  2 +-
 tests/test_crawler.py                  |  6 +++---
 tests/test_dependencies.py             |  2 +-
 tests/test_extension_telnet.py         |  2 --
 tests/test_feedexport.py               |  1 -
 tests/test_item.py                     |  8 ++++----
 tests/test_pipeline_images.py          |  2 +-
 tests/test_spidermiddleware_referer.py |  3 +--
 tests/test_utils_defer.py              |  4 ++--
 tests/test_utils_deprecate.py          |  4 ++--
 tests/test_utils_signal.py             |  2 +-
 12 files changed, 26 insertions(+), 32 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 32fd76445..fee54dcbd 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -164,14 +164,13 @@ flake8-ignore =
     scrapy/robotstxt.py E501
     scrapy/shell.py E501
     scrapy/signalmanager.py E501
-    scrapy/spiderloader.py F841 E501
+    scrapy/spiderloader.py E501
     scrapy/squeues.py E128
     scrapy/squeues.py E501
     scrapy/statscollectors.py E501
     # tests
     tests/__init__.py E402 E501
     tests/mockserver.py E501
-    tests/pipelines.py F841
     tests/spiders.py E501
     tests/test_closespider.py E501
     tests/test_command_fetch.py E501
@@ -180,8 +179,8 @@ flake8-ignore =
     tests/test_commands.py E128 E501
     tests/test_contracts.py E501 E128
     tests/test_crawl.py E501 E741
-    tests/test_crawler.py F841 E501
-    tests/test_dependencies.py F841 E501
+    tests/test_crawler.py E501
+    tests/test_dependencies.py E501
     tests/test_downloader_handlers.py E128 E501
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
@@ -199,13 +198,12 @@ flake8-ignore =
     tests/test_dupefilters.py E501 E741 E128
     tests/test_engine.py E501 E128
     tests/test_exporters.py E501 E128
-    tests/test_extension_telnet.py F841
-    tests/test_feedexport.py E501 F841
+    tests/test_feedexport.py E501
     tests/test_http_cookies.py E501
     tests/test_http_headers.py E501
     tests/test_http_request.py E402 E501 E128 E128
     tests/test_http_response.py E501 E128
-    tests/test_item.py E128 F841
+    tests/test_item.py E128
     tests/test_link.py E501
     tests/test_linkextractors.py E501 E128
     tests/test_loader.py E501 E741 E128
@@ -214,7 +212,7 @@ flake8-ignore =
     tests/test_middleware.py E501 E128
     tests/test_pipeline_crawl.py E501 E128
     tests/test_pipeline_files.py E501
-    tests/test_pipeline_images.py F841 E501
+    tests/test_pipeline_images.py E501
     tests/test_pipeline_media.py E501 E741 E128
     tests/test_proxy_connect.py E501 E741
     tests/test_request_cb_kwargs.py E501
@@ -227,14 +225,14 @@ flake8-ignore =
     tests/test_spidermiddleware_httperror.py E128 E501
     tests/test_spidermiddleware_offsite.py E501 E128
     tests/test_spidermiddleware_output_chain.py E501
-    tests/test_spidermiddleware_referer.py E501 F841 E501
+    tests/test_spidermiddleware_referer.py E501 E501
     tests/test_squeues.py E501 E741
     tests/test_utils_asyncio.py E501
     tests/test_utils_conf.py E501 E128
     tests/test_utils_curl.py E501
     tests/test_utils_datatypes.py E402 E501
-    tests/test_utils_defer.py E501 F841
-    tests/test_utils_deprecate.py F841 E501
+    tests/test_utils_defer.py E501
+    tests/test_utils_deprecate.py E501
     tests/test_utils_http.py E501 E128
     tests/test_utils_iterators.py E501 E128
     tests/test_utils_log.py E741
@@ -242,7 +240,7 @@ flake8-ignore =
     tests/test_utils_reqser.py E501 E128
     tests/test_utils_request.py E501 E128
     tests/test_utils_response.py E501
-    tests/test_utils_signal.py E741 F841
+    tests/test_utils_signal.py E741
     tests/test_utils_sitemap.py E128 E501
     tests/test_utils_url.py E501 E501
     tests/test_webclient.py E501 E128 E402
diff --git a/tests/pipelines.py b/tests/pipelines.py
index cf677cc17..fed2af7d3 100644
--- a/tests/pipelines.py
+++ b/tests/pipelines.py
@@ -6,7 +6,7 @@ Some pipelines used for testing
 class ZeroDivisionErrorPipeline:
 
     def open_spider(self, spider):
-        a = 1 / 0
+        1 / 0
 
     def process_item(self, item, spider):
         return item
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 9151278a5..ecc0cd7af 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -87,7 +87,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
         class MySpider(scrapy.Spider):
             name = 'spider'
 
-        crawler = Crawler(MySpider, {})
+        Crawler(MySpider, {})
         assert get_scrapy_root_handler() is None
 
     def test_spider_custom_settings_log_level(self):
@@ -240,13 +240,13 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
     def test_crawler_runner_asyncio_enabled_true(self):
         if self.reactor_pytest == 'asyncio':
-            runner = CrawlerRunner(settings={
+            CrawlerRunner(settings={
                 "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
             })
         else:
             msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
             with self.assertRaisesRegex(Exception, msg):
-                runner = CrawlerRunner(settings={
+                CrawlerRunner(settings={
                     "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
                 })
 
diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py
index a169acbe6..5d0a1d0c9 100644
--- a/tests/test_dependencies.py
+++ b/tests/test_dependencies.py
@@ -6,7 +6,7 @@ class ScrapyUtilsTest(unittest.TestCase):
     def test_required_openssl_version(self):
         try:
             module = import_module('OpenSSL')
-        except ImportError as ex:
+        except ImportError:
             raise unittest.SkipTest("OpenSSL is not available")
 
         if hasattr(module, '__version__'):
diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py
index 873a97248..1e716b94a 100644
--- a/tests/test_extension_telnet.py
+++ b/tests/test_extension_telnet.py
@@ -11,8 +11,6 @@ class TelnetExtensionTest(unittest.TestCase):
     def _get_console_and_portal(self, settings=None):
         crawler = get_crawler(settings_dict=settings)
         console = TelnetConsole(crawler)
-        username = console.username
-        password = console.password
 
         # This function has some side effects we don't need for this test
         console._get_telnet_vars = lambda: {}
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index e02b0b840..cbc81bc35 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -715,7 +715,6 @@ class FeedExportTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_export_encoding(self):
         items = [dict({'foo': u'Test\xd6'})]
-        header = ['foo']
 
         formats = {
             'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
diff --git a/tests/test_item.py b/tests/test_item.py
index 4017f6e84..58735cc5c 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -264,12 +264,12 @@ class ItemTest(unittest.TestCase):
         """Make sure the DictItem deprecation warning is not issued for
         Item"""
         with catch_warnings(record=True) as warnings:
-            item = Item()
+            Item()
             self.assertEqual(len(warnings), 0)
 
             class SubclassedItem(Item):
                 pass
-            subclassed_item = SubclassedItem()
+            SubclassedItem()
             self.assertEqual(len(warnings), 0)
 
 
@@ -321,13 +321,13 @@ class DictItemTest(unittest.TestCase):
 
     def test_deprecation_warning(self):
         with catch_warnings(record=True) as warnings:
-            dict_item = DictItem()
+            DictItem()
             self.assertEqual(len(warnings), 1)
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
         with catch_warnings(record=True) as warnings:
             class SubclassedDictItem(DictItem):
                 pass
-            subclassed_dict_item = SubclassedDictItem()
+            SubclassedDictItem()
             self.assertEqual(len(warnings), 1)
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
 
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 5018d6802..5ba03ff4c 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -15,7 +15,7 @@ from scrapy.utils.python import to_bytes
 skip = False
 try:
     from PIL import Image
-except ImportError as e:
+except ImportError:
     skip = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
 else:
     encoders = set(('jpeg_encoder', 'jpeg_decoder'))
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 41589177a..ca765518b 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -459,7 +459,6 @@ class TestRequestMetaSettingFallback(TestCase):
         target = 'http://www.example.com'
 
         for settings, response_headers, request_meta, policy_class, check_warning in self.params[3:]:
-            spider = Spider('foo')
             mw = RefererMiddleware(Settings(settings))
 
             response = Response(origin, headers=response_headers)
@@ -511,7 +510,7 @@ class TestSettingsPolicyByName(TestCase):
     def test_invalid_name(self):
         settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'})
         with self.assertRaises(RuntimeError):
-            mw = RefererMiddleware(settings)
+            RefererMiddleware(settings)
 
 
 class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index a3b6e64f1..2d4b88121 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -64,7 +64,7 @@ class DeferUtilsTest(unittest.TestCase):
         gotexc = False
         try:
             yield process_chain([cb1, cb_fail, cb3], 'res', 'v1', 'v2')
-        except TypeError as e:
+        except TypeError:
             gotexc = True
         self.assertTrue(gotexc)
 
@@ -104,7 +104,7 @@ class IterErrbackTest(unittest.TestCase):
         def iterbad():
             for x in range(10):
                 if x == 5:
-                    a = 1 / 0
+                    1 / 0
                 yield x
 
         errors = []
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index adef66c1d..35d35b45d 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -25,7 +25,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
 
     def test_no_warning_on_definition(self):
         with warnings.catch_warnings(record=True) as w:
-            Deprecated = create_deprecated_class('Deprecated', NewName)
+            create_deprecated_class('Deprecated', NewName)
 
         w = self._mywarnings(w)
         self.assertEqual(w, [])
@@ -217,7 +217,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
     def test_deprecate_a_class_with_custom_metaclass(self):
         Meta1 = type('Meta1', (type,), {})
         New = Meta1('New', (), {})
-        Deprecated = create_deprecated_class('Deprecated', New)
+        create_deprecated_class('Deprecated', New)
 
     def test_deprecate_subclass_of_deprecated_class(self):
         with warnings.catch_warnings(record=True) as w:
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index bb211dc60..c83c9398c 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -44,7 +44,7 @@ class SendCatchLogTest(unittest.TestCase):
 
     def error_handler(self, arg, handlers_called):
         handlers_called.add(self.error_handler)
-        a = 1 / 0
+        1 / 0
 
     def ok_handler(self, arg, handlers_called):
         handlers_called.add(self.ok_handler)

From cb8140a42a27ede87b0880372024f2f1804618b8 Mon Sep 17 00:00:00 2001
From: nsirletti <40069643+nsirletti@users.noreply.github.com>
Date: Mon, 11 May 2020 20:20:31 +0200
Subject: [PATCH 127/243] Deprecate Response.body_as_unicode() (#4555)

Co-authored-by: Nicolas Sirletti <n.sirletti@gmail.com>
---
 docs/topics/request-response.rst | 5 -----
 scrapy/http/response/text.py     | 5 +++++
 tests/test_http_response.py      | 9 +++++++++
 3 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 024f46466..15a83f453 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -834,11 +834,6 @@ TextResponse objects
 
     .. automethod:: TextResponse.follow_all
 
-    .. method:: TextResponse.body_as_unicode()
-
-        The same as :attr:`text`, but available as a method. This method is
-        kept for backward compatibility; please prefer ``response.text``.
-
 
 HtmlResponse objects
 --------------------
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 2f0f3820c..5614e6e55 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -5,6 +5,7 @@ discovering (through HTTP headers) to base Response class.
 See documentation in docs/topics/request-response.rst
 """
 
+import warnings
 from contextlib import suppress
 from typing import Generator
 from urllib.parse import urljoin
@@ -14,6 +15,7 @@ from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
                             http_content_type_encoding, resolve_encoding)
 from w3lib.html import strip_html5_whitespace
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.http.response import Response
 from scrapy.utils.python import memoizemethod_noargs, to_unicode
@@ -61,6 +63,9 @@ class TextResponse(Response):
 
     def body_as_unicode(self):
         """Return body as unicode"""
+        warnings.warn('Response.body_as_unicode() is deprecated, '
+                      'please use Response.text instead.',
+                      ScrapyDeprecationWarning)
         return self.text
 
     @property
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 43d6d936a..2f73afe56 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,7 +1,9 @@
 import unittest
+from warnings import catch_warnings
 
 from w3lib.encoding import resolve_encoding
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
                          XmlResponse, Headers)
 from scrapy.selector import Selector
@@ -660,6 +662,13 @@ class TextResponseTest(BaseResponseTest):
         with self.assertRaises(ValueError):
             response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]')
 
+    def test_body_as_unicode_deprecation_warning(self):
+        with catch_warnings(record=True) as warnings:
+            r1 = self.response_class("http://www.example.com", body=u'Hello', encoding='utf-8')
+            self.assertEqual(r1.body_as_unicode(), u'Hello')
+            self.assertEqual(len(warnings), 1)
+            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+
 
 class HtmlResponseTest(TextResponseTest):
 

From cf9be5344a89dd8e14f8241ec69de9c984ec1e05 Mon Sep 17 00:00:00 2001
From: willbeaufoy <will@willbeaufoy.net>
Date: Mon, 11 May 2020 19:35:25 +0100
Subject: [PATCH 128/243] Prevent create_instance() returning None (#4532)

Currently create_instance() can return None if an extension is
incorrectly implemented, but the extension will still show up as
enabled in the logs. This can cause confusion, as in the linked bug.

This change prevents this occurring by throwing an error if
create_instance() will return None.
---
 scrapy/utils/misc.py              | 15 ++++++++++++---
 tests/test_utils_misc/__init__.py | 14 +++++++++++---
 2 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 52cfba208..ab7cf9deb 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -137,17 +137,26 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
     ``*args`` and ``**kwargs`` are forwarded to the constructors.
 
     Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
+
+    Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
+    extension has not been implemented correctly).
     """
     if settings is None:
         if crawler is None:
             raise ValueError("Specify at least one of settings and crawler.")
         settings = crawler.settings
     if crawler and hasattr(objcls, 'from_crawler'):
-        return objcls.from_crawler(crawler, *args, **kwargs)
+        instance = objcls.from_crawler(crawler, *args, **kwargs)
+        method_name = 'from_crawler'
     elif hasattr(objcls, 'from_settings'):
-        return objcls.from_settings(settings, *args, **kwargs)
+        instance = objcls.from_settings(settings, *args, **kwargs)
+        method_name = 'from_settings'
     else:
-        return objcls(*args, **kwargs)
+        instance = objcls(*args, **kwargs)
+        method_name = '__new__'
+    if instance is None:
+        raise TypeError("%s.%s returned None" % (objcls.__qualname__, method_name))
+    return instance
 
 
 @contextmanager
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index 6f945cd01..015a0e5a2 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -114,8 +114,12 @@ class UtilsMiscTestCase(unittest.TestCase):
         #   2. with from_settings() constructor
         #   3. with from_crawler() constructor
         #   4. with from_settings() and from_crawler() constructor
-        spec_sets = ([], ['from_settings'], ['from_crawler'],
-                     ['from_settings', 'from_crawler'])
+        spec_sets = (
+            ['__qualname__'],
+            ['__qualname__', 'from_settings'],
+            ['__qualname__', 'from_crawler'],
+            ['__qualname__', 'from_settings', 'from_crawler'],
+        )
         for specs in spec_sets:
             m = mock.MagicMock(spec_set=specs)
             _test_with_settings(m, settings)
@@ -123,7 +127,7 @@ class UtilsMiscTestCase(unittest.TestCase):
             _test_with_crawler(m, settings, crawler)
 
         # Check adoption of crawler settings
-        m = mock.MagicMock(spec_set=['from_settings'])
+        m = mock.MagicMock(spec_set=['__qualname__', 'from_settings'])
         create_instance(m, None, crawler, *args, **kwargs)
         m.from_settings.assert_called_once_with(crawler.settings, *args,
                                                 **kwargs)
@@ -131,6 +135,10 @@ class UtilsMiscTestCase(unittest.TestCase):
         with self.assertRaises(ValueError):
             create_instance(m, None, None)
 
+        m.from_settings.return_value = None
+        with self.assertRaises(TypeError):
+            create_instance(m, settings, None)
+
     def test_set_environ(self):
         assert os.environ.get('some_test_environ') is None
         with set_environ(some_test_environ='test_value'):

From e01c30f0d54ba838aa3212634cddf3630f77b2d4 Mon Sep 17 00:00:00 2001
From: Jay Rajput <jay24rajput@gmail.com>
Date: Tue, 12 May 2020 01:05:20 +0530
Subject: [PATCH 129/243] Update scrapy/commands/__init__.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 scrapy/commands/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 38b1b77bd..2ba796562 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -108,10 +108,10 @@ class ScrapyCommand:
 
 
 class BaseRunSpiderCommands(ScrapyCommand):
-    ''''
+    """
     The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for
     performing common functionality between crawl.py and runspider.py
-    '''
+    """
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)

From 28d223dd87fb655743f67087be0009b094ddad10 Mon Sep 17 00:00:00 2001
From: Jay Rajput <jay24rajput@gmail.com>
Date: Tue, 12 May 2020 15:28:22 +0530
Subject: [PATCH 130/243] Update __init__.py

---
 scrapy/commands/__init__.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 2ba796562..99276515b 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -118,8 +118,7 @@ class BaseRunSpiderCommands(ScrapyCommand):
         parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
                           help="set spider argument (may be repeated)")
         parser.add_option("-o", "--output", metavar="FILE", action="append",
-                          help="dump scraped items into FILE"
-                          + "(use - for stdout)")
+                          help="dump scraped items into FILE (use - for stdout)")
         parser.add_option("-t", "--output-format", metavar="FORMAT",
                           help="format to use for dumping items with -o")
 

From 07e125f4c916fcd2d4d978a89c7d7e508cc63620 Mon Sep 17 00:00:00 2001
From: Jay Rajput <jay24rajput@gmail.com>
Date: Tue, 12 May 2020 16:31:56 +0530
Subject: [PATCH 131/243] Travis CI fixes in __init__.py

---
 scrapy/commands/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 99276515b..0ddbf2ca0 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -112,7 +112,6 @@ class BaseRunSpiderCommands(ScrapyCommand):
     The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for
     performing common functionality between crawl.py and runspider.py
     """
-
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
         parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",

From 97532a9144cc011e5a6ede84fa7d317c3af60131 Mon Sep 17 00:00:00 2001
From: Aditya Kumar <k.aditya00@gmail.com>
Date: Tue, 12 May 2020 20:40:09 +0530
Subject: [PATCH 132/243] test(spiderloader): no duplicate spider names (#4560)

---
 tests/test_spiderloader/__init__.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index d8be6e277..b6fb27ffe 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -137,6 +137,11 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
             msg = str(w[0].message)
             self.assertIn("several spiders with the same name", msg)
             self.assertIn("'spider3'", msg)
+            self.assertTrue(msg.count("'spider3'") == 2)
+
+            self.assertNotIn("'spider1'", msg)
+            self.assertNotIn("'spider2'", msg)
+            self.assertNotIn("'spider4'", msg)
 
             spiders = set(spider_loader.list())
             self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))
@@ -156,7 +161,13 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
             msg = str(w[0].message)
             self.assertIn("several spiders with the same name", msg)
             self.assertIn("'spider1'", msg)
+            self.assertTrue(msg.count("'spider1'") == 2)
+
             self.assertIn("'spider2'", msg)
+            self.assertTrue(msg.count("'spider2'") == 2)
+
+            self.assertNotIn("'spider3'", msg)
+            self.assertNotIn("'spider4'", msg)
 
             spiders = set(spider_loader.list())
             self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))

From 8d1269bcbc81fa0bb5a69068e07bdbcb0dba8889 Mon Sep 17 00:00:00 2001
From: Aditya Kumar <k.aditya00@gmail.com>
Date: Wed, 13 May 2020 00:12:28 +0530
Subject: [PATCH 133/243] Cover chompjs in documentation (#4562)

---
 docs/topics/dynamic-content.rst | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index 3b85bfe8a..495111b56 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -184,6 +184,18 @@ data from it:
     >>> json.loads(json_data)
     {'field': 'value'}
 
+-   chompjs_ provides an API to parse JavaScript objects into a :class:`dict`.
+
+    For example, if the JavaScript code contains
+    ``var data = {field: "value", secondField: "second value"};``
+    you can extract that data as follows:
+
+    >>> import chompjs
+    >>> javascript = response.css('script::text').get()
+    >>> data = chompjs.parse_js_object(javascript)
+    >>> data
+    {'field': 'value', 'secondField': 'second value'}
+
 -   Otherwise, use js2xml_ to convert the JavaScript code into an XML document
     that you can parse using :ref:`selectors <topics-selectors>`.
 
@@ -241,6 +253,7 @@ along with `scrapy-selenium`_ for seamless integration.
 
 
 .. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29
+.. _chompjs: https://github.com/Nykakin/chompjs
 .. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets
 .. _curl: https://curl.haxx.se/
 .. _headless browser: https://en.wikipedia.org/wiki/Headless_browser

From 33ab0a36635fbd45debbc44584002bd7a4ef7fed Mon Sep 17 00:00:00 2001
From: Jacty <jacty@users.noreply.github.com>
Date: Wed, 13 May 2020 06:11:07 +0800
Subject: [PATCH 134/243] Update __init__.py

---
 scrapy/settings/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index f28fbfaf9..0425b48b3 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -83,7 +83,7 @@ class BaseSettings(MutableMapping):
     def __init__(self, values=None, priority='project'):
         self.frozen = False
         self.attributes = {}
-        if values is not None:
+        if values:
             self.update(values, priority)
 
     def __getitem__(self, opt_name):

From 8971878c1d4d95aff10a1e910688502de892662b Mon Sep 17 00:00:00 2001
From: marc <Marc>
Date: Wed, 13 May 2020 14:11:10 +0200
Subject: [PATCH 135/243] fix new detected flake8 cases

---
 scrapy/core/spidermw.py | 2 +-
 scrapy/utils/console.py | 2 ++
 scrapy/utils/python.py  | 1 +
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 87d08cab7..35264a92b 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -19,7 +19,7 @@ def _isiterable(possible_iterator):
 
 
 def _fname(f):
-    return "%s.%s".format(
+    return "{}.{}".format(
         f.__self__.__class__.__name__,
         f.__func__.__name__
     )
diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index c7a2ace88..133261fd7 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -28,6 +28,7 @@ def _embed_ipython_shell(namespace={}, banner=''):
 def _embed_bpython_shell(namespace={}, banner=''):
     """Start a bpython shell"""
     import bpython
+
     @wraps(_embed_bpython_shell)
     def wrapper(namespace=namespace, banner=''):
         bpython.embed(locals_=namespace, banner=banner)
@@ -37,6 +38,7 @@ def _embed_bpython_shell(namespace={}, banner=''):
 def _embed_ptpython_shell(namespace={}, banner=''):
     """Start a ptpython shell"""
     import ptpython.repl
+
     @wraps(_embed_ptpython_shell)
     def wrapper(namespace=namespace, banner=''):
         print(banner)
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 3d02d9478..9c1f3c2fe 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -152,6 +152,7 @@ def memoizemethod_noargs(method):
     weak reference to its object
     """
     cache = weakref.WeakKeyDictionary()
+
     @wraps(method)
     def new_method(self, *args, **kwargs):
         if self not in cache:

From 2327ecead085a41d1a71a70a12eb988bbf982268 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Wed, 13 May 2020 22:50:04 +0300
Subject: [PATCH 136/243] Rename FEED_STORAGE_BATCH_SIZE to
 FEED_STORAGE_BATCH_ITEM_COUNT

---
 docs/topics/feed-exports.rst        |  8 ++++----
 scrapy/extensions/feedexport.py     | 10 +++++-----
 scrapy/settings/default_settings.py |  2 +-
 tests/test_feedexport.py            | 28 ++++++++++++++--------------
 4 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 2106b41f5..917240d4d 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -220,7 +220,7 @@ These are the settings used for configuring the feed exports:
  * :setting:`FEED_STORAGE_FTP_ACTIVE`
  * :setting:`FEED_STORAGE_S3_ACL`
  * :setting:`FEED_EXPORTERS`
- * :setting:`FEED_EXPORT_BATCH_SIZE`
+ * :setting:`FEED_STORAGE_BATCH_ITEM_COUNT`
 
 .. currentmodule:: scrapy.extensions.feedexport
 
@@ -431,9 +431,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 .. _botocore: https://github.com/boto/botocore
 .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
-.. setting:: FEED_EXPORT_BATCH_SIZE
+.. setting:: FEED_STORAGE_BATCH_ITEM_COUNT
 
-FEED_EXPORT_BATCH_SIZE
+FEED_STORAGE_BATCH_ITEM_COUNT
 ----------------------
 Default: ``None``
 
@@ -448,7 +448,7 @@ Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI.
 
 For instance::
 
-    FEED_EXPORT_BATCH_SIZE=100
+    FEED_STORAGE_BATCH_ITEM_COUNT=100
 
 Your request can be like::
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index a262f5d18..5bc946634 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -242,7 +242,7 @@ class FeedExporter:
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
-        self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_SIZE', None)
+        self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None)
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
@@ -290,7 +290,7 @@ class FeedExporter:
     def _start_new_batch(self, batch_id, uri, feed, spider, template_uri):
         """
         Redirect the output data stream to a new file.
-        Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified.
+        Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified.
         :param batch_id: sequence number of current batch
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
@@ -326,7 +326,7 @@ class FeedExporter:
             slot.start_exporting()
             slot.exporter.export_item(item)
             slot.itemcount += 1
-            # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_SIZE and close the old one
+            # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
             if self.storage_batch_size and slot.itemcount == self.storage_batch_size:
                 uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot)
                 self._close_slot(slot, spider)
@@ -358,12 +358,12 @@ class FeedExporter:
 
     def _batch_deliveries_supported(self, uri):
         """
-        If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(batch_time)s or %(batch_id)s
+        If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s
         to distinguish different files of partial output
         """
         if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
             return True
-        logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
+        logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified')
         return False
 
     def _storage_supported(self, uri):
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index c3463a505..5a7dc533e 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -146,7 +146,7 @@ FEED_STORAGES_BASE = {
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
 }
-FEED_STORAGE_BATCH_SIZE = None
+FEED_STORAGE_BATCH_ITEM_COUNT = None
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
     'json': 'scrapy.exporters.JsonItemExporter',
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 2217bb4ed..1a21eeba9 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1025,7 +1025,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['jl']:
@@ -1041,7 +1041,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
         data = yield self.exported_data(items, settings)
         for batch in data['csv']:
             got_batch = csv.DictReader(to_unicode(batch).splitlines())
@@ -1057,7 +1057,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['xml']:
@@ -1075,7 +1075,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         # XML
@@ -1100,7 +1100,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import pickle
@@ -1117,7 +1117,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import marshal
@@ -1140,7 +1140,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
             {'foo': 'bar3', 'baz': 'quux3', 'egg': ''}
         ]
         settings = {
-            'FEED_STORAGE_BATCH_SIZE': 2
+            'FEED_STORAGE_BATCH_ITEM_COUNT': 2
         }
         header = self.MyItem.fields.keys()
         yield self.assertExported(items, header, rows, settings=settings)
@@ -1151,7 +1151,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},
             },
-            'FEED_STORAGE_BATCH_SIZE': 1
+            'FEED_STORAGE_BATCH_ITEM_COUNT': 1
         }
         crawler = get_crawler(settings_dict=settings)
         self.assertRaises(NotConfigured, FeedExporter, crawler)
@@ -1163,7 +1163,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 'FEEDS': {
                     os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
                 },
-                'FEED_STORAGE_BATCH_SIZE': 1
+                'FEED_STORAGE_BATCH_ITEM_COUNT': 1
             }
             data = yield self.exported_no_data(settings)
             data = dict(data)
@@ -1185,7 +1185,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 },
                 'FEED_STORE_EMPTY': True,
                 'FEED_EXPORT_INDENT': None,
-                'FEED_STORAGE_BATCH_SIZE': 1,
+                'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
             }
             data = yield self.exported_no_data(settings)
             data = dict(data)
@@ -1225,7 +1225,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                     'encoding': 'utf-8',
                 },
             },
-            'FEED_STORAGE_BATCH_SIZE': 1,
+            'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
         }
         data = yield self.exported_data(items, settings)
         for fmt, expected in formats.items():
@@ -1249,7 +1249,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                     'format': 'json',
                 },
             },
-            'FEED_STORAGE_BATCH_SIZE': 1,
+            'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
         }
         data = yield self.exported_data(items, settings)
         self.assertEqual(len(items) + 1, len(data['json']))
@@ -1291,7 +1291,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                     'format': 'json',
                 },
             },
-            'FEED_STORAGE_BATCH_SIZE': 1,
+            'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
         }
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
@@ -1309,7 +1309,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
 
         s3 = boto3.resource('s3')
         my_bucket = s3.Bucket(s3_test_bucket_name)
-        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
 
         with MockServer() as s:
             runner = CrawlerRunner(Settings(settings))

From 8662d3587df74841d4ea640c0432446569e59262 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Wed, 13 May 2020 23:41:01 +0300
Subject: [PATCH 137/243] Documentation and code refactoring

---
 docs/topics/feed-exports.rst    | 21 ++++++++++++---------
 scrapy/extensions/feedexport.py |  7 ++++---
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 917240d4d..0f15044b3 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -437,24 +437,27 @@ FEED_STORAGE_BATCH_ITEM_COUNT
 ----------------------
 Default: ``None``
 
-An integer number which represent number of scraped items stored in each output
+An integer number that represents the number of scraped items stored in each output
 file. Whenever the number of items exceeds this setting, a new file is
-created and output redirects to it.  The name of the new file will be selected
-based on timestamp when the feed is being created and/or batch sequence number.
-Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI.
+created and the output is redirected to it.  The name of the new file will be selected
+based on the timestamp when the feed is being created and/or on the batch sequence number.
+Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`.
 
 * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
-* ``%(batch_id)s`` - gets replaced by sequence number of batch
+(e.g. `2020-03-28T14-45-08.237134`)
 
-For instance::
+* ``%(batch_id)s`` - gets replaced by the batch  sequence number of batch
+(e.g. `2` for the second file)
+
+For instance, if your settings include::
 
     FEED_STORAGE_BATCH_ITEM_COUNT=100
 
-Your request can be like::
+And your :command:`crawl` command line is::
 
   scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
 
-The result directory tree of above can be like::
+The resulting directory tree of above can be like::
 
 ->projectname
 -->dirname
@@ -462,5 +465,5 @@ The result directory tree of above can be like::
 --->2-filename2020-03-28T14-45-09.148903.json
 --->3-filename2020-03-28T14-45-10.046092.json
 
-Where first and second files contain exactly 100 items. The last one contains
+Where the first and second files contain exactly 100 items. The last one contains
 <= 100 items.
\ No newline at end of file
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 5bc946634..4c9362f3a 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -25,6 +25,7 @@ from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import without_none_values
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -337,9 +338,9 @@ class FeedExporter:
                     spider=spider,
                     template_uri=slot.template_uri,
                 ))
-                self.slots[idx] = None
-        self.slots = [slot for slot in self.slots if slot is not None]
-        self.slots.extend(slots)
+            else:
+                slots.append(slot)
+        self.slots = slots
 
     def _load_components(self, setting_prefix):
         conf = without_none_values(self.settings.getwithbase(setting_prefix))

From 69c005f013eb0dc000611853e66371fad17dea9d Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 14 May 2020 10:35:56 +0300
Subject: [PATCH 138/243] Documentation indent fix

---
 docs/topics/feed-exports.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 42c4e2267..dfeea5b7f 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -444,10 +444,10 @@ based on the timestamp when the feed is being created and/or on the batch sequen
 Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`.
 
 * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
-(e.g. `2020-03-28T14-45-08.237134`)
+  (e.g. `2020-03-28T14-45-08.237134`)
 
 * ``%(batch_id)s`` - gets replaced by the batch  sequence number of batch
-(e.g. `2` for the second file)
+  (e.g. `2` for the second file)
 
 For instance, if your settings include::
 

From df8a1d1c0108b15bebdad064d8b7bc61a894f062 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 8 May 2020 20:09:35 -0300
Subject: [PATCH 139/243] Flake8: Remove E741

---
 pytest.ini                                 | 18 +++++------
 scrapy/shell.py                            |  5 ++-
 scrapy/spiders/sitemap.py                  |  3 +-
 scrapy/utils/log.py                        |  2 +-
 tests/test_command_version.py              |  6 ++--
 tests/test_crawl.py                        | 36 +++++++++++-----------
 tests/test_downloadermiddleware_cookies.py |  8 ++---
 tests/test_dupefilters.py                  | 10 +++---
 tests/test_pipeline_media.py               | 10 +++---
 tests/test_proxy_connect.py                | 16 +++++-----
 tests/test_squeues.py                      | 24 +++++++--------
 tests/test_utils_log.py                    | 24 ++++++---------
 tests/test_utils_misc/__init__.py          |  4 +--
 tests/test_utils_signal.py                 | 12 ++++----
 14 files changed, 86 insertions(+), 92 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index fee54dcbd..c141d49a4 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -178,13 +178,13 @@ flake8-ignore =
     tests/test_command_shell.py E501 E128
     tests/test_commands.py E128 E501
     tests/test_contracts.py E501 E128
-    tests/test_crawl.py E501 E741
+    tests/test_crawl.py E501
     tests/test_crawler.py E501
     tests/test_dependencies.py E501
     tests/test_downloader_handlers.py E128 E501
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
-    tests/test_downloadermiddleware_cookies.py E741 E501 E128
+    tests/test_downloadermiddleware_cookies.py E501 E128
     tests/test_downloadermiddleware_defaultheaders.py E501
     tests/test_downloadermiddleware_downloadtimeout.py E501
     tests/test_downloadermiddleware_httpcache.py E501
@@ -195,7 +195,7 @@ flake8-ignore =
     tests/test_downloadermiddleware_retry.py E501 E128
     tests/test_downloadermiddleware_robotstxt.py E501
     tests/test_downloadermiddleware_stats.py E501
-    tests/test_dupefilters.py E501 E741 E128
+    tests/test_dupefilters.py E501 E128
     tests/test_engine.py E501 E128
     tests/test_exporters.py E501 E128
     tests/test_feedexport.py E501
@@ -206,15 +206,15 @@ flake8-ignore =
     tests/test_item.py E128
     tests/test_link.py E501
     tests/test_linkextractors.py E501 E128
-    tests/test_loader.py E501 E741 E128
+    tests/test_loader.py E128 E501 E741
     tests/test_logformatter.py E128 E501
     tests/test_mail.py E128 E501
     tests/test_middleware.py E501 E128
     tests/test_pipeline_crawl.py E501 E128
     tests/test_pipeline_files.py E501
     tests/test_pipeline_images.py E501
-    tests/test_pipeline_media.py E501 E741 E128
-    tests/test_proxy_connect.py E501 E741
+    tests/test_pipeline_media.py E501 E128
+    tests/test_proxy_connect.py E501
     tests/test_request_cb_kwargs.py E501
     tests/test_responsetypes.py E501
     tests/test_robotstxt_interface.py E501 E501
@@ -225,8 +225,8 @@ flake8-ignore =
     tests/test_spidermiddleware_httperror.py E128 E501
     tests/test_spidermiddleware_offsite.py E501 E128
     tests/test_spidermiddleware_output_chain.py E501
-    tests/test_spidermiddleware_referer.py E501 E501
-    tests/test_squeues.py E501 E741
+    tests/test_spidermiddleware_referer.py E501
+    tests/test_squeues.py E501
     tests/test_utils_asyncio.py E501
     tests/test_utils_conf.py E501 E128
     tests/test_utils_curl.py E501
@@ -235,12 +235,10 @@ flake8-ignore =
     tests/test_utils_deprecate.py E501
     tests/test_utils_http.py E501 E128
     tests/test_utils_iterators.py E501 E128
-    tests/test_utils_log.py E741
     tests/test_utils_python.py E501
     tests/test_utils_reqser.py E501 E128
     tests/test_utils_request.py E501 E128
     tests/test_utils_response.py E501
-    tests/test_utils_signal.py E741
     tests/test_utils_sitemap.py E128 E501
     tests/test_utils_url.py E501 E501
     tests/test_webclient.py E501 E128 E402
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 08ce89481..2a3e13ddd 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -146,14 +146,13 @@ class Shell:
         b.append("Useful shortcuts:")
         if self.inthread:
             b.append("  fetch(url[, redirect=True]) "
-                     "Fetch URL and update local objects "
-                     "(by default, redirects are followed)")
+                     "Fetch URL and update local objects (by default, redirects are followed)")
             b.append("  fetch(req)                  "
                      "Fetch a scrapy.Request and update local objects ")
         b.append("  shelp()           Shell help (print this help)")
         b.append("  view(response)    View response in a browser")
 
-        return "\n".join("[s] %s" % l for l in b)
+        return "\n".join("[s] %s" % line for line in b)
 
     def _is_relevant(self, value):
         return isinstance(value, self.relevant_classes)
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index d368c7108..c5360bfa7 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -96,5 +96,4 @@ def iterloc(it, alt=False):
 
         # Also consider alternate URLs (xhtml:link rel="alternate")
         if alt and 'alternate' in d:
-            for l in d['alternate']:
-                yield l
+            yield from d['alternate']
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 203842fc8..6392862ee 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -37,7 +37,7 @@ class TopLevelFormatter(logging.Filter):
         self.loggers = loggers or []
 
     def filter(self, record):
-        if any(record.name.startswith(l + '.') for l in self.loggers):
+        if any(record.name.startswith(logger + '.') for logger in self.loggers):
             record.name = record.name.split('.', 1)[0]
         return True
 
diff --git a/tests/test_command_version.py b/tests/test_command_version.py
index 4ac7fb786..99c01c2b7 100644
--- a/tests/test_command_version.py
+++ b/tests/test_command_version.py
@@ -23,8 +23,10 @@ class VersionTest(ProcessTest, unittest.TestCase):
     def test_verbose_output(self):
         encoding = getattr(sys.stdout, 'encoding') or 'utf-8'
         _, out, _ = yield self.execute(['-v'])
-        headers = [l.partition(":")[0].strip()
-                   for l in out.strip().decode(encoding).splitlines()]
+        headers = [
+            line.partition(":")[0].strip()
+            for line in out.strip().decode(encoding).splitlines()
+        ]
         self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2',
                                    'cssselect', 'parsel', 'w3lib',
                                    'Twisted', 'Python', 'pyOpenSSL',
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 4215ca56c..84f80d103 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -104,44 +104,44 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_retry_503(self):
         crawler = self.runner.create_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
-        self._assert_retried(l)
+        self._assert_retried(log)
 
     @defer.inlineCallbacks
     def test_retry_conn_failed(self):
         crawler = self.runner.create_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as log:
             yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver)
-        self._assert_retried(l)
+        self._assert_retried(log)
 
     @defer.inlineCallbacks
     def test_retry_dns_error(self):
         crawler = self.runner.create_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as log:
             # try to fetch the homepage of a non-existent domain
             yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver)
-        self._assert_retried(l)
+        self._assert_retried(log)
 
     @defer.inlineCallbacks
     def test_start_requests_bug_before_yield(self):
-        with LogCapture('scrapy', level=logging.ERROR) as l:
+        with LogCapture('scrapy', level=logging.ERROR) as log:
             crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
             yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver)
 
-        self.assertEqual(len(l.records), 1)
-        record = l.records[0]
+        self.assertEqual(len(log.records), 1)
+        record = log.records[0]
         self.assertIsNotNone(record.exc_info)
         self.assertIs(record.exc_info[0], ZeroDivisionError)
 
     @defer.inlineCallbacks
     def test_start_requests_bug_yielding(self):
-        with LogCapture('scrapy', level=logging.ERROR) as l:
+        with LogCapture('scrapy', level=logging.ERROR) as log:
             crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
             yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver)
 
-        self.assertEqual(len(l.records), 1)
-        record = l.records[0]
+        self.assertEqual(len(log.records), 1)
+        record = log.records[0]
         self.assertIsNotNone(record.exc_info)
         self.assertIs(record.exc_info[0], ZeroDivisionError)
 
@@ -187,25 +187,25 @@ foo body
 with multiples lines
 '''})
         crawler = self.runner.create_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/raw?{0}".format(query)), mockserver=self.mockserver)
-        self.assertEqual(str(l).count("Got response 200"), 1)
+        self.assertEqual(str(log).count("Got response 200"), 1)
 
     @defer.inlineCallbacks
     def test_retry_conn_lost(self):
         # connection lost after receiving data
         crawler = self.runner.create_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver)
-        self._assert_retried(l)
+        self._assert_retried(log)
 
     @defer.inlineCallbacks
     def test_retry_conn_aborted(self):
         # connection lost before receiving data
         crawler = self.runner.create_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver)
-        self._assert_retried(l)
+        self._assert_retried(log)
 
     def _assert_retried(self, log):
         self.assertEqual(str(log).count("Retrying"), 2)
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index f86c50f50..d54434c8f 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -63,7 +63,7 @@ class CookiesMiddlewareTest(TestCase):
         mw = CookiesMiddleware.from_crawler(crawler)
         with LogCapture('scrapy.downloadermiddlewares.cookies',
                         propagate=False,
-                        level=logging.DEBUG) as l:
+                        level=logging.DEBUG) as log:
             req = Request('http://scrapytest.org/')
             res = Response('http://scrapytest.org/',
                            headers={'Set-Cookie': 'C1=value1; path=/'})
@@ -71,7 +71,7 @@ class CookiesMiddlewareTest(TestCase):
             req2 = Request('http://scrapytest.org/sub1/')
             mw.process_request(req2, crawler.spider)
 
-            l.check(
+            log.check(
                 ('scrapy.downloadermiddlewares.cookies',
                  'DEBUG',
                  'Received cookies from: <200 http://scrapytest.org/>\n'
@@ -87,7 +87,7 @@ class CookiesMiddlewareTest(TestCase):
         mw = CookiesMiddleware.from_crawler(crawler)
         with LogCapture('scrapy.downloadermiddlewares.cookies',
                         propagate=False,
-                        level=logging.DEBUG) as l:
+                        level=logging.DEBUG) as log:
             req = Request('http://scrapytest.org/')
             res = Response('http://scrapytest.org/',
                            headers={'Set-Cookie': 'C1=value1; path=/'})
@@ -95,7 +95,7 @@ class CookiesMiddlewareTest(TestCase):
             req2 = Request('http://scrapytest.org/sub1/')
             mw.process_request(req2, crawler.spider)
 
-            l.check()
+            log.check()
 
     def test_do_not_break_on_non_utf8_header(self):
         req = Request('http://scrapytest.org/')
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 7426107c1..ee1794f4f 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -160,7 +160,7 @@ class RFPDupeFilterTest(unittest.TestCase):
             shutil.rmtree(path)
 
     def test_log(self):
-        with LogCapture() as l:
+        with LogCapture() as log:
             settings = {'DUPEFILTER_DEBUG': False,
                         'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
@@ -177,7 +177,7 @@ class RFPDupeFilterTest(unittest.TestCase):
             dupefilter.log(r2, spider)
 
             assert crawler.stats.get_value('dupefilter/filtered') == 2
-            l.check_present(('scrapy.dupefilters', 'DEBUG',
+            log.check_present(('scrapy.dupefilters', 'DEBUG',
                 ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
                 ' - no more duplicates will be shown'
                 ' (see DUPEFILTER_DEBUG to show all duplicates)')))
@@ -185,7 +185,7 @@ class RFPDupeFilterTest(unittest.TestCase):
             dupefilter.close('finished')
 
     def test_log_debug(self):
-        with LogCapture() as l:
+        with LogCapture() as log:
             settings = {'DUPEFILTER_DEBUG': True,
                         'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
@@ -203,10 +203,10 @@ class RFPDupeFilterTest(unittest.TestCase):
             dupefilter.log(r2, spider)
 
             assert crawler.stats.get_value('dupefilter/filtered') == 2
-            l.check_present(('scrapy.dupefilters', 'DEBUG',
+            log.check_present(('scrapy.dupefilters', 'DEBUG',
                 ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
                 ' (referer: None)')))
-            l.check_present(('scrapy.dupefilters', 'DEBUG',
+            log.check_present(('scrapy.dupefilters', 'DEBUG',
                 ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
                 ' (referer: http://scrapytest.org/INDEX.html)')))
 
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 949f0dea1..d7b0d32b2 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -63,21 +63,21 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
         fail = Failure(Exception())
         results = [(True, 1), (False, fail)]
 
-        with LogCapture() as l:
+        with LogCapture() as log:
             new_item = self.pipe.item_completed(results, item, self.info)
 
         assert new_item is item
-        assert len(l.records) == 1
-        record = l.records[0]
+        assert len(log.records) == 1
+        record = log.records[0]
         assert record.levelname == 'ERROR'
         self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail))
 
         # disable failure logging and check again
         self.pipe.LOG_FAILED_RESULTS = False
-        with LogCapture() as l:
+        with LogCapture() as log:
             new_item = self.pipe.item_completed(results, item, self.info)
         assert new_item is item
-        assert len(l.records) == 0
+        assert len(log.records) == 0
 
     @inlineCallbacks
     def test_default_process_item(self):
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 188ec68dd..4763a5417 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -76,35 +76,35 @@ class ProxyConnectTestCase(TestCase):
     @defer.inlineCallbacks
     def test_https_connect_tunnel(self):
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(200, l)
+        self._assert_got_response_code(200, log)
 
     @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6)
     @defer.inlineCallbacks
     def test_https_connect_tunnel_error(self):
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as log:
             yield crawler.crawl("https://localhost:99999/status?n=200")
-        self._assert_got_tunnel_error(l)
+        self._assert_got_tunnel_error(log)
 
     @defer.inlineCallbacks
     def test_https_tunnel_auth_error(self):
         os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         # The proxy returns a 407 error code but it does not reach the client;
         # he just sees a TunnelError.
-        self._assert_got_tunnel_error(l)
+        self._assert_got_tunnel_error(log)
 
     @defer.inlineCallbacks
     def test_https_tunnel_without_leak_proxy_authorization_header(self):
         request = Request(self.mockserver.url("/echo", is_secure=True))
         crawler = get_crawler(SingleRequestSpider)
-        with LogCapture() as l:
+        with LogCapture() as log:
             yield crawler.crawl(seed=request)
-        self._assert_got_response_code(200, l)
+        self._assert_got_response_code(200, log)
         echo = json.loads(crawler.spider.meta['responses'][0].text)
         self.assertTrue('Proxy-Authorization' not in echo['headers'])
 
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index d2cf9135f..becacce62 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -89,12 +89,12 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
 
     def test_serialize_loader(self):
         q = self.queue()
-        l = TestLoader()
-        q.push(l)
-        l2 = q.pop()
-        assert isinstance(l2, TestLoader)
-        assert l2.default_item_class is TestItem
-        self.assertEqual(l2.name_out('x'), 'xx')
+        loader = TestLoader()
+        q.push(loader)
+        loader2 = q.pop()
+        assert isinstance(loader2, TestLoader)
+        assert loader2.default_item_class is TestItem
+        self.assertEqual(loader2.name_out('x'), 'xx')
 
     def test_serialize_request_recursive(self):
         q = self.queue()
@@ -173,12 +173,12 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
 
     def test_serialize_loader(self):
         q = self.queue()
-        l = TestLoader()
-        q.push(l)
-        l2 = q.pop()
-        assert isinstance(l2, TestLoader)
-        assert l2.default_item_class is TestItem
-        self.assertEqual(l2.name_out('x'), 'xx')
+        loader = TestLoader()
+        q.push(loader)
+        loader2 = q.pop()
+        assert isinstance(loader2, TestLoader)
+        assert loader2.default_item_class is TestItem
+        self.assertEqual(loader2.name_out('x'), 'xx')
 
     def test_serialize_request_recursive(self):
         q = self.queue()
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 25cd904bc..535f56691 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -34,31 +34,27 @@ class TopLevelFormatterTest(unittest.TestCase):
 
     def test_top_level_logger(self):
         logger = logging.getLogger('test')
-        with self.handler as l:
+        with self.handler as log:
             logger.warning('test log msg')
-
-        l.check(('test', 'WARNING', 'test log msg'))
+        log.check(('test', 'WARNING', 'test log msg'))
 
     def test_children_logger(self):
         logger = logging.getLogger('test.test1')
-        with self.handler as l:
+        with self.handler as log:
             logger.warning('test log msg')
-
-        l.check(('test', 'WARNING', 'test log msg'))
+        log.check(('test', 'WARNING', 'test log msg'))
 
     def test_overlapping_name_logger(self):
         logger = logging.getLogger('test2')
-        with self.handler as l:
+        with self.handler as log:
             logger.warning('test log msg')
-
-        l.check(('test2', 'WARNING', 'test log msg'))
+        log.check(('test2', 'WARNING', 'test log msg'))
 
     def test_different_name_logger(self):
         logger = logging.getLogger('different')
-        with self.handler as l:
+        with self.handler as log:
             logger.warning('test log msg')
-
-        l.check(('different', 'WARNING', 'test log msg'))
+        log.check(('different', 'WARNING', 'test log msg'))
 
 
 class LogCounterHandlerTest(unittest.TestCase):
@@ -107,6 +103,6 @@ class StreamLoggerTest(unittest.TestCase):
         sys.stdout = self.stdout
 
     def test_redirect(self):
-        with LogCapture() as l:
+        with LogCapture() as log:
             print('test log msg')
-        l.check(('test', 'ERROR', 'test log msg'))
+        log.check(('test', 'ERROR', 'test log msg'))
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index 015a0e5a2..28205e0d9 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -67,12 +67,12 @@ class UtilsMiscTestCase(unittest.TestCase):
         assert hasattr(arg_to_iter(100), '__iter__')
         assert hasattr(arg_to_iter('lala'), '__iter__')
         assert hasattr(arg_to_iter([1, 2, 3]), '__iter__')
-        assert hasattr(arg_to_iter(l for l in 'abcd'), '__iter__')
+        assert hasattr(arg_to_iter(c for c in 'abcd'), '__iter__')
 
         self.assertEqual(list(arg_to_iter(None)), [])
         self.assertEqual(list(arg_to_iter('lala')), ['lala'])
         self.assertEqual(list(arg_to_iter(100)), [100])
-        self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c'])
+        self.assertEqual(list(arg_to_iter(c for c in 'abc')), ['a', 'b', 'c'])
         self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3])
         self.assertEqual(list(arg_to_iter({'a': 1})), [{'a': 1}])
         self.assertEqual(list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")])
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index c83c9398c..b66588efb 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -20,7 +20,7 @@ class SendCatchLogTest(unittest.TestCase):
 
         dispatcher.connect(self.error_handler, signal=test_signal)
         dispatcher.connect(self.ok_handler, signal=test_signal)
-        with LogCapture() as l:
+        with LogCapture() as log:
             result = yield defer.maybeDeferred(
                 self._get_result, test_signal, arg='test',
                 handlers_called=handlers_called
@@ -28,8 +28,8 @@ class SendCatchLogTest(unittest.TestCase):
 
         assert self.error_handler in handlers_called
         assert self.ok_handler in handlers_called
-        self.assertEqual(len(l.records), 1)
-        record = l.records[0]
+        self.assertEqual(len(log.records), 1)
+        record = log.records[0]
         self.assertIn('error_handler', record.getMessage())
         self.assertEqual(record.levelname, 'ERROR')
         self.assertEqual(result[0][0], self.error_handler)
@@ -95,8 +95,8 @@ class SendCatchLogTest2(unittest.TestCase):
 
         test_signal = object()
         dispatcher.connect(test_handler, test_signal)
-        with LogCapture() as l:
+        with LogCapture() as log:
             send_catch_log(test_signal)
-        self.assertEqual(len(l.records), 1)
-        self.assertIn("Cannot return deferreds from signal handler", str(l))
+        self.assertEqual(len(log.records), 1)
+        self.assertIn("Cannot return deferreds from signal handler", str(log))
         dispatcher.disconnect(test_handler, test_signal)

From fffb0a5b6a7eaf365ecd7ef43e7e45cf2ea7ff2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com>
Date: Fri, 15 May 2020 15:23:09 +0200
Subject: [PATCH 140/243] replace set constructor with set literals (#4573)

---
 tests/test_engine.py                | 8 ++++----
 tests/test_http_request.py          | 6 +++---
 tests/test_pipeline_crawl.py        | 4 ++--
 tests/test_pipeline_images.py       | 2 +-
 tests/test_spiderloader/__init__.py | 8 ++++----
 tests/test_utils_datatypes.py       | 2 +-
 tests/test_utils_misc/__init__.py   | 8 ++++----
 tests/test_utils_url.py             | 2 +-
 8 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index acfe94f63..d781665dc 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -218,8 +218,8 @@ class EngineTest(unittest.TestCase):
     def _assert_visited_urls(self):
         must_be_visited = ["/", "/redirect", "/redirected",
                            "/item1.html", "/item2.html", "/item999.html"]
-        urls_visited = set([rp[0].url for rp in self.run.respplug])
-        urls_expected = set([self.run.geturl(p) for p in must_be_visited])
+        urls_visited = {rp[0].url for rp in self.run.respplug}
+        urls_expected = {self.run.geturl(p) for p in must_be_visited}
         assert urls_expected <= urls_visited, "URLs not visited: %s" % list(urls_expected - urls_visited)
 
     def _assert_scheduled_requests(self, urls_to_visit=None):
@@ -227,8 +227,8 @@ class EngineTest(unittest.TestCase):
 
         paths_expected = ['/item999.html', '/item2.html', '/item1.html']
 
-        urls_requested = set([rq[0].url for rq in self.run.reqplug])
-        urls_expected = set([self.run.geturl(p) for p in paths_expected])
+        urls_requested = {rq[0].url for rq in self.run.reqplug}
+        urls_expected = {self.run.geturl(p) for p in paths_expected}
         assert urls_expected <= urls_requested
         scheduled_requests_count = len(self.run.reqplug)
         dropped_requests_count = len(self.run.reqdropped)
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index a672963f3..349c23f36 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -549,8 +549,8 @@ class FormRequestTest(RequestTest):
         self.assertEqual(urlparse(r1.url).hostname, "www.example.com")
         self.assertEqual(urlparse(r1.url).path, "/this/get.php")
         fs = _qs(r1)
-        self.assertEqual(set(fs[b'test']), set([b'val1', b'val2']))
-        self.assertEqual(set(fs[b'one']), set([b'two', b'three']))
+        self.assertEqual(set(fs[b'test']), {b'val1', b'val2'})
+        self.assertEqual(set(fs[b'one']), {b'two', b'three'})
         self.assertEqual(fs[b'test2'], [b'xxx'])
         self.assertEqual(fs[b'six'], [b'seven'])
 
@@ -1047,7 +1047,7 @@ class FormRequestTest(RequestTest):
             </form>''')
         req = self.request_class.from_response(res)
         fs = _qs(req)
-        self.assertEqual(set(fs), set([b'h2', b'i2', b'i1', b'i3', b'h1', b'i5', b'i4']))
+        self.assertEqual(set(fs), {b'h2', b'i2', b'i1', b'i3', b'h1', b'i5', b'i4'})
 
     def test_from_response_xpath(self):
         response = _buildresponse(
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 6d15aaf31..74263e50c 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -51,10 +51,10 @@ class FileDownloadCrawlTestCase(TestCase):
     store_setting_key = 'FILES_STORE'
     media_key = 'files'
     media_urls_key = 'file_urls'
-    expected_checksums = set([
+    expected_checksums = {
         '5547178b89448faf0015a13f904c936e',
         'c2281c83670e31d8aaab7cb642b824db',
-        'ed3f6538dc15d4d9179dae57319edc5f'])
+        'ed3f6538dc15d4d9179dae57319edc5f'}
 
     def setUp(self):
         self.mockserver = MockServer()
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 5ba03ff4c..76aea470f 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -18,7 +18,7 @@ try:
 except ImportError:
     skip = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
 else:
-    encoders = set(('jpeg_encoder', 'jpeg_decoder'))
+    encoders = {'jpeg_encoder', 'jpeg_decoder'}
     if not encoders.issubset(set(Image.core.__dict__)):
         skip = 'Missing JPEG encoders'
 
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 265970b43..d922c6059 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -42,7 +42,7 @@ class SpiderLoaderTest(unittest.TestCase):
     def test_list(self):
         self.assertEqual(
             set(self.spider_loader.list()),
-            set(['spider1', 'spider2', 'spider3', 'spider4']))
+            {'spider1', 'spider2', 'spider3', 'spider4'})
 
     def test_load(self):
         spider1 = self.spider_loader.load("spider1")
@@ -57,7 +57,7 @@ class SpiderLoaderTest(unittest.TestCase):
             ['spider2'])
         self.assertEqual(
             set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))),
-            set(['spider1', 'spider2']))
+            {'spider1', 'spider2'})
         self.assertEqual(
             self.spider_loader.find_by_request(Request('http://scrapy999.org/test')),
             [])
@@ -151,7 +151,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
             self.assertNotIn("'spider4'", msg)
 
             spiders = set(spider_loader.list())
-            self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))
+            self.assertEqual(spiders, {'spider1', 'spider2', 'spider3', 'spider4'})
 
     def test_multiple_dupename_warning(self):
         # copy 2 spider modules so as to have duplicate spider name
@@ -177,4 +177,4 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
             self.assertNotIn("'spider4'", msg)
 
             spiders = set(spider_loader.list())
-            self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))
+            self.assertEqual(spiders, {'spider1', 'spider2', 'spider3', 'spider4'})
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index e5aa56eb9..0a4c6034a 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -217,7 +217,7 @@ class SequenceExcludeTest(unittest.TestCase):
 
     def test_set(self):
         """Anything that is not in the supplied sequence will evaluate as 'in' the container."""
-        seq = set([-3, "test", 1.1])
+        seq = {-3, "test", 1.1}
         d = SequenceExclude(seq)
         self.assertIn(0, d)
         self.assertIn("foo", d)
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index 28205e0d9..9bb996d27 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -26,20 +26,20 @@ class UtilsMiscTestCase(unittest.TestCase):
             'tests.test_utils_misc.test_walk_modules.mod.mod0',
             'tests.test_utils_misc.test_walk_modules.mod1',
         ]
-        self.assertEqual(set([m.__name__ for m in mods]), set(expected))
+        self.assertEqual({m.__name__ for m in mods}, set(expected))
 
         mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod')
         expected = [
             'tests.test_utils_misc.test_walk_modules.mod',
             'tests.test_utils_misc.test_walk_modules.mod.mod0',
         ]
-        self.assertEqual(set([m.__name__ for m in mods]), set(expected))
+        self.assertEqual({m.__name__ for m in mods}, set(expected))
 
         mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod1')
         expected = [
             'tests.test_utils_misc.test_walk_modules.mod1',
         ]
-        self.assertEqual(set([m.__name__ for m in mods]), set(expected))
+        self.assertEqual({m.__name__ for m in mods}, set(expected))
 
         self.assertRaises(ImportError, walk_modules, 'nomodule999')
 
@@ -54,7 +54,7 @@ class UtilsMiscTestCase(unittest.TestCase):
                 'testegg.spiders.b',
                 'testegg'
             ]
-            self.assertEqual(set([m.__name__ for m in mods]), set(expected))
+            self.assertEqual({m.__name__ for m in mods}, set(expected))
         finally:
             sys.path.remove(egg)
 
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 16e7449c9..4ab944949 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -55,7 +55,7 @@ class UrlUtilsTest(unittest.TestCase):
         self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', spider))
         self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', spider))
 
-        spider = Spider(name='example.com', allowed_domains=set(('example.com', 'example.net')))
+        spider = Spider(name='example.com', allowed_domains={'example.com', 'example.net'})
         self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider))
 
         spider = Spider(name='example.com', allowed_domains=('example.com', 'example.net'))

From 1cdcf8b08b8f1e68c5b107b6ae39b2da1aedd245 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Fri, 15 May 2020 19:46:36 +0300
Subject: [PATCH 141/243] Minor fixes

---
 docs/topics/feed-exports.rst    | 19 ++++++++++---------
 scrapy/extensions/feedexport.py | 20 ++++++++++----------
 2 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index dfeea5b7f..638733b6a 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -437,17 +437,18 @@ FEED_STORAGE_BATCH_ITEM_COUNT
 -----------------------------
 Default: ``None``
 
-An integer number that represents the number of scraped items stored in each output
-file. Whenever the number of items exceeds this setting, a new file is
-created and the output is redirected to it.  The name of the new file will be selected
-based on the timestamp when the feed is being created and/or on the batch sequence number.
-Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`.
+If assigned an integer number higher than ``0``, Scrapy generates multiple output files
+storing up to the specified number of items in each output file.
+
+When generating multiple output files, you must use at least one of the following
+placeholders in :setting:`FEED_URI` to indicate how the different output file names are
+generated:
 
 * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
-  (e.g. `2020-03-28T14-45-08.237134`)
+  (e.g. ``2020-03-28T14-45-08.237134``)
 
 * ``%(batch_id)s`` - gets replaced by the batch  sequence number of batch
-  (e.g. `2` for the second file)
+  (e.g. ``2`` for the second file)
 
 For instance, if your settings include::
 
@@ -457,7 +458,7 @@ And your :command:`crawl` command line is::
 
   scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
 
-The resulting directory tree of above can be like::
+The command line above can generate a directory tree like::
 
 ->projectname
 -->dirname
@@ -466,4 +467,4 @@ The resulting directory tree of above can be like::
 --->3-filename2020-03-28T14-45-10.046092.json
 
 Where the first and second files contain exactly 100 items. The last one contains
-<= 100 items.
\ No newline at end of file
+100 items or fever.
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 4c9362f3a..3d691c580 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -180,7 +180,7 @@ class FTPFeedStorage(BlockingFeedStorage):
 
 
 class _FeedSlot:
-    def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, template_uri):
+    def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template):
         self.file = file
         self.exporter = exporter
         self.storage = storage
@@ -188,7 +188,7 @@ class _FeedSlot:
         self.batch_id = batch_id
         self.format = format
         self.store_empty = store_empty
-        self.template_uri = template_uri
+        self.uri_template = uri_template
         self.uri = uri
         # flags
         self.itemcount = 0
@@ -260,7 +260,7 @@ class FeedExporter:
                 uri=uri % uri_params,
                 feed=feed,
                 spider=spider,
-                template_uri=uri,
+                uri_template=uri,
             ))
 
     def close_spider(self, spider):
@@ -288,7 +288,7 @@ class FeedExporter:
                                             extra={'spider': spider}))
         return d
 
-    def _start_new_batch(self, batch_id, uri, feed, spider, template_uri):
+    def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
         """
         Redirect the output data stream to a new file.
         Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified.
@@ -296,7 +296,7 @@ class FeedExporter:
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
         :param spider: user spider
-        :param template_uri: template uri which contains %(batch_time)s or %(batch_id)s to create new uri
+        :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)s to create new uri
         """
         storage = self._get_storage(uri)
         file = storage.open(spider)
@@ -315,7 +315,7 @@ class FeedExporter:
             format=feed['format'],
             store_empty=feed['store_empty'],
             batch_id=batch_id,
-            template_uri=template_uri,
+            uri_template=uri_template,
         )
         if slot.store_empty:
             slot.start_exporting()
@@ -329,14 +329,14 @@ class FeedExporter:
             slot.itemcount += 1
             # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
             if self.storage_batch_size and slot.itemcount == self.storage_batch_size:
-                uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot)
+                uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
                 self._close_slot(slot, spider)
                 slots.append(self._start_new_batch(
                     batch_id=slot.batch_id + 1,
-                    uri=slot.template_uri % uri_params,
-                    feed=self.feeds[slot.template_uri],
+                    uri=slot.uri_template % uri_params,
+                    feed=self.feeds[slot.uri_template],
                     spider=spider,
-                    template_uri=slot.template_uri,
+                    uri_template=slot.uri_template,
                 ))
             else:
                 slots.append(slot)

From 36c3c9713e59f5d22bf51354920b5093e2d30b73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 15 May 2020 19:37:56 +0200
Subject: [PATCH 142/243] Run tests with Python 3.5.0 (#4518)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Run tests with Python 3.5.0

* Use Ubuntu Trusty 14.04 to test Python 3.5.0

* Use mitmproxy<3.0.0 on Python 3.5.0

* Skip tests requiring mitmproxy in Python 3.5.0

* Change the minimum Python version from 3.5 to 3.5.1

* Do not prevent Scrapy from working with Python 3.5.0

* Force system Python 3.5.1

* Do not install a non-system Python in Python 3.5.1 jobs

* Switch to Trusty to be able to test Xenial’s Python version ¯\_(ツ)_/¯

* Add missing trusty

* Stop breaking old PyPy

* Allow installing Scrapy on Python 3.5.0
---
 .travis.yml                 | 10 +++++++---
 README.rst                  |  2 +-
 docs/faq.rst                |  2 +-
 docs/intro/install.rst      |  2 +-
 tests/test_proxy_connect.py |  5 ++++-
 5 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index bcbf75a43..d6ec88e06 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,12 +17,16 @@ matrix:
       python: 3.7  # Keep in sync with .readthedocs.yml
 
     - env: TOXENV=pypy3
+    - env: TOXENV=pinned
+      python: 3.5.1
+      dist: trusty
+    - env: TOXENV=asyncio
+      python: 3.5.1  # We use additional code to support 3.5.3 and earlier
+      dist: trusty
     - env: TOXENV=py
       python: 3.5
-    - env: TOXENV=pinned
-      python: 3.5
     - env: TOXENV=asyncio
-      python: 3.5.2
+      python: 3.5  # We use specific code to support >= 3.5.4, < 3.6
     - env: TOXENV=py
       python: 3.6
     - env: TOXENV=py
diff --git a/README.rst b/README.rst
index ce5973bcd..fd84e127e 100644
--- a/README.rst
+++ b/README.rst
@@ -40,7 +40,7 @@ including a list of features.
 Requirements
 ============
 
-* Python 3.5+
+* Python 3.5.1+
 * Works on Linux, Windows, macOS, BSD
 
 Install
diff --git a/docs/faq.rst b/docs/faq.rst
index 75a0f4864..936f315b3 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -69,7 +69,7 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
 What Python versions does Scrapy support?
 -----------------------------------------
 
-Scrapy is supported under Python 3.5+
+Scrapy is supported under Python 3.5.1+
 under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
 Python 3 support was added in Scrapy 1.1.
 PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 6356e0eea..4af80d801 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -7,7 +7,7 @@ Installation guide
 Installing Scrapy
 =================
 
-Scrapy runs on Python 3.5 or above under CPython (default Python
+Scrapy runs on Python 3.5.1 or above under CPython (default Python
 implementation) and PyPy (starting with PyPy 5.9).
 
 If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 4763a5417..eb4ecc91d 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -4,6 +4,7 @@ import re
 import sys
 from subprocess import Popen, PIPE
 from urllib.parse import urlsplit, urlunsplit
+from unittest import skipIf
 
 import pytest
 from testfixtures import LogCapture
@@ -56,6 +57,8 @@ def _wrong_credentials(proxy_url):
     return urlunsplit(bad_auth_proxy)
 
 
+@skipIf(sys.version_info < (3, 5, 4),
+        "requires mitmproxy < 3.0.0, which these tests do not support")
 class ProxyConnectTestCase(TestCase):
 
     def setUp(self):
@@ -80,7 +83,7 @@ class ProxyConnectTestCase(TestCase):
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         self._assert_got_response_code(200, log)
 
-    @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6)
+    @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info >= (3, 6))
     @defer.inlineCallbacks
     def test_https_connect_tunnel_error(self):
         crawler = get_crawler(SimpleSpider)

From 4cdd00e21f4bfe22ba9b8fabe034a5e4d34dab75 Mon Sep 17 00:00:00 2001
From: Jay Rajput <jay24rajput@gmail.com>
Date: Sat, 16 May 2020 00:25:57 +0530
Subject: [PATCH 143/243] Changed BaseRunSpiderCommands to BaseRunSpiderCommand

---
 scrapy/commands/__init__.py  | 2 +-
 scrapy/commands/crawl.py     | 4 ++--
 scrapy/commands/runspider.py | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 0ddbf2ca0..81fae089b 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -107,7 +107,7 @@ class ScrapyCommand:
         raise NotImplementedError
 
 
-class BaseRunSpiderCommands(ScrapyCommand):
+class BaseRunSpiderCommand(ScrapyCommand):
     """
     The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for
     performing common functionality between crawl.py and runspider.py
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index c39c5a9ef..e1724c1e6 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -1,8 +1,8 @@
-from scrapy.commands import BaseRunSpiderCommands
+from scrapy.commands import BaseRunSpiderCommand
 from scrapy.exceptions import UsageError
 
 
-class Command(BaseRunSpiderCommands):
+class Command(BaseRunSpiderCommand):
 
     requires_project = True
 
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index bd24a369e..befee021b 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -4,7 +4,7 @@ from importlib import import_module
 
 from scrapy.utils.spider import iter_spider_classes
 from scrapy.exceptions import UsageError
-from scrapy.commands import BaseRunSpiderCommands
+from scrapy.commands import BaseRunSpiderCommand
 
 
 def _import_file(filepath):
@@ -23,7 +23,7 @@ def _import_file(filepath):
     return module
 
 
-class Command(BaseRunSpiderCommands):
+class Command(BaseRunSpiderCommand):
 
     requires_project = False
     default_settings = {'SPIDER_LOADER_WARN_ONLY': True}

From 10ae1a284f759b541d086e3d1a13cda96b6e2040 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Fri, 15 May 2020 22:50:54 +0300
Subject: [PATCH 144/243] Minor fixes

---
 docs/topics/feed-exports.rst    |  2 +-
 scrapy/extensions/feedexport.py | 10 +++++-----
 tests/test_feedexport.py        |  2 +-
 tox.ini                         |  2 +-
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 638733b6a..6f7db20c4 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -467,4 +467,4 @@ The command line above can generate a directory tree like::
 --->3-filename2020-03-28T14-45-10.046092.json
 
 Where the first and second files contain exactly 100 items. The last one contains
-100 items or fever.
+100 items or fewer.
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 3d691c580..cc26ae173 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -243,11 +243,11 @@ class FeedExporter:
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
-        self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None)
+        self.storage_batch_item_count = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None)
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
-            if not self._batch_deliveries_supported(uri):
+            if not self._settings_are_valid(uri):
                 raise NotConfigured
             if not self._exporter_supported(feed['format']):
                 raise NotConfigured
@@ -328,7 +328,7 @@ class FeedExporter:
             slot.exporter.export_item(item)
             slot.itemcount += 1
             # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
-            if self.storage_batch_size and slot.itemcount == self.storage_batch_size:
+            if self.storage_batch_item_count and slot.itemcount == self.storage_batch_item_count:
                 uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
                 self._close_slot(slot, spider)
                 slots.append(self._start_new_batch(
@@ -357,12 +357,12 @@ class FeedExporter:
             return True
         logger.error("Unknown feed format: %(format)s", {'format': format})
 
-    def _batch_deliveries_supported(self, uri):
+    def _settings_are_valid(self, uri):
         """
         If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s
         to distinguish different files of partial output
         """
-        if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
+        if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
             return True
         logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified')
         return False
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index d1374f291..88f9a5933 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -986,7 +986,7 @@ class FeedExportTest(FeedExportTestBase):
         self.assertEqual(data['csv'], b'')
 
 
-class PartialDeliveriesTest(FeedExportTestBase):
+class BatchDeliveriesTest(FeedExportTestBase):
     __test__ = True
     _file_mark = '_%(batch_time)s_#%(batch_id)s_'
 
diff --git a/tox.ini b/tox.ini
index 6dd944dff..7507a14a6 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,9 +12,9 @@ deps =
     -ctests/constraints.txt
     -rtests/requirements-py3.txt
     # Extras
+    boto3>=1.13.0
     botocore>=1.3.23
     Pillow>=3.4.2
-    boto3>=1.13.0
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID

From 604fe33bad36f1269677e98d0bfec1f60c95aa53 Mon Sep 17 00:00:00 2001
From: Jay Rajput <jay24rajput@gmail.com>
Date: Sat, 16 May 2020 01:53:49 +0530
Subject: [PATCH 145/243] Update scrapy/commands/__init__.py

Changed typo in a comment for BaseRunSpiderCommand

Co-authored-by: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
---
 scrapy/commands/__init__.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 49152ca28..ab850dcb3 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -108,8 +108,7 @@ class ScrapyCommand:
 
 class BaseRunSpiderCommand(ScrapyCommand):
     """
-    The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for
-    performing common functionality between crawl.py and runspider.py
+    Common class used to share functionality between the crawl and runspider commands
     """
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)

From a915af2e4592e8a0367c44a73c95cee3f835887d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 11 May 2020 16:13:54 -0300
Subject: [PATCH 146/243] Remove monkeypatches module, E402 flake8 code

---
 pytest.ini                        |  4 +--
 scrapy/__init__.py                | 47 ++++++++++++++++---------------
 scrapy/_monkeypatches.py          | 11 --------
 scrapy/linkextractors/__init__.py |  2 +-
 scrapy/spiders/__init__.py        |  6 ++--
 5 files changed, 30 insertions(+), 40 deletions(-)
 delete mode 100644 scrapy/_monkeypatches.py

diff --git a/pytest.ini b/pytest.ini
index 0e289866c..0797e8ac8 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -27,18 +27,16 @@ flake8-ignore =
     # Exclude files that are meant to provide top-level imports
     # E402: Module level import not at top of file
     # F401: Module imported but unused
+    scrapy/__init__.py E402
     scrapy/core/downloader/handlers/http.py F401
     scrapy/http/__init__.py F401
     scrapy/linkextractors/__init__.py E402 F401
     scrapy/spiders/__init__.py E402 F401
 
     # Issues pending a review:
-    scrapy/__init__.py E402
     scrapy/selector/__init__.py F403
-    scrapy/spiders/__init__.py E402
     scrapy/utils/http.py F403
     scrapy/utils/markup.py F403
     scrapy/utils/multipart.py F403
     scrapy/utils/url.py F403 F405
     tests/test_loader.py E741
-    tests/test_webclient.py E402
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index fb8357f3c..e791deaa6 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -2,33 +2,11 @@
 Scrapy - a web crawling and web scraping framework written for Python
 """
 
-__all__ = ['__version__', 'version_info', 'twisted_version',
-           'Spider', 'Request', 'FormRequest', 'Selector', 'Item', 'Field']
-
-# Scrapy version
 import pkgutil
-__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
-version_info = tuple(int(v) if v.isdigit() else v
-                     for v in __version__.split('.'))
-del pkgutil
-
-# Check minimum required Python version
 import sys
-if sys.version_info < (3, 5):
-    print("Scrapy %s requires Python 3.5" % __version__)
-    sys.exit(1)
-
-# Ignore noisy twisted deprecation warnings
 import warnings
-warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
-del warnings
-
-# Apply monkey patches to fix issues in external libraries
-from scrapy import _monkeypatches
-del _monkeypatches
 
 from twisted import version as _txv
-twisted_version = (_txv.major, _txv.minor, _txv.micro)
 
 # Declare top-level shortcuts
 from scrapy.spiders import Spider
@@ -36,4 +14,29 @@ from scrapy.http import Request, FormRequest
 from scrapy.selector import Selector
 from scrapy.item import Item, Field
 
+
+__all__ = [
+    '__version__', 'version_info', 'twisted_version', 'Spider',
+    'Request', 'FormRequest', 'Selector', 'Item', 'Field',
+]
+
+
+# Scrapy and Twisted versions
+__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
+version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.'))
+twisted_version = (_txv.major, _txv.minor, _txv.micro)
+
+
+# Check minimum required Python version
+if sys.version_info < (3, 5):
+    print("Scrapy %s requires Python 3.5" % __version__)
+    sys.exit(1)
+
+
+# Ignore noisy twisted deprecation warnings
+warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
+
+
+del pkgutil
 del sys
+del warnings
diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py
deleted file mode 100644
index f74f89bda..000000000
--- a/scrapy/_monkeypatches.py
+++ /dev/null
@@ -1,11 +0,0 @@
-import copyreg
-
-
-# Undo what Twisted's perspective broker adds to pickle register
-# to prevent bugs like Twisted#7989 while serializing requests
-import twisted.persisted.styles  # NOQA
-# Remove only entries with twisted serializers for non-twisted types.
-for k, v in frozenset(copyreg.dispatch_table.items()):
-    if not str(getattr(k, '__module__', '')).startswith('twisted') \
-            and str(getattr(v, '__module__', '')).startswith('twisted'):
-        copyreg.dispatch_table.pop(k)
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index ae019c70f..984a5c4e1 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -133,4 +133,4 @@ class FilteringLinkExtractor:
 
 
 # Top-level imports
-from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor  # noqa: F401
+from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index ba1c866f8..02f87f8f5 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -110,6 +110,6 @@ class Spider(object_ref):
 
 
 # Top-level imports
-from scrapy.spiders.crawl import CrawlSpider, Rule  # noqa: F401
-from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider  # noqa: F401
-from scrapy.spiders.sitemap import SitemapSpider  # noqa: F401
+from scrapy.spiders.crawl import CrawlSpider, Rule
+from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider
+from scrapy.spiders.sitemap import SitemapSpider

From a22f97052e9b14631a977600c1f59bd468c85601 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 15 May 2020 21:18:26 -0300
Subject: [PATCH 147/243] Cleanup import in selector module

---
 pytest.ini                  | 2 +-
 scrapy/selector/__init__.py | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 0797e8ac8..663c5cc78 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -31,10 +31,10 @@ flake8-ignore =
     scrapy/core/downloader/handlers/http.py F401
     scrapy/http/__init__.py F401
     scrapy/linkextractors/__init__.py E402 F401
+    scrapy/selector/__init__.py F401
     scrapy/spiders/__init__.py E402 F401
 
     # Issues pending a review:
-    scrapy/selector/__init__.py F403
     scrapy/utils/http.py F403
     scrapy/utils/markup.py F403
     scrapy/utils/multipart.py F403
diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py
index a9240c1f6..85c500d66 100644
--- a/scrapy/selector/__init__.py
+++ b/scrapy/selector/__init__.py
@@ -1,4 +1,6 @@
 """
 Selectors
 """
-from scrapy.selector.unified import *  # noqa: F401
+
+# top-level imports
+from scrapy.selector.unified import Selector, SelectorList

From bcc40c40771ad223471ab77ad47233c498312095 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 18 May 2020 22:12:31 +0500
Subject: [PATCH 148/243] better deprecation warning for
 Response.body_as_unicode() (#4579)

---
 scrapy/http/response/text.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 5614e6e55..0603b6653 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -65,7 +65,7 @@ class TextResponse(Response):
         """Return body as unicode"""
         warnings.warn('Response.body_as_unicode() is deprecated, '
                       'please use Response.text instead.',
-                      ScrapyDeprecationWarning)
+                      ScrapyDeprecationWarning, stacklevel=2)
         return self.text
 
     @property

From a7d070f3bb350cbe1f7b580350d5f491f59d47d8 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Mon, 18 May 2020 22:25:29 +0300
Subject: [PATCH 149/243] Change log level to error

---
 scrapy/extensions/feedexport.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index cc26ae173..ce7fc372d 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -364,7 +364,7 @@ class FeedExporter:
         """
         if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
             return True
-        logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified')
+        logger.error('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified')
         return False
 
     def _storage_supported(self, uri):

From afb2e5011fb9d9aa0ca568d376b0c4cef4193a9e Mon Sep 17 00:00:00 2001
From: nsirletti <n.sirletti@gmail.com>
Date: Sat, 16 May 2020 12:48:21 +0200
Subject: [PATCH 150/243] Allow doc to be downloadable on readthedocs.org

---
 .readthedocs.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.readthedocs.yml b/.readthedocs.yml
index 17eba34f3..e4d3f02cc 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -1,4 +1,5 @@
 version: 2
+formats: all
 sphinx:
   configuration: docs/conf.py
   fail_on_warning: true

From c2a0cca0fe6bc8342efa6034a78b9b8161aa2177 Mon Sep 17 00:00:00 2001
From: nsirletti <n.sirletti@gmail.com>
Date: Sat, 16 May 2020 15:07:16 +0200
Subject: [PATCH 151/243] Suppress Sphinx warnings about MIME types other than
 .rst

---
 docs/conf.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/conf.py b/docs/conf.py
index 8ab38a090..468c1d190 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -100,6 +100,9 @@ exclude_trees = ['.build']
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'sphinx'
 
+# List of Sphinx warnings that will not be raised
+suppress_warnings = ['epub.unknown_project_files']
+
 
 # Options for HTML output
 # -----------------------

From f418c6d5f7307c3beae57dfc519be04657a8e1b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 19 May 2020 20:04:47 +0200
Subject: [PATCH 152/243] logging.rst: remove unused, misleading import

---
 docs/topics/logging.rst | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 675e65ef1..e81091651 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -262,7 +262,6 @@ scrapy.utils.log module
     This is an example on how to redirect ``INFO`` or higher messages to a file::
 
         import logging
-        from scrapy.utils.log import configure_logging
 
         logging.basicConfig(
             filename='log.txt',

From eb8493f09c56dea4dad129882fbf7b4a4da08e1a Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Wed, 20 May 2020 09:21:34 -0400
Subject: [PATCH 153/243] Incompatible with pytest 5.4 and 5.4.1

---
 tests/requirements-py3.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index d207c5fb0..05b794d81 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -2,7 +2,7 @@
 jmespath
 mitmproxy; python_version >= '3.6'
 mitmproxy<4.0.0; python_version < '3.6'
-pytest < 5.4
+pytest != 5.4, != 5.4.1
 pytest-cov
 pytest-twisted >= 1.11
 pytest-xdist

From 276721a5dc96a4239645db1a78fe056df671e10e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 9 May 2020 22:02:40 -0300
Subject: [PATCH 154/243] Stop response download

---
 docs/faq.rst                              | 13 +++++
 docs/topics/exceptions.rst                | 46 +++++++++++++++---
 docs/topics/request-response.rst          | 49 ++++++++++++++++++-
 docs/topics/signals.rst                   | 11 ++++-
 scrapy/core/downloader/handlers/http11.py | 59 +++++++++++++----------
 scrapy/core/scraper.py                    | 10 +++-
 scrapy/exceptions.py                      | 13 +++++
 scrapy/utils/signal.py                    |  7 +--
 tests/spiders.py                          | 33 +++++++++++++
 tests/test_crawl.py                       | 40 +++++++++++++--
 tests/test_engine.py                      | 57 ++++++++++++++++++++--
 11 files changed, 291 insertions(+), 47 deletions(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index 0b6bd6a86..c06cb945b 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -371,6 +371,19 @@ Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switch
 different reactor is possible by using the :setting:`TWISTED_REACTOR` setting.
 
 
+.. _faq-stop-response-download:
+
+How can I cancel the download of a given response?
+--------------------------------------------------
+
+In some situations, it might be useful to stop the download of a certain response.
+For instance, if you only need the first part of a large response and you would like
+to save resources by avoiding the download of the whole body.
+In that case, you could attach a handler to the :class:`~scrapy.signals.bytes_received`
+signal and raise a :exc:`~scrapy.exceptions.StopDownload` exception. Please refer to
+the :ref:`topics-stop-response-download` topic for additional information and examples.
+
+
 .. _has been reported: https://github.com/scrapy/scrapy/issues/2905
 .. _user agents: https://en.wikipedia.org/wiki/User_agent
 .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst
index 09cb8ed66..10b23edc4 100644
--- a/docs/topics/exceptions.rst
+++ b/docs/topics/exceptions.rst
@@ -14,13 +14,6 @@ Built-in Exceptions reference
 
 Here's a list of all exceptions included in Scrapy and their usage.
 
-DropItem
---------
-
-.. exception:: DropItem
-
-The exception that must be raised by item pipeline stages to stop processing an
-Item. For more information see :ref:`topics-item-pipeline`.
 
 CloseSpider
 -----------
@@ -47,6 +40,14 @@ DontCloseSpider
 This exception can be raised in a :signal:`spider_idle` signal handler to
 prevent the spider from being closed.
 
+DropItem
+--------
+
+.. exception:: DropItem
+
+The exception that must be raised by item pipeline stages to stop processing an
+Item. For more information see :ref:`topics-item-pipeline`.
+
 IgnoreRequest
 -------------
 
@@ -77,3 +78,34 @@ NotSupported
 
 This exception is raised to indicate an unsupported feature.
 
+StopDownload
+-------------
+
+.. versionadded:: 2.2
+
+.. exception:: StopDownload(fail=True)
+
+Raised from a :class:`~scrapy.signals.bytes_received` signal handler to
+indicate that no further bytes should be downloaded for a response.
+
+The ``fail`` boolean parameter controls which method will handle the resulting
+response:
+
+* If ``fail=True``, the request errback is called. The response object is
+  available as the ``response`` attribute of the received
+  :class:`~twisted.python.failure.Failure` object. This is the default behaviour.
+
+* If ``fail=False``, the request callback is called instead.
+
+In both cases, the response could have its body truncated: the body contains
+all bytes received up until the exception is raised, including the bytes
+received in the signal handler that raises the exception. Also, the response
+object is marked with ``"download_stopped"`` in its :attr:`Response.flags`
+attribute.
+
+.. note:: ``fail`` is a keyword-only parameter, i.e. raising
+    ``StopDownload(False)`` or ``StopDownload(True)`` will raise
+    a :class:`TypeError`.
+
+See the documentation for the :class:`~scrapy.signals.bytes_received` signal
+and the :ref:`topics-stop-response-download` topic for additional information and examples.
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 15a83f453..ae25ff7e4 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -385,6 +385,51 @@ The meta key is used set retry times per request. When initialized, the
 :reqmeta:`max_retry_times` meta key takes higher precedence over the
 :setting:`RETRY_TIMES` setting.
 
+
+.. _topics-stop-response-download:
+
+Stopping the download of a Response
+===================================
+
+Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a
+:class:`~scrapy.signals.bytes_received` signal handler will stop the
+download of a given response. See the following example::
+
+    import scrapy
+
+
+    class StopSpider(scrapy.Spider):
+        name = "stop"
+        start_urls = ["https://docs.scrapy.org/en/latest/"]
+
+        @classmethod
+        def from_crawler(cls, crawler):
+            spider = super().from_crawler(crawler)
+            crawler.signals.connect(spider.on_bytes_received, signal=scrapy.signals.bytes_received)
+            return spider
+
+        def parse(self, response):
+            # 'last_chars' show that the full response was not downloaded
+            yield {"len": len(response.text), "last_chars": response.text[-40:]}
+
+        def on_bytes_received(self, data, request, spider):
+            raise scrapy.exceptions.StopDownload(fail=False)
+
+which produces the following output::
+
+    2020-05-19 17:26:12 [scrapy.core.engine] INFO: Spider opened
+    2020-05-19 17:26:12 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
+    2020-05-19 17:26:13 [scrapy.core.downloader.handlers.http11] DEBUG: Download stopped for <GET https://docs.scrapy.org/en/latest/> from signal handler StopSpider.on_bytes_received
+    2020-05-19 17:26:13 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://docs.scrapy.org/en/latest/> (referer: None) ['download_stopped']
+    2020-05-19 17:26:13 [scrapy.core.scraper] DEBUG: Scraped from <200 https://docs.scrapy.org/en/latest/>
+    {'len': 279, 'last_chars': 'dth, initial-scale=1.0">\n  \n  <title>Scr'}
+    2020-05-19 17:26:13 [scrapy.core.engine] INFO: Closing spider (finished)
+
+By default, resulting responses are handled by their corresponding errbacks. To
+call their callback instead, like in this example, pass ``fail=False`` to the
+:exc:`~scrapy.exceptions.StopDownload` exception.
+
+
 .. _topics-request-response-ref-request-subclasses:
 
 Request subclasses
@@ -716,9 +761,9 @@ Response objects
         .. versionadded:: 2.1.0
 
         The IP address of the server from which the Response originated.
-        
+
         This attribute is currently only populated by the HTTP 1.1 download
-        handler, i.e. for ``http(s)`` responses. For other handlers, 
+        handler, i.e. for ``http(s)`` responses. For other handlers,
         :attr:`ip_address` is always ``None``.
 
     .. method:: Response.copy()
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 7fe63a7b0..fe4fb0834 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -373,6 +373,8 @@ request_left_downloader
 bytes_received
 ~~~~~~~~~~~~~~
 
+.. versionadded:: 2.2
+
 .. signal:: bytes_received
 .. function:: bytes_received(data, request, spider)
 
@@ -385,14 +387,19 @@ bytes_received
     This signal does not support returning deferreds from its handlers.
 
     :param data: the data received by the download handler
-    :type spider: :class:`bytes` object
+    :type data: :class:`bytes` object
 
-    :param request: the request that generated the response
+    :param request: the request that generated the download
     :type request: :class:`~scrapy.http.Request` object
 
     :param spider: the spider associated with the response
     :type spider: :class:`~scrapy.spiders.Spider` object
 
+.. note:: Handlers of this signal can stop the download of a response while it
+    is in progress by raising the :exc:`~scrapy.exceptions.StopDownload`
+    exception. Please refer to the :ref:`topics-stop-response-download` topic
+    for additional information and examples.
+
 Response signals
 ----------------
 
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index c21491f52..555a3a032 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -12,6 +12,7 @@ from urllib.parse import urldefrag
 from twisted.internet import defer, protocol, ssl
 from twisted.internet.endpoints import TCP4ClientEndpoint
 from twisted.internet.error import TimeoutError
+from twisted.python.failure import Failure
 from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI
 from twisted.web.http import _DataLoss, PotentialDataLoss
 from twisted.web.http_headers import Headers as TxHeaders
@@ -21,7 +22,7 @@ from zope.interface import implementer
 from scrapy import signals
 from scrapy.core.downloader.tls import openssl_methods
 from scrapy.core.downloader.webclient import _parse
-from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload
 from scrapy.http import Headers
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.misc import create_instance, load_object
@@ -431,7 +432,7 @@ class ScrapyAgent:
     def _cb_bodydone(self, result, request, url):
         headers = Headers(result["txresponse"].headers.getAllRawHeaders())
         respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
-        return respcls(
+        response = respcls(
             url=url,
             status=int(result["txresponse"].code),
             headers=headers,
@@ -440,6 +441,14 @@ class ScrapyAgent:
             certificate=result["certificate"],
             ip_address=result["ip_address"],
         )
+        if result.get("failure"):
+            # This failure is not the same object that will reach the errback,
+            # so we need to temporarily store the response in the exception.
+            # It will be moved to the failure in core/scraper.py
+            failure = result["failure"]
+            failure.value.response = response
+            return failure
+        return response
 
 
 @implementer(IBodyProducer)
@@ -477,6 +486,16 @@ class _ResponseReader(protocol.Protocol):
         self._ip_address = None
         self._crawler = crawler
 
+    def _finish_response(self, flags=None, failure=None):
+        self._finished.callback({
+            "txresponse": self._txresponse,
+            "body": self._bodybuf.getvalue(),
+            "flags": flags,
+            "certificate": self._certificate,
+            "ip_address": self._ip_address,
+            "failure": failure,
+        })
+
     def connectionMade(self):
         if self._certificate is None:
             with suppress(AttributeError):
@@ -493,12 +512,21 @@ class _ResponseReader(protocol.Protocol):
         self._bodybuf.write(bodyBytes)
         self._bytes_received += len(bodyBytes)
 
-        self._crawler.signals.send_catch_log(
+        bytes_received_result = self._crawler.signals.send_catch_log(
             signal=signals.bytes_received,
             data=bodyBytes,
             request=self._request,
             spider=self._crawler.spider,
         )
+        for handler, result in bytes_received_result:
+            if isinstance(result, Failure) and isinstance(result.value, StopDownload):
+                logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
+                             {"request": self._request, "handler": handler.__qualname__})
+                self.transport._producer.loseConnection()
+                self._finish_response(
+                    flags=["download_stopped"],
+                    failure=result if result.value.fail else None,
+                )
 
         if self._maxsize and self._bytes_received > self._maxsize:
             logger.error("Received (%(bytes)s) bytes larger than download "
@@ -521,36 +549,17 @@ class _ResponseReader(protocol.Protocol):
         if self._finished.called:
             return
 
-        body = self._bodybuf.getvalue()
         if reason.check(ResponseDone):
-            self._finished.callback({
-                "txresponse": self._txresponse,
-                "body": body,
-                "flags": None,
-                "certificate": self._certificate,
-                "ip_address": self._ip_address,
-            })
+            self._finish_response()
             return
 
         if reason.check(PotentialDataLoss):
-            self._finished.callback({
-                "txresponse": self._txresponse,
-                "body": body,
-                "flags": ["partial"],
-                "certificate": self._certificate,
-                "ip_address": self._ip_address,
-            })
+            self._finish_response(flags=["partial"])
             return
 
         if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):
             if not self._fail_on_dataloss:
-                self._finished.callback({
-                    "txresponse": self._txresponse,
-                    "body": body,
-                    "flags": ["dataloss"],
-                    "certificate": self._certificate,
-                    "ip_address": self._ip_address,
-                })
+                self._finish_response(flags=["dataloss"])
                 return
 
             elif not self._fail_on_dataloss_warned:
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 6785e103d..c52ada83b 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -11,7 +11,7 @@ from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errba
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
 from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
-from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
+from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest, StopDownload
 from scrapy import signals
 from scrapy.http import Request, Response
 from scrapy.item import _BaseItem
@@ -147,6 +147,14 @@ class Scraper:
 
     def call_spider(self, result, request, spider):
         result.request = request
+        # StopDownload exceptions: make the partial response an attribute of the failure
+        if (
+            isinstance(result, Failure)
+            and isinstance(result.value, StopDownload)
+            and hasattr(result.value, "response")
+        ):
+            result.response = result.value.response
+            delattr(result.value, "response")
         dfd = defer_result(result)
         callback = request.callback or spider.parse
         warn_on_generator_with_return_value(spider, callback)
diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py
index 7c4bb3d00..45f152321 100644
--- a/scrapy/exceptions.py
+++ b/scrapy/exceptions.py
@@ -41,6 +41,18 @@ class CloseSpider(Exception):
         self.reason = reason
 
 
+class StopDownload(Exception):
+    """
+    Stop the download of the body for a given response.
+    The 'fail' boolean parameter indicates whether or not the resulting partial response
+    should be handled by the request errback. Note that 'fail' is a keyword-only argument.
+    """
+
+    def __init__(self, *, fail=True):
+        super().__init__()
+        self.fail = fail
+
+
 # Items
 
 
@@ -59,6 +71,7 @@ class NotSupported(Exception):
 
 class UsageError(Exception):
     """To indicate a command-line usage error"""
+
     def __init__(self, *a, **kw):
         self.print_help = kw.pop('print_help', True)
         super(UsageError, self).__init__(*a, **kw)
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index a311e9257..115707182 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -5,13 +5,14 @@ import logging
 from twisted.internet.defer import DeferredList, Deferred
 from twisted.python.failure import Failure
 
-from pydispatch.dispatcher import Any, Anonymous, liveReceivers, \
-    getAllReceivers, disconnect
+from pydispatch.dispatcher import Anonymous, Any, disconnect, getAllReceivers, liveReceivers
 from pydispatch.robustapply import robustApply
 
+from scrapy.exceptions import StopDownload
 from scrapy.utils.defer import maybeDeferred_coro
 from scrapy.utils.log import failure_to_exc_info
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -23,7 +24,7 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
     """Like pydispatcher.robust.sendRobust but it also logs errors and returns
     Failures instead of exceptions.
     """
-    dont_log = named.pop('dont_log', _IgnoredException)
+    dont_log = (named.pop('dont_log', _IgnoredException), StopDownload)
     spider = named.get('spider', None)
     responses = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
diff --git a/tests/spiders.py b/tests/spiders.py
index 33d5d02e1..ebe81a1a9 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -7,6 +7,8 @@ from urllib.parse import urlencode
 
 from twisted.internet import defer
 
+from scrapy import signals
+from scrapy.exceptions import StopDownload
 from scrapy.http import Request
 from scrapy.item import Item
 from scrapy.linkextractors import LinkExtractor
@@ -267,3 +269,34 @@ class CrawlSpiderWithErrback(MockServerSpider, CrawlSpider):
 
     def errback(self, failure):
         self.logger.info('[errback] status %i', failure.value.response.status)
+
+
+class BytesReceivedCallbackSpider(MetaSpider):
+
+    full_response_length = 2**18
+
+    @classmethod
+    def from_crawler(cls, crawler, *args, **kwargs):
+        spider = super().from_crawler(crawler, *args, **kwargs)
+        crawler.signals.connect(spider.bytes_received, signals.bytes_received)
+        return spider
+
+    def start_requests(self):
+        body = b"a" * self.full_response_length
+        url = self.mockserver.url("/alpayload")
+        yield Request(url, method="POST", body=body, errback=self.errback)
+
+    def parse(self, response):
+        self.meta["response"] = response
+
+    def errback(self, failure):
+        self.meta["failure"] = failure
+
+    def bytes_received(self, data, request, spider):
+        raise StopDownload(fail=False)
+
+
+class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider):
+
+    def bytes_received(self, data, request, spider):
+        raise StopDownload(fail=True)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 84f80d103..5e70064c2 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -9,17 +9,31 @@ from pytest import mark
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.internet.ssl import Certificate
+from twisted.python.failure import Failure
 from twisted.trial.unittest import TestCase
 
 from scrapy import signals
 from scrapy.crawler import CrawlerRunner
+from scrapy.exceptions import StopDownload
 from scrapy.http import Request
+from scrapy.http.response import Response
 from scrapy.utils.python import to_unicode
 from tests.mockserver import MockServer
-from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider,
-                           SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback,
-                           AsyncDefSpider, AsyncDefAsyncioSpider, AsyncDefAsyncioReturnSpider,
-                           AsyncDefAsyncioReqsReturnSpider)
+from tests.spiders import (
+    AsyncDefAsyncioReqsReturnSpider,
+    AsyncDefAsyncioReturnSpider,
+    AsyncDefAsyncioSpider,
+    AsyncDefSpider,
+    BrokenStartRequestsSpider,
+    BytesReceivedCallbackSpider,
+    BytesReceivedErrbackSpider,
+    CrawlSpiderWithErrback,
+    DelaySpider,
+    DuplicateStartRequestsSpider,
+    FollowAllSpider,
+    SimpleSpider,
+    SingleRequestSpider,
+)
 
 
 class CrawlTestCase(TestCase):
@@ -457,3 +471,21 @@ with multiples lines
         ip_address = crawler.spider.meta['responses'][0].ip_address
         self.assertIsInstance(ip_address, IPv4Address)
         self.assertEqual(str(ip_address), gethostbyname(expected_netloc))
+
+    @defer.inlineCallbacks
+    def test_stop_download_callback(self):
+        crawler = self.runner.create_crawler(BytesReceivedCallbackSpider)
+        yield crawler.crawl(mockserver=self.mockserver)
+        self.assertIsNone(crawler.spider.meta.get("failure"))
+        self.assertIsInstance(crawler.spider.meta["response"], Response)
+        self.assertLess(len(crawler.spider.meta["response"].text), crawler.spider.full_response_length)
+
+    @defer.inlineCallbacks
+    def test_stop_download_errback(self):
+        crawler = self.runner.create_crawler(BytesReceivedErrbackSpider)
+        yield crawler.crawl(mockserver=self.mockserver)
+        self.assertIsNone(crawler.spider.meta.get("response"))
+        self.assertIsInstance(crawler.spider.meta["failure"], Failure)
+        self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload)
+        self.assertIsInstance(crawler.spider.meta["failure"].response, Response)
+        self.assertLess(len(crawler.spider.meta["failure"].response.text), crawler.spider.full_response_length)
diff --git a/tests/test_engine.py b/tests/test_engine.py
index d781665dc..6696ee52e 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -16,13 +16,15 @@ import sys
 from collections import defaultdict
 from urllib.parse import urlparse
 
+from pydispatch import dispatcher
+from testfixtures import LogCapture
 from twisted.internet import reactor, defer
 from twisted.trial import unittest
 from twisted.web import server, static, util
-from pydispatch import dispatcher
 
 from scrapy import signals
 from scrapy.core.engine import ExecutionEngine
+from scrapy.exceptions import StopDownload
 from scrapy.http import Request
 from scrapy.item import Item, Field
 from scrapy.linkextractors import LinkExtractor
@@ -90,7 +92,7 @@ def start_test_site(debug=False):
     r = static.File(root_dir)
     r.putChild(b"redirect", util.Redirect(b"/redirected"))
     r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
-    numbers = [str(x).encode("utf8") for x in range(2**14)]
+    numbers = [str(x).encode("utf8") for x in range(2**18)]
     r.putChild(b"numbers", static.Data(b"".join(numbers), "text/plain"))
 
     port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1")
@@ -188,6 +190,16 @@ class CrawlerRun:
         self.signals_caught[sig] = signalargs
 
 
+class StopDownloadCrawlerRun(CrawlerRun):
+    """
+    Make sure raising the StopDownload exception stops the download of the response body
+    """
+
+    def bytes_received(self, data, request, spider):
+        super().bytes_received(data, request, spider)
+        raise StopDownload(fail=False)
+
+
 class EngineTest(unittest.TestCase):
 
     @defer.inlineCallbacks
@@ -316,7 +328,7 @@ class EngineTest(unittest.TestCase):
                 # signal was fired multiple times
                 self.assertTrue(len(data) > 1)
                 # bytes were received in order
-                numbers = [str(x).encode("utf8") for x in range(2**14)]
+                numbers = [str(x).encode("utf8") for x in range(2**18)]
                 self.assertEqual(joined_data, b"".join(numbers))
 
     def _assert_signals_caught(self):
@@ -357,6 +369,45 @@ class EngineTest(unittest.TestCase):
         self.assertEqual(len(e.open_spiders), 0)
 
 
+class StopDownloadEngineTest(EngineTest):
+
+    @defer.inlineCallbacks
+    def test_crawler(self):
+        for spider in TestSpider, DictItemsSpider:
+            self.run = StopDownloadCrawlerRun(spider)
+            with LogCapture() as log:
+                yield self.run.run()
+                log.check_present(("scrapy.core.downloader.handlers.http11",
+                                   "DEBUG",
+                                   "Download stopped for <GET http://localhost:{}/redirected> from signal handler"
+                                   " StopDownloadCrawlerRun.bytes_received".format(self.run.portno)))
+                log.check_present(("scrapy.core.downloader.handlers.http11",
+                                   "DEBUG",
+                                   "Download stopped for <GET http://localhost:{}/> from signal handler"
+                                   " StopDownloadCrawlerRun.bytes_received".format(self.run.portno)))
+                log.check_present(("scrapy.core.downloader.handlers.http11",
+                                   "DEBUG",
+                                   "Download stopped for <GET http://localhost:{}/numbers> from signal handler"
+                                   " StopDownloadCrawlerRun.bytes_received".format(self.run.portno)))
+            self._assert_visited_urls()
+            self._assert_scheduled_requests(urls_to_visit=9)
+            self._assert_downloaded_responses()
+            self._assert_signals_caught()
+            self._assert_bytes_received()
+
+    def _assert_bytes_received(self):
+        self.assertEqual(9, len(self.run.bytes))
+        for request, data in self.run.bytes.items():
+            joined_data = b"".join(data)
+            self.assertTrue(len(data) == 1)  # signal was fired only once
+            if self.run.getpath(request.url) == "/numbers":
+                # Received bytes are not the complete response. The exact amount depends
+                # on the buffer size, which can vary, so we only check that the amount
+                # of received bytes is strictly less than the full response.
+                numbers = [str(x).encode("utf8") for x in range(2**18)]
+                self.assertTrue(len(joined_data) < len(b"".join(numbers)))
+
+
 if __name__ == "__main__":
     if len(sys.argv) > 1 and sys.argv[1] == 'runserver':
         start_test_site(debug=True)

From 9514393bd1c9137c6a3b9964679de04bccbd0570 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 20 May 2020 15:28:54 -0300
Subject: [PATCH 155/243] Reduce amount of lines

---
 scrapy/core/downloader/handlers/http11.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 555a3a032..a710761c5 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -523,10 +523,8 @@ class _ResponseReader(protocol.Protocol):
                 logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
                              {"request": self._request, "handler": handler.__qualname__})
                 self.transport._producer.loseConnection()
-                self._finish_response(
-                    flags=["download_stopped"],
-                    failure=result if result.value.fail else None,
-                )
+                failure = result if result.value.fail else None
+                self._finish_response(flags=["download_stopped"], failure=failure)
 
         if self._maxsize and self._bytes_received > self._maxsize:
             logger.error("Received (%(bytes)s) bytes larger than download "

From fbe915d9cbd8ab801a679b05265cff416764d558 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 20 May 2020 20:20:30 -0300
Subject: [PATCH 156/243] StopDownload: check partial response contents

---
 tests/spiders.py    | 2 ++
 tests/test_crawl.py | 6 ++++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/spiders.py b/tests/spiders.py
index ebe81a1a9..05078cc04 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -293,10 +293,12 @@ class BytesReceivedCallbackSpider(MetaSpider):
         self.meta["failure"] = failure
 
     def bytes_received(self, data, request, spider):
+        self.meta["bytes_received"] = data
         raise StopDownload(fail=False)
 
 
 class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider):
 
     def bytes_received(self, data, request, spider):
+        self.meta["bytes_received"] = data
         raise StopDownload(fail=True)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 5e70064c2..e2426e0da 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -478,7 +478,8 @@ with multiples lines
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertIsNone(crawler.spider.meta.get("failure"))
         self.assertIsInstance(crawler.spider.meta["response"], Response)
-        self.assertLess(len(crawler.spider.meta["response"].text), crawler.spider.full_response_length)
+        self.assertEqual(crawler.spider.meta["response"].body, crawler.spider.meta.get("bytes_received"))
+        self.assertLess(len(crawler.spider.meta["response"].body), crawler.spider.full_response_length)
 
     @defer.inlineCallbacks
     def test_stop_download_errback(self):
@@ -488,4 +489,5 @@ with multiples lines
         self.assertIsInstance(crawler.spider.meta["failure"], Failure)
         self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload)
         self.assertIsInstance(crawler.spider.meta["failure"].response, Response)
-        self.assertLess(len(crawler.spider.meta["failure"].response.text), crawler.spider.full_response_length)
+        self.assertEqual(crawler.spider.meta["failure"].response.body, crawler.spider.meta.get("bytes_received"))
+        self.assertLess(len(crawler.spider.meta["failure"].response.body), crawler.spider.full_response_length)

From 677e619d3761e6669c247786bb95822ce38c8080 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 21 May 2020 14:57:03 +0300
Subject: [PATCH 157/243] Fix too long lines

---
 scrapy/extensions/feedexport.py |  4 +++-
 tests/test_feedexport.py        | 20 ++++++++++++++------
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index ce7fc372d..1f745be98 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -364,7 +364,9 @@ class FeedExporter:
         """
         if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
             return True
-        logger.error('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified')
+        logger.error(
+            '%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified'
+        )
         return False
 
     def _storage_supported(self, uri):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 08ee24768..fecb17e29 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1204,12 +1204,20 @@ class BatchDeliveriesTest(FeedExportTestBase):
         items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]
 
         formats = {
-            'json': [u'[\n{"bar": "BAR"}\n]'.encode('utf-8'),
-                     u'[\n{"bar": "BAR1"}\n]'.encode('utf-8')],
-            'xml': [u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'.encode('latin-1'),
-                    u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO1</foo>\n  </item>\n</items>'.encode('latin-1')],
-            'csv': [u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
-                    u'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')],
+            'json': ['[\n{"bar": "BAR"}\n]'.encode('utf-8'),
+                     '[\n{"bar": "BAR1"}\n]'.encode('utf-8')],
+            'xml': [
+                (
+                    '<?xml version="1.0" encoding="latin-1"?>\n'
+                    '<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'
+                ).encode('latin-1'),
+                (
+                    '<?xml version="1.0" encoding="latin-1"?>\n'
+                    '<items>\n  <item>\n    <foo>FOO1</foo>\n  </item>\n</items>'
+                ).encode('latin-1')
+            ],
+            'csv': ['bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
+                    'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')],
         }
 
         settings = {

From f6ed5edc31e7cc66225c0860e1534a6230511954 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 18 Nov 2016 09:14:54 -0300
Subject: [PATCH 158/243] CookiesMiddleware: keep cookies from 'Cookie' request
 header

---
 docs/topics/downloader-middleware.rst      |   5 +
 docs/topics/logging.rst                    |   3 +
 scrapy/downloadermiddlewares/cookies.py    |  78 ++++++++---
 tests/test_downloadermiddleware_cookies.py | 145 ++++++++++++++++++---
 4 files changed, 190 insertions(+), 41 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 1a87d07b6..323e553e5 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -202,6 +202,11 @@ CookiesMiddleware
    sends them back on subsequent requests (from that spider), just like web
    browsers do.
 
+   .. caution:: When non-UTF8 encoded byte sequences are passed to a
+      :class:`~scrapy.http.Request`, the ``CookiesMiddleware`` will log
+      a warning. Refer to :ref:`topics-logging-advanced-customization`
+      to customize the logging behaviour.
+
 The following settings can be used to configure the cookie middleware:
 
 * :setting:`COOKIES_ENABLED`
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index e81091651..55065a1a3 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -202,6 +202,9 @@ A custom log format can be set for different actions by extending
 .. autoclass:: scrapy.logformatter.LogFormatter
    :members:
 
+
+.. _topics-logging-advanced-customization:
+
 Advanced customization
 ----------------------
 
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index d57f04bc3..77048f389 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -29,8 +29,7 @@ class CookiesMiddleware:
 
         cookiejarkey = request.meta.get("cookiejar")
         jar = self.jars[cookiejarkey]
-        cookies = self._get_request_cookies(jar, request)
-        for cookie in cookies:
+        for cookie in self._get_request_cookies(jar, request):
             jar.set_cookie_if_ok(cookie, request)
 
         # set Cookie header
@@ -68,28 +67,65 @@ class CookiesMiddleware:
                 msg = "Received cookies from: {}\n{}".format(response, cookies)
                 logger.debug(msg, extra={'spider': spider})
 
-    def _format_cookie(self, cookie):
-        # build cookie string
-        cookie_str = '%s=%s' % (cookie['name'], cookie['value'])
-
-        if cookie.get('path', None):
-            cookie_str += '; Path=%s' % cookie['path']
-        if cookie.get('domain', None):
-            cookie_str += '; Domain=%s' % cookie['domain']
+    def _format_cookie(self, cookie, request):
+        """
+        Given a dict consisting of cookie components, return its string representation.
+        Decode from bytes if necessary.
+        """
+        decoded = {}
+        for key in ("name", "value", "path", "domain"):
+            if not cookie.get(key):
+                if key in ("name", "value"):
+                    msg = "Invalid cookie found in request {}: {} ('{}' is missing)"
+                    logger.warning(msg.format(request, cookie, key))
+                    return
+                continue
+            if isinstance(cookie[key], str):
+                decoded[key] = cookie[key]
+            else:
+                try:
+                    decoded[key] = cookie[key].decode("utf8")
+                except UnicodeDecodeError:
+                    logger.warning("Non UTF-8 encoded cookie found in request %s: %s",
+                                   request, cookie)
+                    decoded[key] = cookie[key].decode("latin1", errors="replace")
 
+        cookie_str = "{}={}".format(decoded.pop("name"), decoded.pop("value"))
+        for key, value in decoded.items():  # path, domain
+            cookie_str += "; {}={}".format(key.capitalize(), value)
         return cookie_str
 
     def _get_request_cookies(self, jar, request):
-        if isinstance(request.cookies, dict):
-            cookie_list = [
-                {'name': k, 'value': v}
-                for k, v in request.cookies.items()
-            ]
-        else:
-            cookie_list = request.cookies
+        """
+        Extract cookies from a Request. Values from the `Request.cookies` attribute
+        take precedence over values from the `Cookie` request header.
+        """
+        def get_cookies_from_header(jar, request):
+            cookie_header = request.headers.get("Cookie")
+            if not cookie_header:
+                return []
+            cookie_gen_bytes = (s.strip() for s in cookie_header.split(b";"))
+            cookie_list_unicode = []
+            for cookie_bytes in cookie_gen_bytes:
+                try:
+                    cookie_unicode = cookie_bytes.decode("utf8")
+                except UnicodeDecodeError:
+                    logger.warning("Non UTF-8 encoded cookie found in request %s: %s",
+                                   request, cookie_bytes)
+                    cookie_unicode = cookie_bytes.decode("latin1", errors="replace")
+                cookie_list_unicode.append(cookie_unicode)
+            response = Response(request.url, headers={"Set-Cookie": cookie_list_unicode})
+            return jar.make_cookies(response, request)
 
-        cookies = [self._format_cookie(x) for x in cookie_list]
-        headers = {'Set-Cookie': cookies}
-        response = Response(request.url, headers=headers)
+        def get_cookies_from_attribute(jar, request):
+            if not request.cookies:
+                return []
+            elif isinstance(request.cookies, dict):
+                cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
+            else:
+                cookies = request.cookies
+            formatted = filter(None, (self._format_cookie(c, request) for c in cookies))
+            response = Response(request.url, headers={"Set-Cookie": formatted})
+            return jar.make_cookies(response, request)
 
-        return jar.make_cookies(response, request)
+        return get_cookies_from_header(jar, request) + get_cookies_from_attribute(jar, request)
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index d54434c8f..9ccc2110b 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -1,20 +1,21 @@
-import re
 import logging
-from unittest import TestCase
 from testfixtures import LogCapture
+from unittest import TestCase
 
+from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
+from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
+from scrapy.exceptions import NotConfigured
 from scrapy.http import Response, Request
 from scrapy.spiders import Spider
+from scrapy.utils.python import to_bytes
 from scrapy.utils.test import get_crawler
-from scrapy.exceptions import NotConfigured
-from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
 
 
 class CookiesMiddlewareTest(TestCase):
 
     def assertCookieValEqual(self, first, second, msg=None):
         def split_cookies(cookies):
-            return sorted(re.split(r";\s*", cookies.decode("latin1")))
+            return sorted([s.strip() for s in to_bytes(cookies).split(b";")])
         return self.assertEqual(split_cookies(first), split_cookies(second), msg=msg)
 
     def setUp(self):
@@ -61,12 +62,13 @@ class CookiesMiddlewareTest(TestCase):
     def test_setting_enabled_cookies_debug(self):
         crawler = get_crawler(settings_dict={'COOKIES_DEBUG': True})
         mw = CookiesMiddleware.from_crawler(crawler)
-        with LogCapture('scrapy.downloadermiddlewares.cookies',
-                        propagate=False,
-                        level=logging.DEBUG) as log:
+        with LogCapture(
+            'scrapy.downloadermiddlewares.cookies',
+            propagate=False,
+            level=logging.DEBUG,
+        ) as log:
             req = Request('http://scrapytest.org/')
-            res = Response('http://scrapytest.org/',
-                           headers={'Set-Cookie': 'C1=value1; path=/'})
+            res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'})
             mw.process_response(req, res, crawler.spider)
             req2 = Request('http://scrapytest.org/sub1/')
             mw.process_request(req2, crawler.spider)
@@ -85,12 +87,13 @@ class CookiesMiddlewareTest(TestCase):
     def test_setting_disabled_cookies_debug(self):
         crawler = get_crawler(settings_dict={'COOKIES_DEBUG': False})
         mw = CookiesMiddleware.from_crawler(crawler)
-        with LogCapture('scrapy.downloadermiddlewares.cookies',
-                        propagate=False,
-                        level=logging.DEBUG) as log:
+        with LogCapture(
+            'scrapy.downloadermiddlewares.cookies',
+            propagate=False,
+            level=logging.DEBUG,
+        ) as log:
             req = Request('http://scrapytest.org/')
-            res = Response('http://scrapytest.org/',
-                           headers={'Set-Cookie': 'C1=value1; path=/'})
+            res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'})
             mw.process_response(req, res, crawler.spider)
             req2 = Request('http://scrapytest.org/sub1/')
             mw.process_request(req2, crawler.spider)
@@ -102,8 +105,7 @@ class CookiesMiddlewareTest(TestCase):
         assert self.mw.process_request(req, self.spider) is None
         assert 'Cookie' not in req.headers
 
-        headers = {'Set-Cookie': b'C1=in\xa3valid; path=/',
-                   'Other': b'ignore\xa3me'}
+        headers = {'Set-Cookie': b'C1=in\xa3valid; path=/', 'Other': b'ignore\xa3me'}
         res = Response('http://scrapytest.org/', headers=headers)
         assert self.mw.process_response(req, res, self.spider) is res
 
@@ -124,7 +126,10 @@ class CookiesMiddlewareTest(TestCase):
         assert 'Cookie' not in req.headers
 
         # check that returned cookies are not merged back to jar
-        res = Response('http://scrapytest.org/dontmerge', headers={'Set-Cookie': 'dont=mergeme; path=/'})
+        res = Response(
+            'http://scrapytest.org/dontmerge',
+            headers={'Set-Cookie': 'dont=mergeme; path=/'},
+        )
         assert self.mw.process_response(req, res, self.spider) is res
 
         # check that cookies are merged back
@@ -179,7 +184,11 @@ class CookiesMiddlewareTest(TestCase):
         self.assertCookieValEqual(req2.headers.get('Cookie'), b"C1=value1; galleta=salada")
 
     def test_cookiejar_key(self):
-        req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'}, meta={'cookiejar': "store1"})
+        req = Request(
+            'http://scrapytest.org/',
+            cookies={'galleta': 'salada'},
+            meta={'cookiejar': "store1"},
+        )
         assert self.mw.process_request(req, self.spider) is None
         self.assertEqual(req.headers.get('Cookie'), b'galleta=salada')
 
@@ -191,7 +200,11 @@ class CookiesMiddlewareTest(TestCase):
         assert self.mw.process_request(req2, self.spider) is None
         self.assertCookieValEqual(req2.headers.get('Cookie'), b'C1=value1; galleta=salada')
 
-        req3 = Request('http://scrapytest.org/', cookies={'galleta': 'dulce'}, meta={'cookiejar': "store2"})
+        req3 = Request(
+            'http://scrapytest.org/',
+            cookies={'galleta': 'dulce'},
+            meta={'cookiejar': "store2"},
+        )
         assert self.mw.process_request(req3, self.spider) is None
         self.assertEqual(req3.headers.get('Cookie'), b'galleta=dulce')
 
@@ -229,3 +242,95 @@ class CookiesMiddlewareTest(TestCase):
         assert self.mw.process_request(request, self.spider) is None
         self.assertIn('Cookie', request.headers)
         self.assertEqual(b'currencyCookie=USD', request.headers['Cookie'])
+
+    def test_keep_cookie_from_default_request_headers_middleware(self):
+        DEFAULT_REQUEST_HEADERS = dict(Cookie='default=value; asdf=qwerty')
+        mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items())
+        # overwrite with values from 'cookies' request argument
+        req1 = Request('http://example.org', cookies={'default': 'something'})
+        assert mw_default_headers.process_request(req1, self.spider) is None
+        assert self.mw.process_request(req1, self.spider) is None
+        self.assertCookieValEqual(req1.headers['Cookie'], b'default=something; asdf=qwerty')
+        # keep both
+        req2 = Request('http://example.com', cookies={'a': 'b'})
+        assert mw_default_headers.process_request(req2, self.spider) is None
+        assert self.mw.process_request(req2, self.spider) is None
+        self.assertCookieValEqual(req2.headers['Cookie'], b'default=value; a=b; asdf=qwerty')
+
+    def test_keep_cookie_header(self):
+        # keep only cookies from 'Cookie' request header
+        req1 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'})
+        assert self.mw.process_request(req1, self.spider) is None
+        self.assertCookieValEqual(req1.headers['Cookie'], 'a=b; c=d')
+        # keep cookies from both 'Cookie' request header and 'cookies' keyword
+        req2 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'}, cookies={'e': 'f'})
+        assert self.mw.process_request(req2, self.spider) is None
+        self.assertCookieValEqual(req2.headers['Cookie'], 'a=b; c=d; e=f')
+        # overwrite values from 'Cookie' request header with 'cookies' keyword
+        req3 = Request(
+            'http://scrapytest.org',
+            headers={'Cookie': 'a=b; c=d'},
+            cookies={'a': 'new', 'e': 'f'},
+        )
+        assert self.mw.process_request(req3, self.spider) is None
+        self.assertCookieValEqual(req3.headers['Cookie'], 'a=new; c=d; e=f')
+
+    def test_request_cookies_encoding(self):
+        # 1) UTF8-encoded bytes
+        req1 = Request('http://example.org', cookies={'a': u'á'.encode('utf8')})
+        assert self.mw.process_request(req1, self.spider) is None
+        self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
+
+        # 2) Non UTF8-encoded bytes
+        req2 = Request('http://example.org', cookies={'a': u'á'.encode('latin1')})
+        assert self.mw.process_request(req2, self.spider) is None
+        self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
+
+        # 3) Unicode string
+        req3 = Request('http://example.org', cookies={'a': u'á'})
+        assert self.mw.process_request(req3, self.spider) is None
+        self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
+
+    def test_request_headers_cookie_encoding(self):
+        # 1) UTF8-encoded bytes
+        req1 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('utf8')})
+        assert self.mw.process_request(req1, self.spider) is None
+        self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
+
+        # 2) Non UTF8-encoded bytes
+        req2 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('latin1')})
+        assert self.mw.process_request(req2, self.spider) is None
+        self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
+
+        # 3) Unicode string
+        req3 = Request('http://example.org', headers={'Cookie': u'a=á'})
+        assert self.mw.process_request(req3, self.spider) is None
+        self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
+
+    def test_invalid_cookies(self):
+        """
+        Invalid cookies are logged as warnings and discarded
+        """
+        with LogCapture(
+            'scrapy.downloadermiddlewares.cookies',
+            propagate=False,
+            level=logging.INFO,
+        ) as lc:
+            cookies1 = [{'value': 'bar'}, {'name': 'key', 'value': 'value1'}]
+            req1 = Request('http://example.org/1', cookies=cookies1)
+            assert self.mw.process_request(req1, self.spider) is None
+            cookies2 = [{'name': 'foo'}, {'name': 'key', 'value': 'value2'}]
+            req2 = Request('http://example.org/2', cookies=cookies2)
+            assert self.mw.process_request(req2, self.spider) is None
+            lc.check(
+                ("scrapy.downloadermiddlewares.cookies",
+                 "WARNING",
+                 "Invalid cookie found in request <GET http://example.org/1>:"
+                 " {'value': 'bar'} ('name' is missing)"),
+                ("scrapy.downloadermiddlewares.cookies",
+                 "WARNING",
+                 "Invalid cookie found in request <GET http://example.org/2>:"
+                 " {'name': 'foo'} ('value' is missing)"),
+            )
+        self.assertCookieValEqual(req1.headers['Cookie'], 'key=value1')
+        self.assertCookieValEqual(req2.headers['Cookie'], 'key=value2')

From 08756cd87a7699dd0f73dcdc3178671fb6fec2d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 22 May 2020 18:06:54 +0200
Subject: [PATCH 159/243] Fix the description of the ScreenshotPipeline example
 item pipeline

---
 docs/topics/item-pipeline.rst | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 533f84630..c9194caa1 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -167,11 +167,13 @@ method and how to clean up the resources properly.::
 Take screenshot of item
 -----------------------
 
-This example demonstrates how to return a
-:class:`~twisted.internet.defer.Deferred` from the :meth:`process_item` method.
-It uses Splash_ to render screenshot of item url. Pipeline
-makes request to locally running instance of Splash_. After request is downloaded,
-it saves the screenshot to a file and adds filename to the item.
+This example demonstrates how to use :doc:`coroutine syntax <coroutines>` in
+the :meth:`process_item` method.
+
+This item pipeline makes a request to a locally-running instance of Splash_ to
+render a screenshot of the item URL. After the request response is downloaded,
+the item pipeline saves the screenshot to a file and adds the filename to the
+item.
 
 ::
 

From 5b2af852892faa3bf293225e6198f0cb1bc3bd85 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 22 May 2020 15:09:58 -0400
Subject: [PATCH 160/243] Link to issue describing troublesome pytest versions

---
 tests/requirements-py3.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 05b794d81..91fa1c5b5 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -2,6 +2,7 @@
 jmespath
 mitmproxy; python_version >= '3.6'
 mitmproxy<4.0.0; python_version < '3.6'
+# https://github.com/pytest-dev/pytest-twisted/issues/93
 pytest != 5.4, != 5.4.1
 pytest-cov
 pytest-twisted >= 1.11

From dd96f94e8cc1517b7021e35e46cbdc92580c6333 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Fri, 22 May 2020 23:30:33 +0300
Subject: [PATCH 161/243] Push datetime.utcnow() to its own variable

---
 scrapy/extensions/feedexport.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 1f745be98..45c2971a6 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -398,8 +398,9 @@ class FeedExporter:
         params = {}
         for k in dir(spider):
             params[k] = getattr(spider, k)
-        params['time'] = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')
-        params['batch_time'] = datetime.utcnow().isoformat().replace(':', '-')
+        utc_now = datetime.utcnow()
+        params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-')
+        params['batch_time'] = utc_now.isoformat().replace(':', '-')
         params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
         uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
         uripar_function(params, spider)

From 0cc0e51ca330ae5d88d7daaf027d13dd8783603f Mon Sep 17 00:00:00 2001
From: Rohit Sharma <rohit_sh1@yahoo.ca>
Date: Sun, 24 May 2020 13:38:44 -0700
Subject: [PATCH 162/243] remove uneeded escape sequence

removed uneeded wscape sequences from method arguments in the docs
folder
---
 docs/topics/api.rst       |  2 +-
 docs/topics/contracts.rst |  2 +-
 docs/topics/exporters.rst | 12 ++++++------
 docs/topics/loaders.rst   | 24 ++++++++++++------------
 docs/topics/spiders.rst   |  2 +-
 5 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 1c461a511..52509ffdf 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -91,7 +91,7 @@ how you :ref:`configure the downloader middlewares
         provided while constructing the crawler, and it is created after the
         arguments given in the :meth:`crawl` method.
 
-    .. method:: crawl(\*args, \**kwargs)
+    .. method:: crawl(*args, **kwargs)
 
         Starts the crawler by instantiating its spider class with the given
         ``args`` and ``kwargs`` arguments, while setting the execution engine in
diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index 319f577bc..b8b3078c4 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -78,7 +78,7 @@ override three methods:
 
 .. module:: scrapy.contracts
 
-.. class:: Contract(method, \*args)
+.. class:: Contract(method, *args)
 
     :param method: callback function to which the contract is associated
     :type method: function
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index de8b51195..7daf25ab3 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -236,7 +236,7 @@ PythonItemExporter
 XmlItemExporter
 ---------------
 
-.. class:: XmlItemExporter(file, item_element='item', root_element='items', \**kwargs)
+.. class:: XmlItemExporter(file, item_element='item', root_element='items', **kwargs)
 
    Exports Items in XML format to the specified file object.
 
@@ -290,7 +290,7 @@ XmlItemExporter
 CsvItemExporter
 ---------------
 
-.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', \**kwargs)
+.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', **kwargs)
 
    Exports Items in CSV format to the given file-like object. If the
    :attr:`fields_to_export` attribute is set, it will be used to define the
@@ -323,7 +323,7 @@ CsvItemExporter
 PickleItemExporter
 ------------------
 
-.. class:: PickleItemExporter(file, protocol=0, \**kwargs)
+.. class:: PickleItemExporter(file, protocol=0, **kwargs)
 
    Exports Items in pickle format to the given file-like object.
 
@@ -343,7 +343,7 @@ PickleItemExporter
 PprintItemExporter
 ------------------
 
-.. class:: PprintItemExporter(file, \**kwargs)
+.. class:: PprintItemExporter(file, **kwargs)
 
    Exports Items in pretty print format to the specified file object.
 
@@ -363,7 +363,7 @@ PprintItemExporter
 JsonItemExporter
 ----------------
 
-.. class:: JsonItemExporter(file, \**kwargs)
+.. class:: JsonItemExporter(file, **kwargs)
 
    Exports Items in JSON format to the specified file-like object, writing all
    objects as a list of objects. The additional ``__init__`` method arguments are
@@ -392,7 +392,7 @@ JsonItemExporter
 JsonLinesItemExporter
 ---------------------
 
-.. class:: JsonLinesItemExporter(file, \**kwargs)
+.. class:: JsonLinesItemExporter(file, **kwargs)
 
    Exports Items in JSON format to the specified file-like object, writing one
    JSON-encoded item per line. The additional ``__init__`` method arguments are passed
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 5f75ccbff..eb804f1db 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -273,7 +273,7 @@ There are several ways to modify Item Loader context values:
 ItemLoader objects
 ==================
 
-.. class:: ItemLoader([item, selector, response], \**kwargs)
+.. class:: ItemLoader([item, selector, response], **kwargs)
 
     Return a new Item Loader for populating the given Item. If no item is
     given, one is instantiated automatically using the class in
@@ -303,7 +303,7 @@ ItemLoader objects
 
     :class:`ItemLoader` instances have the following methods:
 
-    .. method:: get_value(value, \*processors, \**kwargs)
+    .. method:: get_value(value, *processors, **kwargs)
 
         Process the given ``value`` by the given ``processors`` and keyword
         arguments.
@@ -321,7 +321,7 @@ ItemLoader objects
         >>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
         'FOO`
 
-    .. method:: add_value(field_name, value, \*processors, \**kwargs)
+    .. method:: add_value(field_name, value, *processors, **kwargs)
 
         Process and then add the given ``value`` for the given field.
 
@@ -343,11 +343,11 @@ ItemLoader objects
             loader.add_value('name', u'name: foo', TakeFirst(), re='name: (.+)')
             loader.add_value(None, {'name': u'foo', 'sex': u'male'})
 
-    .. method:: replace_value(field_name, value, \*processors, \**kwargs)
+    .. method:: replace_value(field_name, value, *processors, **kwargs)
 
         Similar to :meth:`add_value` but replaces the collected data with the
         new value instead of adding it.
-    .. method:: get_xpath(xpath, \*processors, \**kwargs)
+    .. method:: get_xpath(xpath, *processors, **kwargs)
 
         Similar to :meth:`ItemLoader.get_value` but receives an XPath instead of a
         value, which is used to extract a list of unicode strings from the
@@ -367,7 +367,7 @@ ItemLoader objects
             # HTML snippet: <p id="price">the price is $1200</p>
             loader.get_xpath('//p[@id="price"]', TakeFirst(), re='the price is (.*)')
 
-    .. method:: add_xpath(field_name, xpath, \*processors, \**kwargs)
+    .. method:: add_xpath(field_name, xpath, *processors, **kwargs)
 
         Similar to :meth:`ItemLoader.add_value` but receives an XPath instead of a
         value, which is used to extract a list of unicode strings from the
@@ -385,12 +385,12 @@ ItemLoader objects
             # HTML snippet: <p id="price">the price is $1200</p>
             loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)')
 
-    .. method:: replace_xpath(field_name, xpath, \*processors, \**kwargs)
+    .. method:: replace_xpath(field_name, xpath, *processors, **kwargs)
 
         Similar to :meth:`add_xpath` but replaces collected data instead of
         adding it.
 
-    .. method:: get_css(css, \*processors, \**kwargs)
+    .. method:: get_css(css, *processors, **kwargs)
 
         Similar to :meth:`ItemLoader.get_value` but receives a CSS selector
         instead of a value, which is used to extract a list of unicode strings
@@ -410,7 +410,7 @@ ItemLoader objects
             # HTML snippet: <p id="price">the price is $1200</p>
             loader.get_css('p#price', TakeFirst(), re='the price is (.*)')
 
-    .. method:: add_css(field_name, css, \*processors, \**kwargs)
+    .. method:: add_css(field_name, css, *processors, **kwargs)
 
         Similar to :meth:`ItemLoader.add_value` but receives a CSS selector
         instead of a value, which is used to extract a list of unicode strings
@@ -428,7 +428,7 @@ ItemLoader objects
             # HTML snippet: <p id="price">the price is $1200</p>
             loader.add_css('price', 'p#price', re='the price is (.*)')
 
-    .. method:: replace_css(field_name, css, \*processors, \**kwargs)
+    .. method:: replace_css(field_name, css, *processors, **kwargs)
 
         Similar to :meth:`add_css` but replaces collected data instead of
         adding it.
@@ -678,7 +678,7 @@ Here is a list of all built-in processors:
     >>> proc(['one', 'two', 'three'])
     'one<br>two<br>three'
 
-.. class:: Compose(\*functions, \**default_loader_context)
+.. class:: Compose(*functions, **default_loader_context)
 
     A processor which is constructed from the composition of the given
     functions. This means that each input value of this processor is passed to
@@ -706,7 +706,7 @@ Here is a list of all built-in processors:
     active Loader context accessible through the :meth:`ItemLoader.context`
     attribute.
 
-.. class:: MapCompose(\*functions, \**default_loader_context)
+.. class:: MapCompose(*functions, **default_loader_context)
 
     A processor which is constructed from the composition of the given
     functions, similar to the :class:`Compose` processor. The difference with
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 231db6cea..8ff5e7292 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -121,7 +121,7 @@ scrapy.Spider
       send log messages through it as described on
       :ref:`topics-logging-from-spiders`.
 
-   .. method:: from_crawler(crawler, \*args, \**kwargs)
+   .. method:: from_crawler(crawler, *args, **kwargs)
 
        This is the class method used by Scrapy to create your spiders.
 

From 898bdd3feb23de793ae0b4c725545c32050fc0ea Mon Sep 17 00:00:00 2001
From: Will Beaufoy <will@willbeaufoy.net>
Date: Sun, 10 May 2020 13:49:43 +0100
Subject: [PATCH 163/243] Update docs README with build instructions using tox

None of the existing commands built the docs properly for me (I had to
revert the changes in 901892d to docs/conf.py to get them to build
properly, and even then no tooltips displayed).

Building them with tox worked for me, but other developers say they
can still use the original method, so the docs now contain both.
---
 docs/README.rst | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/docs/README.rst b/docs/README.rst
index 0a343cd19..0b7afa548 100644
--- a/docs/README.rst
+++ b/docs/README.rst
@@ -57,3 +57,12 @@ There is a way to recreate the doc automatically when you make changes, you
 need to install watchdog (``pip install watchdog``) and then use::
 
     make watch
+
+Alternative method using tox
+----------------------------
+
+To compile the documentation to HTML run the following command::
+
+    tox -e docs
+
+Documentation will be generated (in HTML format) inside the ``.tox/docs/tmp/html`` dir.

From 492197e4773798fcb19ba934c03d5b48e0b7c1f2 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 27 May 2020 10:45:07 -0300
Subject: [PATCH 164/243] StopDownload: store response in the exception instead
 of the failure

---
 docs/topics/exceptions.rst                |  9 ++++++---
 scrapy/core/downloader/handlers/http11.py |  8 ++------
 scrapy/core/scraper.py                    | 10 +---------
 tests/test_crawl.py                       | 10 +++++++---
 4 files changed, 16 insertions(+), 21 deletions(-)

diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst
index 10b23edc4..583a50ab8 100644
--- a/docs/topics/exceptions.rst
+++ b/docs/topics/exceptions.rst
@@ -91,9 +91,12 @@ indicate that no further bytes should be downloaded for a response.
 The ``fail`` boolean parameter controls which method will handle the resulting
 response:
 
-* If ``fail=True``, the request errback is called. The response object is
-  available as the ``response`` attribute of the received
-  :class:`~twisted.python.failure.Failure` object. This is the default behaviour.
+* If ``fail=True`` (default), the request errback is called. The response object is
+  available as the ``response`` attribute of the ``StopDownload`` exception,
+  which is in turn stored as the ``value`` attribute of the received
+  :class:`~twisted.python.failure.Failure` object. This means that in an errback
+  defined as ``def errback(self, failure)``, the response can be accessed though
+  ``failure.value.response``.
 
 * If ``fail=False``, the request callback is called instead.
 
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index a710761c5..22c9ac520 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -442,12 +442,8 @@ class ScrapyAgent:
             ip_address=result["ip_address"],
         )
         if result.get("failure"):
-            # This failure is not the same object that will reach the errback,
-            # so we need to temporarily store the response in the exception.
-            # It will be moved to the failure in core/scraper.py
-            failure = result["failure"]
-            failure.value.response = response
-            return failure
+            result["failure"].value.response = response
+            return result["failure"]
         return response
 
 
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index c52ada83b..6785e103d 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -11,7 +11,7 @@ from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errba
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
 from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
-from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest, StopDownload
+from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
 from scrapy import signals
 from scrapy.http import Request, Response
 from scrapy.item import _BaseItem
@@ -147,14 +147,6 @@ class Scraper:
 
     def call_spider(self, result, request, spider):
         result.request = request
-        # StopDownload exceptions: make the partial response an attribute of the failure
-        if (
-            isinstance(result, Failure)
-            and isinstance(result.value, StopDownload)
-            and hasattr(result.value, "response")
-        ):
-            result.response = result.value.response
-            delattr(result.value, "response")
         dfd = defer_result(result)
         callback = request.callback or spider.parse
         warn_on_generator_with_return_value(spider, callback)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index e2426e0da..0115b8fb9 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -488,6 +488,10 @@ with multiples lines
         self.assertIsNone(crawler.spider.meta.get("response"))
         self.assertIsInstance(crawler.spider.meta["failure"], Failure)
         self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload)
-        self.assertIsInstance(crawler.spider.meta["failure"].response, Response)
-        self.assertEqual(crawler.spider.meta["failure"].response.body, crawler.spider.meta.get("bytes_received"))
-        self.assertLess(len(crawler.spider.meta["failure"].response.body), crawler.spider.full_response_length)
+        self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response)
+        self.assertEqual(
+            crawler.spider.meta["failure"].value.response.body,
+            crawler.spider.meta.get("bytes_received"))
+        self.assertLess(
+            len(crawler.spider.meta["failure"].value.response.body),
+            crawler.spider.full_response_length)

From 44d5801b05280ad633e63a6b25db264d6586d882 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 27 May 2020 11:13:21 -0300
Subject: [PATCH 165/243] Fix KeyError in LocalWeakReferencedCache

---
 scrapy/utils/datatypes.py     | 4 ++--
 tests/test_utils_datatypes.py | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index f59f4cc55..2a92d0588 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -105,8 +105,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
     def __getitem__(self, key):
         try:
             return super(LocalWeakReferencedCache, self).__getitem__(key)
-        except TypeError:
-            return None  # key is not weak-referenceable, it's not cached
+        except (TypeError, KeyError):
+            return None  # key is either not weak-referenceable or not cached
 
 
 class SequenceExclude:
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 0a4c6034a..aa18ef1f3 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -271,6 +271,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase):
         self.assertNotIn(r1, cache)
         self.assertIn(r2, cache)
         self.assertIn(r3, cache)
+        self.assertEqual(cache[r1], None)
         self.assertEqual(cache[r2], 2)
         self.assertEqual(cache[r3], 3)
         del r2

From 5cef927944152f11d296566767c1d794b34b38e4 Mon Sep 17 00:00:00 2001
From: Bulat Khabibullin <xb.bolo@gmail.com>
Date: Mon, 1 Jun 2020 07:57:23 +0300
Subject: [PATCH 166/243] Implement TextResponse.json() (#4574)

---
 docs/topics/request-response.rst |  5 +++++
 scrapy/http/response/text.py     | 12 ++++++++++++
 tests/test_http_response.py      | 21 +++++++++++++++++++++
 3 files changed, 38 insertions(+)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index ae25ff7e4..bbd715766 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -879,6 +879,11 @@ TextResponse objects
 
     .. automethod:: TextResponse.follow_all
 
+    .. automethod:: TextResponse.json()
+
+        Returns a Python object from deserialized JSON document.
+        The result is cached after the first call.
+
 
 HtmlResponse objects
 --------------------
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 0603b6653..40cf3f483 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -5,6 +5,7 @@ discovering (through HTTP headers) to base Response class.
 See documentation in docs/topics/request-response.rst
 """
 
+import json
 import warnings
 from contextlib import suppress
 from typing import Generator
@@ -21,10 +22,13 @@ from scrapy.http.response import Response
 from scrapy.utils.python import memoizemethod_noargs, to_unicode
 from scrapy.utils.response import get_base_url
 
+_NONE = object()
+
 
 class TextResponse(Response):
 
     _DEFAULT_ENCODING = 'ascii'
+    _cached_decoded_json = _NONE
 
     def __init__(self, *args, **kwargs):
         self._encoding = kwargs.pop('encoding', None)
@@ -68,6 +72,14 @@ class TextResponse(Response):
                       ScrapyDeprecationWarning, stacklevel=2)
         return self.text
 
+    def json(self):
+        """
+        Deserialize a JSON document to a Python object.
+        """
+        if self._cached_decoded_json is _NONE:
+            self._cached_decoded_json = json.loads(self.text)
+        return self._cached_decoded_json
+
     @property
     def text(self):
         """ Body as unicode """
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 039e863f4..e0ca3c0e6 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,4 +1,5 @@
 import unittest
+from unittest import mock
 from warnings import catch_warnings
 
 from w3lib.encoding import resolve_encoding
@@ -685,6 +686,26 @@ class TextResponseTest(BaseResponseTest):
             self.assertEqual(len(warnings), 1)
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
 
+    def test_json_response(self):
+        json_body = b"""{"ip": "109.187.217.200"}"""
+        json_response = self.response_class("http://www.example.com", body=json_body)
+        self.assertEqual(json_response.json(), {'ip': '109.187.217.200'})
+
+        text_body = b"""<html><body>text</body></html>"""
+        text_response = self.response_class("http://www.example.com", body=text_body)
+        with self.assertRaises(ValueError):
+            text_response.json()
+
+    def test_cache_json_response(self):
+        json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""]
+        for json_body in json_valid_bodies:
+            json_response = self.response_class("http://www.example.com", body=json_body)
+
+            with mock.patch('json.loads') as mock_json:
+                for _ in range(2):
+                    json_response.json()
+                mock_json.assert_called_once_with(json_body.decode())
+
 
 class HtmlResponseTest(TextResponseTest):
 

From 91e505edfe811c8faa6e457ecdae42c3644e6537 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Tue, 2 Jun 2020 05:32:14 -0300
Subject: [PATCH 167/243] Return single element from coroutine callback (#4609)

---
 scrapy/utils/spider.py |  8 ++++++--
 tests/spiders.py       | 11 +++++++++++
 tests/test_crawl.py    | 16 ++++++++++++++++
 3 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 1b8a82829..7e7a50c88 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -1,5 +1,5 @@
-import logging
 import inspect
+import logging
 
 from scrapy.spiders import Spider
 from scrapy.utils.defer import deferred_from_coro
@@ -18,7 +18,11 @@ def iterate_spider_output(result):
         d = deferred_from_coro(collect_asyncgen(result))
         d.addCallback(iterate_spider_output)
         return d
-    return arg_to_iter(deferred_from_coro(result))
+    elif inspect.iscoroutine(result):
+        d = deferred_from_coro(result)
+        d.addCallback(iterate_spider_output)
+        return d
+    return arg_to_iter(result)
 
 
 def iter_spider_classes(module):
diff --git a/tests/spiders.py b/tests/spiders.py
index 05078cc04..a360d8206 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -119,6 +119,17 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider):
         return [{'id': 1}, {'id': 2}]
 
 
+class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):
+
+    name = "asyncdef_asyncio_return_single_element"
+
+    async def parse(self, response):
+        await asyncio.sleep(0.1)
+        status = await get_from_asyncio_queue(response.status)
+        self.logger.info("Got response %d" % status)
+        return {"foo": 42}
+
+
 class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
 
     name = 'asyncdef_asyncio_reqs_return'
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 0115b8fb9..df920f2a2 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -21,6 +21,7 @@ from scrapy.utils.python import to_unicode
 from tests.mockserver import MockServer
 from tests.spiders import (
     AsyncDefAsyncioReqsReturnSpider,
+    AsyncDefAsyncioReturnSingleElementSpider,
     AsyncDefAsyncioReturnSpider,
     AsyncDefAsyncioSpider,
     AsyncDefSpider,
@@ -364,6 +365,21 @@ with multiples lines
         self.assertIn({'id': 1}, items)
         self.assertIn({'id': 2}, items)
 
+    @mark.only_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_asyncio_parse_items_single_element(self):
+        items = []
+
+        def _on_item_scraped(item):
+            items.append(item)
+
+        crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSingleElementSpider)
+        crawler.signals.connect(_on_item_scraped, signals.item_scraped)
+        with LogCapture() as log:
+            yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+        self.assertIn("Got response 200", str(log))
+        self.assertIn({"foo": 42}, items)
+
     @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
     @mark.only_asyncio()
     @defer.inlineCallbacks

From 0cabf406d5c50060ac2da4e5c76d704606bee990 Mon Sep 17 00:00:00 2001
From: Matthias Meschede <MMesch@users.noreply.github.com>
Date: Mon, 1 Jun 2020 17:41:52 +0200
Subject: [PATCH 168/243] set write permission to startproject folder

---
 scrapy/commands/startproject.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index ebe3a9c2c..cfaa25154 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -4,6 +4,7 @@ import string
 from importlib import import_module
 from os.path import join, exists, abspath
 from shutil import ignore_patterns, move, copy2, copystat
+import stat
 
 import scrapy
 from scrapy.commands import ScrapyCommand
@@ -79,6 +80,28 @@ class Command(ScrapyCommand):
                 copy2(srcname, dstname)
         copystat(src, dst)
 
+    def _set_rw_permissions(self, path):
+        """
+        Sets permissions of a directory tree to +rw and +rwx for folders.
+        This is necessary if the start template files come without write
+        permissions.
+        """
+        mode_rw = (stat.S_IRUSR
+                   | stat.S_IWUSR
+                   | stat.S_IRGRP
+                   | stat.S_IROTH)
+
+        mode_x = (stat.S_IXUSR
+                  | stat.S_IXGRP
+                  | stat.S_IXOTH)
+
+        os.chmod(path, mode_rw | mode_x)
+        for root, dirs, files in os.walk(path):
+            for dir in dirs:
+                os.chmod(join(root, dir), mode_rw | mode_x)
+            for file in files:
+                os.chmod(join(root, file), mode_rw)
+
     def run(self, args, opts):
         if len(args) not in (1, 2):
             raise UsageError()
@@ -99,6 +122,9 @@ class Command(ScrapyCommand):
             return
 
         self._copytree(self.templates_dir, abspath(project_dir))
+
+        self._set_rw_permissions(abspath(project_dir))
+
         move(join(project_dir, 'module'), join(project_dir, project_name))
         for paths in TEMPLATES_TO_RENDER:
             path = join(*paths)

From 2df3b54c7d2666a95793ea6788e88d1ae945e0f5 Mon Sep 17 00:00:00 2001
From: Matthias Meschede <MMesch@users.noreply.github.com>
Date: Fri, 5 Jun 2020 09:29:05 +0200
Subject: [PATCH 169/243] refactor

---
 scrapy/commands/startproject.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index cfaa25154..3d1f5914f 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -79,6 +79,7 @@ class Command(ScrapyCommand):
             else:
                 copy2(srcname, dstname)
         copystat(src, dst)
+        self._set_rw_permissions(dst)
 
     def _set_rw_permissions(self, path):
         """
@@ -123,8 +124,6 @@ class Command(ScrapyCommand):
 
         self._copytree(self.templates_dir, abspath(project_dir))
 
-        self._set_rw_permissions(abspath(project_dir))
-
         move(join(project_dir, 'module'), join(project_dir, project_name))
         for paths in TEMPLATES_TO_RENDER:
             path = join(*paths)

From 76a2cbf0ff7a060833812704a0416ba617ddc8b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 9 Jun 2020 21:30:19 +0200
Subject: [PATCH 170/243] Apply minor style changes

---
 scrapy/commands/startproject.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 3d1f5914f..852281959 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -1,10 +1,10 @@
 import re
 import os
+import stat
 import string
 from importlib import import_module
 from os.path import join, exists, abspath
 from shutil import ignore_patterns, move, copy2, copystat
-import stat
 
 import scrapy
 from scrapy.commands import ScrapyCommand
@@ -123,7 +123,6 @@ class Command(ScrapyCommand):
             return
 
         self._copytree(self.templates_dir, abspath(project_dir))
-
         move(join(project_dir, 'module'), join(project_dir, project_name))
         for paths in TEMPLATES_TO_RENDER:
             path = join(*paths)

From 8b549392f924ddad9536e55c6120638daf688dfd Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Thu, 11 Jun 2020 09:53:59 -0300
Subject: [PATCH 171/243] Bump minimum Python version to 3.5.2 (#4615)

---
 .travis.yml            | 6 ++----
 README.rst             | 2 +-
 docs/faq.rst           | 2 +-
 docs/intro/install.rst | 2 +-
 scrapy/__init__.py     | 4 ++--
 setup.py               | 2 +-
 6 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index d6ec88e06..e44f85237 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,11 +18,9 @@ matrix:
 
     - env: TOXENV=pypy3
     - env: TOXENV=pinned
-      python: 3.5.1
-      dist: trusty
+      python: 3.5.2
     - env: TOXENV=asyncio
-      python: 3.5.1  # We use additional code to support 3.5.3 and earlier
-      dist: trusty
+      python: 3.5.2  # We use additional code to support 3.5.3 and earlier
     - env: TOXENV=py
       python: 3.5
     - env: TOXENV=asyncio
diff --git a/README.rst b/README.rst
index fd84e127e..0e3939e9b 100644
--- a/README.rst
+++ b/README.rst
@@ -40,7 +40,7 @@ including a list of features.
 Requirements
 ============
 
-* Python 3.5.1+
+* Python 3.5.2+
 * Works on Linux, Windows, macOS, BSD
 
 Install
diff --git a/docs/faq.rst b/docs/faq.rst
index c06cb945b..9cdb7d09d 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -69,7 +69,7 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
 What Python versions does Scrapy support?
 -----------------------------------------
 
-Scrapy is supported under Python 3.5.1+
+Scrapy is supported under Python 3.5.2+
 under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
 Python 3 support was added in Scrapy 1.1.
 PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 4af80d801..fb64d443c 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -7,7 +7,7 @@ Installation guide
 Installing Scrapy
 =================
 
-Scrapy runs on Python 3.5.1 or above under CPython (default Python
+Scrapy runs on Python 3.5.2 or above under CPython (default Python
 implementation) and PyPy (starting with PyPy 5.9).
 
 If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index e791deaa6..f0259a9b7 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -28,8 +28,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
 
 
 # Check minimum required Python version
-if sys.version_info < (3, 5):
-    print("Scrapy %s requires Python 3.5" % __version__)
+if sys.version_info < (3, 5, 2):
+    print("Scrapy %s requires Python 3.5.2" % __version__)
     sys.exit(1)
 
 
diff --git a/setup.py b/setup.py
index 1b3c6771a..71dc3232d 100644
--- a/setup.py
+++ b/setup.py
@@ -66,7 +66,7 @@ setup(
         'Topic :: Software Development :: Libraries :: Application Frameworks',
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
-    python_requires='>=3.5',
+    python_requires='>=3.5.2',
     install_requires=[
         'Twisted>=17.9.0',
         'cryptography>=2.0',

From ec98dabfab60283303a9208ccd8177d9f995ba72 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Sun, 14 Jun 2020 06:45:27 -0300
Subject: [PATCH 172/243] Support for dataclass and attrs items (#3881)

---
 docs/conf.py                                  |   1 +
 docs/faq.rst                                  |   8 +-
 docs/topics/architecture.rst                  |   2 +-
 docs/topics/coroutines.rst                    |  13 +-
 docs/topics/exporters.rst                     |  34 ++-
 docs/topics/feed-exports.rst                  |   4 +-
 docs/topics/item-pipeline.rst                 |  54 ++--
 docs/topics/items.rst                         | 241 ++++++++++++++----
 docs/topics/leaks.rst                         |  15 +-
 docs/topics/loaders.rst                       |  53 ++--
 docs/topics/media-pipeline.rst                |  33 ++-
 docs/topics/settings.rst                      |   4 +-
 docs/topics/signals.rst                       |  10 +-
 docs/topics/spider-middleware.rst             |  13 +-
 docs/topics/spiders.rst                       |  18 +-
 scrapy/commands/parse.py                      |   6 +-
 scrapy/contracts/default.py                   |  28 +-
 scrapy/core/scraper.py                        |  27 +-
 scrapy/exporters.py                           |  38 +--
 scrapy/item.py                                |   2 +-
 scrapy/loader/__init__.py                     |  18 +-
 scrapy/pipelines/files.py                     |  23 +-
 scrapy/pipelines/images.py                    |  19 +-
 scrapy/shell.py                               |  11 +-
 scrapy/spiders/feed.py                        |   2 +-
 .../project/module/middlewares.py.tmpl        |   8 +-
 .../project/module/pipelines.py.tmpl          |   4 +
 scrapy/utils/serialize.py                     |   6 +-
 setup.py                                      |   1 +
 tests/requirements-py3.txt                    |   2 +
 tests/test_engine.py                          |  40 ++-
 tests/test_loader.py                          |  47 +++-
 tests/test_pipeline_files.py                  | 144 ++++++++---
 tests/test_pipeline_images.py                 | 129 +++++++---
 tests/test_utils_serialize.py                 |  43 +++-
 tox.ini                                       |   3 +-
 36 files changed, 753 insertions(+), 351 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 29b2fc406..86734fae7 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -281,6 +281,7 @@ coverage_ignore_pyobjects = [
 # -------------------------------------
 
 intersphinx_mapping = {
+    'attrs': ('https://www.attrs.org/en/stable/', None),
     'coverage': ('https://coverage.readthedocs.io/en/stable', None),
     'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
     'pytest': ('https://docs.pytest.org/en/latest', None),
diff --git a/docs/faq.rst b/docs/faq.rst
index 9cdb7d09d..d5ea3cb87 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -342,15 +342,15 @@ method for this purpose. For example::
 
     from copy import deepcopy
 
-    from scrapy.item import Item
-
+    from itemadapter import is_item, ItemAdapter
 
     class MultiplyItemsMiddleware:
 
         def process_spider_output(self, response, result, spider):
             for item in result:
-                if isinstance(item, (Item, dict)):
-                    for _ in range(item['multiply_by']):
+                if is_item(item):
+                    adapter = ItemAdapter(item)
+                    for _ in range(adapter['multiply_by']):
                         yield deepcopy(item)
 
 Does Scrapy support IPv6 addresses?
diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index ae25dfa2f..074c59241 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -104,7 +104,7 @@ Spiders
 -------
 
 Spiders are custom classes written by Scrapy users to parse responses and
-extract items (aka scraped items) from them or additional requests to
+extract :ref:`items <topics-items>` from them or additional requests to
 follow. For more information see :ref:`topics-spiders`.
 
 .. _component-pipelines:
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 7a9ecd4d5..a0952d323 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -53,21 +53,28 @@ There are several use cases for coroutines in Scrapy. Code that would
 return Deferreds when written for previous Scrapy versions, such as downloader
 middlewares and signal handlers, can be rewritten to be shorter and cleaner::
 
+    from itemadapter import ItemAdapter
+
     class DbPipeline:
         def _update_item(self, data, item):
-            item['field'] = data
+            adapter = ItemAdapter(item)
+            adapter['field'] = data
             return item
 
         def process_item(self, item, spider):
-            dfd = db.get_some_data(item['id'])
+            adapter = ItemAdapter(item)
+            dfd = db.get_some_data(adapter['id'])
             dfd.addCallback(self._update_item, item)
             return dfd
 
 becomes::
 
+    from itemadapter import ItemAdapter
+
     class DbPipeline:
         async def process_item(self, item, spider):
-            item['field'] = await db.get_some_data(item['id'])
+            adapter = ItemAdapter(item)
+            adapter['field'] = await db.get_some_data(adapter['id'])
             return item
 
 Coroutines may be used to call asynchronous code. This includes other
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 7daf25ab3..e5c99e5b1 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -40,6 +40,7 @@ Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses multiple
 Item Exporters to group scraped items to different files according to the
 value of one of their fields::
 
+    from itemadapter import ItemAdapter
     from scrapy.exporters import XmlItemExporter
 
     class PerYearXmlExportPipeline:
@@ -53,7 +54,8 @@ value of one of their fields::
                 exporter.finish_exporting()
 
         def _exporter_for_item(self, item):
-            year = item['year']
+            adapter = ItemAdapter(item)
+            year = adapter['year']
             if year not in self.year_to_exporter:
                 f = open('{}.xml'.format(year), 'wb')
                 exporter = XmlItemExporter(f)
@@ -167,9 +169,10 @@ BaseItemExporter
       value unchanged except for ``unicode`` values which are encoded to
       ``str`` using the encoding declared in the :attr:`encoding` attribute.
 
-      :param field: the field being serialized. If a raw dict is being
-          exported (not :class:`~.Item`) *field* value is an empty dict.
-      :type field: :class:`~scrapy.item.Field` object or an empty dict
+      :param field: the field being serialized. If the source :ref:`item object
+          <item-types>` does not define field metadata, *field* is an empty
+          :class:`dict`.
+      :type field: :class:`~scrapy.item.Field` object or a :class:`dict` instance
 
       :param name: the name of the field being serialized
       :type name: str
@@ -192,14 +195,17 @@ BaseItemExporter
 
    .. attribute:: fields_to_export
 
-      A list with the name of the fields that will be exported, or None if you
-      want to export all fields. Defaults to None.
+      A list with the name of the fields that will be exported, or ``None`` if
+      you want to export all fields. Defaults to ``None``.
 
       Some exporters (like :class:`CsvItemExporter`) respect the order of the
       fields defined in this attribute.
 
-      Some exporters may require fields_to_export list in order to export the
-      data properly when spiders return dicts (not :class:`~Item` instances).
+      When using :ref:`item objects <item-types>` that do not expose all their
+      possible fields, exporters that do not support exporting a different
+      subset of fields per item will only export the fields found in the first
+      item exported. Use ``fields_to_export`` to define all the fields to be
+      exported.
 
    .. attribute:: export_empty_fields
 
@@ -238,7 +244,7 @@ XmlItemExporter
 
 .. class:: XmlItemExporter(file, item_element='item', root_element='items', **kwargs)
 
-   Exports Items in XML format to the specified file object.
+   Exports items in XML format to the specified file object.
 
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@@ -292,7 +298,7 @@ CsvItemExporter
 
 .. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', **kwargs)
 
-   Exports Items in CSV format to the given file-like object. If the
+   Exports items in CSV format to the given file-like object. If the
    :attr:`fields_to_export` attribute is set, it will be used to define the
    CSV columns and their order. The :attr:`export_empty_fields` attribute has
    no effect on this exporter.
@@ -325,7 +331,7 @@ PickleItemExporter
 
 .. class:: PickleItemExporter(file, protocol=0, **kwargs)
 
-   Exports Items in pickle format to the given file-like object.
+   Exports items in pickle format to the given file-like object.
 
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@@ -345,7 +351,7 @@ PprintItemExporter
 
 .. class:: PprintItemExporter(file, **kwargs)
 
-   Exports Items in pretty print format to the specified file object.
+   Exports items in pretty print format to the specified file object.
 
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@@ -365,7 +371,7 @@ JsonItemExporter
 
 .. class:: JsonItemExporter(file, **kwargs)
 
-   Exports Items in JSON format to the specified file-like object, writing all
+   Exports items in JSON format to the specified file-like object, writing all
    objects as a list of objects. The additional ``__init__`` method arguments are
    passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover
    arguments to the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any
@@ -394,7 +400,7 @@ JsonLinesItemExporter
 
 .. class:: JsonLinesItemExporter(file, **kwargs)
 
-   Exports Items in JSON format to the specified file-like object, writing one
+   Exports items in JSON format to the specified file-like object, writing one
    JSON-encoded item per line. The additional ``__init__`` method arguments are passed
    to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to
    the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 9e5968a29..24d69040c 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -298,8 +298,8 @@ Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
 
 Use FEED_EXPORT_FIELDS option to define fields to export and their order.
 
-When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses fields
-defined in dicts or :class:`~.Item` subclasses a spider is yielding.
+When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses the fields
+defined in :ref:`item objects <topics-items>` yielded by your spider.
 
 If an exporter requires a fixed set of fields (this is the case for
 :ref:`CSV <topics-feed-format-csv>` export format) and FEED_EXPORT_FIELDS
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index c9194caa1..cd6a6d47e 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -27,15 +27,19 @@ Each item pipeline component is a Python class that must implement the following
 
 .. method:: process_item(self, item, spider)
 
-   This method is called for every item pipeline component. :meth:`process_item`
-   must either: return a dict with data, return an :class:`~scrapy.item.Item`
-   (or any descendant class) object, return a
-   :class:`~twisted.internet.defer.Deferred` or raise
-   :exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer
-   processed by further pipeline components.
+   This method is called for every item pipeline component.
 
-   :param item: the item scraped
-   :type item: :class:`~scrapy.item.Item` object or a dict
+   `item` is an :ref:`item object <item-types>`, see
+   :ref:`supporting-item-types`.
+
+   :meth:`process_item` must either: return an :ref:`item object <item-types>`,
+   return a :class:`~twisted.internet.defer.Deferred` or raise a
+   :exc:`~scrapy.exceptions.DropItem` exception.
+
+   Dropped items are no longer processed by further pipeline components.
+
+   :param item: the scraped item
+   :type item: :ref:`item object <item-types>`
 
    :param spider: the spider which scraped the item
    :type spider: :class:`~scrapy.spiders.Spider` object
@@ -79,16 +83,17 @@ Let's take a look at the following hypothetical pipeline that adjusts the
 (``price_excludes_vat`` attribute), and drops those items which don't
 contain a price::
 
+    from itemadapter import ItemAdapter
     from scrapy.exceptions import DropItem
-
     class PricePipeline:
 
         vat_factor = 1.15
 
         def process_item(self, item, spider):
-            if item.get('price'):
-                if item.get('price_excludes_vat'):
-                    item['price'] = item['price'] * self.vat_factor
+            adapter = ItemAdapter(item)
+            if adapter.get('price'):
+                if adapter.get('price_excludes_vat'):
+                    adapter['price'] = adapter['price'] * self.vat_factor
                 return item
             else:
                 raise DropItem("Missing price in %s" % item)
@@ -103,6 +108,8 @@ format::
 
    import json
 
+   from itemadapter import ItemAdapter
+
    class JsonWriterPipeline:
 
        def open_spider(self, spider):
@@ -112,7 +119,7 @@ format::
            self.file.close()
 
        def process_item(self, item, spider):
-           line = json.dumps(dict(item)) + "\n"
+           line = json.dumps(ItemAdapter(item).asdict()) + "\n"
            self.file.write(line)
            return item
 
@@ -131,6 +138,7 @@ The main point of this example is to show how to use :meth:`from_crawler`
 method and how to clean up the resources properly.::
 
     import pymongo
+    from itemadapter import ItemAdapter
 
     class MongoPipeline:
 
@@ -155,7 +163,7 @@ method and how to clean up the resources properly.::
             self.client.close()
 
         def process_item(self, item, spider):
-            self.db[self.collection_name].insert_one(dict(item))
+            self.db[self.collection_name].insert_one(ItemAdapter(item).asdict())
             return item
 
 .. _MongoDB: https://www.mongodb.com/
@@ -177,10 +185,11 @@ item.
 
 ::
 
-    import scrapy
     import hashlib
     from urllib.parse import quote
 
+    import scrapy
+    from itemadapter import ItemAdapter
 
     class ScreenshotPipeline:
         """Pipeline that uses Splash to render screenshot of
@@ -189,7 +198,8 @@ item.
         SPLASH_URL = "http://localhost:8050/render.png?url={}"
 
         async def process_item(self, item, spider):
-            encoded_item_url = quote(item["url"])
+            adapter = ItemAdapter(item)
+            encoded_item_url = quote(adapter["url"])
             screenshot_url = self.SPLASH_URL.format(encoded_item_url)
             request = scrapy.Request(screenshot_url)
             response = await spider.crawler.engine.download(request, spider)
@@ -199,14 +209,14 @@ item.
                 return item
 
             # Save screenshot to file, filename will be hash of url.
-            url = item["url"]
+            url = adapter["url"]
             url_hash = hashlib.md5(url.encode("utf8")).hexdigest()
             filename = "{}.png".format(url_hash)
             with open(filename, "wb") as f:
                 f.write(response.body)
 
             # Store filename in item.
-            item["screenshot_filename"] = filename
+            adapter["screenshot_filename"] = filename
             return item
 
 .. _Splash: https://splash.readthedocs.io/en/stable/
@@ -219,6 +229,7 @@ already processed. Let's say that our items have a unique id, but our spider
 returns multiples items with the same id::
 
 
+    from itemadapter import ItemAdapter
     from scrapy.exceptions import DropItem
 
     class DuplicatesPipeline:
@@ -227,10 +238,11 @@ returns multiples items with the same id::
             self.ids_seen = set()
 
         def process_item(self, item, spider):
-            if item['id'] in self.ids_seen:
-                raise DropItem("Duplicate item found: %s" % item)
+            adapter = ItemAdapter(item)
+            if adapter['id'] in self.ids_seen:
+                raise DropItem("Duplicate item found: %r" % item)
             else:
-                self.ids_seen.add(item['id'])
+                self.ids_seen.add(adapter['id'])
                 return item
 
 
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 0941a8a1b..65bf156ac 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -8,29 +8,155 @@ Items
    :synopsis: Item and Field classes
 
 The main goal in scraping is to extract structured data from unstructured
-sources, typically, web pages. Scrapy spiders can return the extracted data
-as Python dicts. While convenient and familiar, Python dicts lack structure:
-it is easy to make a typo in a field name or return inconsistent data,
-especially in a larger project with many spiders.
+sources, typically, web pages. :ref:`Spiders <topics-spiders>` may return the
+extracted data as `items`, Python objects that define key-value pairs.
 
-To define common output data format Scrapy provides the :class:`Item` class.
-:class:`Item` objects are simple containers used to collect the scraped data.
-They provide an API similar to :class:`dict` API with a convenient syntax
-for declaring their available fields.
+Scrapy supports :ref:`multiple types of items <item-types>`. When you create an
+item, you may use whichever type of item you want. When you write code that
+receives an item, your code should :ref:`work for any item type
+<supporting-item-types>`.
 
-Various Scrapy components use extra information provided by Items:
-exporters look at declared fields to figure out columns to export,
-serialization can be customized using Item fields metadata, :mod:`trackref`
-tracks Item instances to help find memory leaks
-(see :ref:`topics-leaks-trackrefs`), etc.
+.. _item-types:
+
+Item Types
+==========
+
+Scrapy supports the following types of items, via the `itemadapter`_ library:
+:ref:`dictionaries <dict-items>`, :ref:`Item objects <item-objects>`,
+:ref:`dataclass objects <dataclass-items>`, and :ref:`attrs objects <attrs-items>`.
+
+.. _itemadapter: https://github.com/scrapy/itemadapter
+
+.. _dict-items:
+
+Dictionaries
+------------
+
+As an item type, :class:`dict` is convenient and familiar.
+
+.. _item-objects:
+
+Item objects
+------------
+
+:class:`Item` provides a :class:`dict`-like API plus additional features that
+make it the most feature-complete item type:
+
+.. class:: Item([arg])
+
+    :class:`Item` objects replicate the standard :class:`dict` API, including
+    its ``__init__`` method.
+
+    :class:`Item` allows defining field names, so that:
+
+    -   :class:`KeyError` is raised when using undefined field names (i.e.
+        prevents typos going unnoticed)
+
+    -   :ref:`Item exporters <topics-exporters>` can export all fields by
+        default even if the first scraped object does not have values for all
+        of them
+
+    :class:`Item` also allows defining field metadata, which can be used to
+    :ref:`customize serialization <topics-exporters-field-serialization>`.
+
+    :mod:`trackref` tracks :class:`Item` objects to help find memory leaks
+    (see :ref:`topics-leaks-trackrefs`).
+
+    :class:`Item` objects also provide the following additional API members:
+
+    .. automethod:: copy
+
+    .. automethod:: deepcopy
+
+    .. attribute:: fields
+
+        A dictionary containing *all declared fields* for this Item, not only
+        those populated. The keys are the field names and the values are the
+        :class:`Field` objects used in the :ref:`Item declaration
+        <topics-items-declaring>`.
+
+Example::
+
+    from scrapy.item import Item, Field
+
+    class CustomItem(Item):
+        one_field = Field()
+        another_field = Field()
+
+.. _dataclass-items:
+
+Dataclass objects
+-----------------
+
+.. versionadded:: 2.2
+
+:func:`~dataclasses.dataclass` allows defining item classes with field names,
+so that :ref:`item exporters <topics-exporters>` can export all fields by
+default even if the first scraped object does not have values for all of them.
+
+Additionally, ``dataclass`` items also allow to:
+
+* define the type and default value of each defined field.
+
+* define custom field metadata through :func:`dataclasses.field`, which can be used to
+  :ref:`customize serialization <topics-exporters-field-serialization>`.
+
+They work natively in Python 3.7 or later, or using the `dataclasses
+backport`_ in Python 3.6.
+
+.. _dataclasses backport: https://pypi.org/project/dataclasses/
+
+Example::
+
+    from dataclasses import dataclass
+
+    @dataclass
+    class CustomItem:
+        one_field: str
+        another_field: int
+
+.. note:: Field types are not enforced at run time.
+
+.. _attrs-items:
+
+attr.s objects
+--------------
+
+.. versionadded:: 2.2
+
+:func:`attr.s` allows defining item classes with field names,
+so that :ref:`item exporters <topics-exporters>` can export all fields by
+default even if the first scraped object does not have values for all of them.
+
+Additionally, ``attr.s`` items also allow to:
+
+* define the type and default value of each defined field.
+
+* define custom field :ref:`metadata <attrs:metadata>`, which can be used to
+  :ref:`customize serialization <topics-exporters-field-serialization>`.
+
+In order to use this type, the :doc:`attrs package <attrs:index>` needs to be installed.
+
+Example::
+
+    import attr
+
+    @attr.s
+    class CustomItem:
+        one_field = attr.ib()
+        another_field = attr.ib()
+
+
+Working with Item objects
+=========================
 
 .. _topics-items-declaring:
 
-Declaring Items
-===============
+Declaring Item subclasses
+-------------------------
 
-Items are declared using a simple class definition syntax and :class:`Field`
-objects. Here is an example::
+Item subclasses are declared using a simple class definition syntax and
+:class:`Field` objects. Here is an example::
 
     import scrapy
 
@@ -48,10 +174,11 @@ objects. Here is an example::
 .. _Django: https://www.djangoproject.com/
 .. _Django Models: https://docs.djangoproject.com/en/dev/topics/db/models/
 
+
 .. _topics-items-fields:
 
-Item Fields
-===========
+Declaring fields
+----------------
 
 :class:`Field` objects are used to specify metadata for each field. For
 example, the serializer function for the ``last_updated`` field illustrated in
@@ -72,15 +199,31 @@ It's important to note that the :class:`Field` objects used to declare the item
 do not stay assigned as class attributes. Instead, they can be accessed through
 the :attr:`Item.fields` attribute.
 
-Working with Items
-==================
+.. class:: Field([arg])
+
+    The :class:`Field` class is just an alias to the built-in :class:`dict` class and
+    doesn't provide any extra functionality or attributes. In other words,
+    :class:`Field` objects are plain-old Python dicts. A separate class is used
+    to support the :ref:`item declaration syntax <topics-items-declaring>`
+    based on class attributes.
+
+.. note:: Field metadata can also be declared for ``dataclass`` and ``attrs``
+    items. Please refer to the documentation for `dataclasses.field`_ and
+    `attr.ib`_ for additional information.
+
+    .. _dataclasses.field: https://docs.python.org/3/library/dataclasses.html#dataclasses.field
+    .. _attr.ib: https://www.attrs.org/en/stable/api.html#attr.ib
+
+
+Working with Item objects
+-------------------------
 
 Here are some examples of common tasks performed with items, using the
 ``Product`` item :ref:`declared above  <topics-items-declaring>`. You will
 notice the API is very similar to the :class:`dict` API.
 
 Creating items
---------------
+''''''''''''''
 
 >>> product = Product(name='Desktop PC', price=1000)
 >>> print(product)
@@ -88,7 +231,7 @@ Product(name='Desktop PC', price=1000)
 
 
 Getting field values
---------------------
+''''''''''''''''''''
 
 >>> product['name']
 Desktop PC
@@ -128,7 +271,7 @@ False
 
 
 Setting field values
---------------------
+''''''''''''''''''''
 
 >>> product['last_updated'] = 'today'
 >>> product['last_updated']
@@ -141,7 +284,7 @@ KeyError: 'Product does not support field: lala'
 
 
 Accessing all populated values
-------------------------------
+''''''''''''''''''''''''''''''
 
 To access all populated values, just use the typical :class:`dict` API:
 
@@ -155,7 +298,7 @@ To access all populated values, just use the typical :class:`dict` API:
 .. _copying-items:
 
 Copying items
--------------
+'''''''''''''
 
 To copy an item, you must first decide whether you want a shallow copy or a
 deep copy.
@@ -183,7 +326,7 @@ To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead
 
 
 Other common tasks
-------------------
+''''''''''''''''''
 
 Creating dicts from items:
 
@@ -201,8 +344,8 @@ Traceback (most recent call last):
 KeyError: 'Product does not support field: lala'
 
 
-Extending Items
-===============
+Extending Item subclasses
+-------------------------
 
 You can extend Items (to add more fields or to change some metadata for some
 fields) by declaring a subclass of your original Item.
@@ -222,39 +365,25 @@ appending more values, or changing existing values, like this::
 That adds (or replaces) the ``serializer`` metadata key for the ``name`` field,
 keeping all the previously existing metadata values.
 
-Item objects
-============
 
-.. class:: Item([arg])
+.. _supporting-item-types:
 
-    Return a new Item optionally initialized from the given argument.
+Supporting All Item Types
+=========================
 
-    Items replicate the standard :class:`dict` API, including its ``__init__``
-    method, and also provide the following additional API members:
+In code that receives an item, such as methods of :ref:`item pipelines
+<topics-item-pipeline>` or :ref:`spider middlewares
+<topics-spider-middleware>`, it is a good practice to use the
+:class:`~itemadapter.ItemAdapter` class and the
+:func:`~itemadapter.is_item` function to write code that works for
+any :ref:`supported item type <item-types>`:
 
-    .. automethod:: copy
+.. autoclass:: itemadapter.ItemAdapter
 
-    .. automethod:: deepcopy
+.. autofunction:: itemadapter.is_item
 
-    .. attribute:: fields
 
-        A dictionary containing *all declared fields* for this Item, not only
-        those populated. The keys are the field names and the values are the
-        :class:`Field` objects used in the :ref:`Item declaration
-        <topics-items-declaring>`.
-
-Field objects
-=============
-
-.. class:: Field([arg])
-
-    The :class:`Field` class is just an alias to the built-in :class:`dict` class and
-    doesn't provide any extra functionality or attributes. In other words,
-    :class:`Field` objects are plain-old Python dicts. A separate class is used
-    to support the :ref:`item declaration syntax <topics-items-declaring>`
-    based on class attributes.
-
-Other classes related to Item
-=============================
+Other classes related to items
+==============================
 
 .. autoclass:: ItemMeta
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index ceb708c7e..3224241fc 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -4,7 +4,7 @@
 Debugging memory leaks
 ======================
 
-In Scrapy, objects such as Requests, Responses and Items have a finite
+In Scrapy, objects such as requests, responses and items have a finite
 lifetime: they are created, used for a while, and finally destroyed.
 
 From all those objects, the Request is probably the one with the longest
@@ -61,8 +61,8 @@ Debugging memory leaks with ``trackref``
 ========================================
 
 :mod:`trackref` is a module provided by Scrapy to debug the most common cases of
-memory leaks. It basically tracks the references to all live Requests,
-Responses, Item and Selector objects.
+memory leaks. It basically tracks the references to all live Request,
+Response, Item, Spider and Selector objects.
 
 You can enter the telnet console and inspect how many objects (of the classes
 mentioned above) are currently alive using the ``prefs()`` function which is an
@@ -200,11 +200,10 @@ Debugging memory leaks with muppy
 
 ``trackref`` provides a very convenient mechanism for tracking down memory
 leaks, but it only keeps track of the objects that are more likely to cause
-memory leaks (Requests, Responses, Items, and Selectors). However, there are
-other cases where the memory leaks could come from other (more or less obscure)
-objects. If this is your case, and you can't find your leaks using ``trackref``,
-you still have another resource: the muppy library.
-
+memory leaks. However, there are other cases where the memory leaks could come
+from other (more or less obscure) objects. If this is your case, and you can't
+find your leaks using ``trackref``, you still have another resource: the muppy
+library.
 
 You can use muppy from `Pympler`_.
 
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index eb804f1db..6645bf123 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -7,13 +7,12 @@ Item Loaders
 .. module:: scrapy.loader
    :synopsis: Item Loader class
 
-Item Loaders provide a convenient mechanism for populating scraped :ref:`Items
-<topics-items>`. Even though Items can be populated using their own
-dictionary-like API, Item Loaders provide a much more convenient API for
-populating them from a scraping process, by automating some common tasks like
-parsing the raw extracted data before assigning it.
+Item Loaders provide a convenient mechanism for populating scraped :ref:`items
+<topics-items>`. Even though items can be populated directly, Item Loaders provide a
+much more convenient API for populating them from a scraping process, by automating
+some common tasks like parsing the raw extracted data before assigning it.
 
-In other words, :ref:`Items <topics-items>` provide the *container* of
+In other words, :ref:`items <topics-items>` provide the *container* of
 scraped data, while Item Loaders provide the mechanism for *populating* that
 container.
 
@@ -25,10 +24,10 @@ Using Item Loaders to populate items
 ====================================
 
 To use an Item Loader, you must first instantiate it. You can either
-instantiate it with a dict-like object (e.g. Item or dict) or without one, in
-which case an Item is automatically instantiated in the Item Loader ``__init__`` method
-using the Item class specified in the :attr:`ItemLoader.default_item_class`
-attribute.
+instantiate it with an :ref:`item object <topics-items>` or without one, in which
+case an instance of :class:`~scrapy.item.Item` is automatically created in the
+Item Loader ``__init__`` method using the :class:`~scrapy.item.Item` subclass
+specified in the :attr:`ItemLoader.default_item_class` attribute.
 
 Then, you start collecting values into the Item Loader, typically using
 :ref:`Selectors <topics-selectors>`. You can add more than one value to
@@ -88,7 +87,7 @@ received (through the :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`
 :meth:`~ItemLoader.add_value` methods) and the result of the input processor is
 collected and kept inside the ItemLoader. After collecting all data, the
 :meth:`ItemLoader.load_item` method is called to populate and get the populated
-:class:`~scrapy.item.Item` object.  That's when the output processor is
+:ref:`item object <topics-items>`.  That's when the output processor is
 called with the data previously collected (and processed using the input
 processor). The result of the output processor is the final value that gets
 assigned to the item.
@@ -153,12 +152,10 @@ Last, but not least, Scrapy comes with some :ref:`commonly used processors
 <topics-loaders-available-processors>` built-in for convenience.
 
 
-
 Declaring Item Loaders
 ======================
 
-Item Loaders are declared like Items, by using a class definition syntax. Here
-is an example::
+Item Loaders are declared using a class definition syntax. Here is an example::
 
     from scrapy.loader import ItemLoader
     from scrapy.loader.processors import TakeFirst, MapCompose, Join
@@ -275,9 +272,9 @@ ItemLoader objects
 
 .. class:: ItemLoader([item, selector, response], **kwargs)
 
-    Return a new Item Loader for populating the given Item. If no item is
-    given, one is instantiated automatically using the class in
-    :attr:`default_item_class`.
+    Return a new Item Loader for populating the given :ref:`item object
+    <topics-items>`. If no item object is given, one is instantiated
+    automatically using the class in :attr:`default_item_class`.
 
     When instantiated with a ``selector`` or a ``response`` parameters
     the :class:`ItemLoader` class provides convenient mechanisms for extracting
@@ -286,7 +283,7 @@ ItemLoader objects
     :param item: The item instance to populate using subsequent calls to
         :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
         or :meth:`~ItemLoader.add_value`.
-    :type item: :class:`~scrapy.item.Item` object
+    :type item: :ref:`item object <topics-items>`
 
     :param selector: The selector to extract data from, when using the
         :meth:`add_xpath` (resp. :meth:`add_css`) or :meth:`replace_xpath`
@@ -444,17 +441,19 @@ ItemLoader objects
 
         Create a nested loader with an xpath selector.
         The supplied selector is applied relative to selector associated
-        with this :class:`ItemLoader`. The nested loader shares the :class:`Item`
-        with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`,
-        :meth:`add_value`, :meth:`replace_value`, etc. will behave as expected.
+        with this :class:`ItemLoader`. The nested loader shares the :ref:`item
+        object <topics-items>` with the parent :class:`ItemLoader` so calls to
+        :meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
+        behave as expected.
 
     .. method:: nested_css(css)
 
         Create a nested loader with a css selector.
         The supplied selector is applied relative to selector associated
-        with this :class:`ItemLoader`. The nested loader shares the :class:`Item`
-        with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`,
-        :meth:`add_value`, :meth:`replace_value`, etc. will behave as expected.
+        with this :class:`ItemLoader`. The nested loader shares the :ref:`item
+        object <topics-items>` with the parent :class:`ItemLoader` so calls to
+        :meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
+        behave as expected.
 
     .. method:: get_collected_values(field_name)
 
@@ -477,7 +476,7 @@ ItemLoader objects
 
     .. attribute:: item
 
-        The :class:`~scrapy.item.Item` object being parsed by this Item Loader.
+        The :ref:`item object <topics-items>` being parsed by this Item Loader.
         This is mostly used as a property so when attempting to override this
         value, you may want to check out :attr:`default_item_class` first.
 
@@ -488,8 +487,8 @@ ItemLoader objects
 
     .. attribute:: default_item_class
 
-        An Item class (or factory), used to instantiate items when not given in
-        the ``__init__`` method.
+        An :ref:`item object <topics-items>` class or factory, used to
+        instantiate items when not given in the ``__init__`` method.
 
     .. attribute:: default_input_processor
 
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 86550d7a4..01de3dedb 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -156,7 +156,7 @@ following forms::
 
     ftp://username:password@address:port/path
     ftp://address:port/path
-    
+
 If ``username`` and ``password`` are not provided, they are taken from the :setting:`FTP_USER` and
 :setting:`FTP_PASSWORD` settings respectively.
 
@@ -243,20 +243,22 @@ Usage example
 .. setting:: IMAGES_URLS_FIELD
 .. setting:: IMAGES_RESULT_FIELD
 
-In order to use a media pipeline first, :ref:`enable it
+In order to use a media pipeline, first :ref:`enable it
 <topics-media-pipeline-enabling>`.
 
-Then, if a spider returns a dict with the URLs key (``file_urls`` or
-``image_urls``, for the Files or Images Pipeline respectively), the pipeline will
-put the results under respective key (``files`` or ``images``).
+Then, if a spider returns an :ref:`item object <topics-items>` with the URLs
+field (``file_urls`` or ``image_urls``, for the Files or Images Pipeline
+respectively), the pipeline will put the results under the respective field
+(``files`` or ``images``).
 
-If you prefer to use :class:`~.Item`, then define a custom item with the
-necessary fields, like in this example for Images Pipeline::
+When using :ref:`item types <item-types>` for which fields are defined beforehand,
+you must define both the URLs field and the results field. For example, when
+using the images pipeline, items must define both the ``image_urls`` and the
+``images`` field. For instance, using the :class:`~scrapy.item.Item` class::
 
     import scrapy
 
     class MyItem(scrapy.Item):
-
         # ... other item fields ...
         image_urls = scrapy.Field()
         images = scrapy.Field()
@@ -445,8 +447,11 @@ See here the methods that you can override in your custom Files Pipeline:
       :meth:`~get_media_requests` method and return a Request for each
       file URL::
 
+         from itemadapter import ItemAdapter
+
          def get_media_requests(self, item, info):
-             for file_url in item['file_urls']:
+             adapter = ItemAdapter(item)
+             for file_url in adapter['file_urls']:
                  yield scrapy.Request(file_url)
 
       Those requests will be processed by the pipeline and, when they have finished
@@ -509,13 +514,15 @@ See here the methods that you can override in your custom Files Pipeline:
       store the downloaded file paths (passed in results) in the ``file_paths``
       item field, and we drop the item if it doesn't contain any files::
 
+          from itemadapter import ItemAdapter
           from scrapy.exceptions import DropItem
 
           def item_completed(self, results, item, info):
               file_paths = [x['path'] for ok, x in results if ok]
               if not file_paths:
                   raise DropItem("Item contains no files")
-              item['file_paths'] = file_paths
+              adapter = ItemAdapter(item)
+              adapter['file_paths'] = file_paths
               return item
 
       By default, the :meth:`item_completed` method returns the item.
@@ -589,8 +596,9 @@ Here is a full example of the Images Pipeline whose methods are exemplified
 above::
 
     import scrapy
-    from scrapy.pipelines.images import ImagesPipeline
+    from itemadapter import ItemAdapter
     from scrapy.exceptions import DropItem
+    from scrapy.pipelines.images import ImagesPipeline
 
     class MyImagesPipeline(ImagesPipeline):
 
@@ -602,7 +610,8 @@ above::
             image_paths = [x['path'] for ok, x in results if ok]
             if not image_paths:
                 raise DropItem("Item contains no images")
-            item['image_paths'] = image_paths
+            adapter = ItemAdapter(item)
+            adapter['image_paths'] = image_paths
             return item
 
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index f06d9db3c..5178f272f 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -236,8 +236,8 @@ CONCURRENT_ITEMS
 
 Default: ``100``
 
-Maximum number of concurrent items (per response) to process in parallel in the
-Item Processor (also known as the :ref:`Item Pipeline <topics-item-pipeline>`).
+Maximum number of concurrent items (per response) to process in parallel in
+:ref:`item pipelines <topics-item-pipeline>`.
 
 .. setting:: CONCURRENT_REQUESTS
 
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index fe4fb0834..255ba9d3f 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -151,8 +151,8 @@ item_scraped
 
     This signal supports returning deferreds from its handlers.
 
-    :param item: the item scraped
-    :type item: dict or :class:`~scrapy.item.Item` object
+    :param item: the scraped item
+    :type item: :ref:`item object <item-types>`
 
     :param spider: the spider which scraped the item
     :type spider: :class:`~scrapy.spiders.Spider` object
@@ -172,7 +172,7 @@ item_dropped
     This signal supports returning deferreds from its handlers.
 
     :param item: the item dropped from the :ref:`topics-item-pipeline`
-    :type item: dict or :class:`~scrapy.item.Item` object
+    :type item: :ref:`item object <item-types>`
 
     :param spider: the spider which scraped the item
     :type spider: :class:`~scrapy.spiders.Spider` object
@@ -196,8 +196,8 @@ item_error
 
     This signal supports returning deferreds from its handlers.
 
-    :param item: the item dropped from the :ref:`topics-item-pipeline`
-    :type item: dict or :class:`~scrapy.item.Item` object
+    :param item: the item that caused the error in the :ref:`topics-item-pipeline`
+    :type item: :ref:`item object <item-types>`
 
     :param response: the response being processed when the exception was raised
     :type response: :class:`~scrapy.http.Response` object
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index d49a2209d..c6cbdba76 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -102,29 +102,28 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         it has processed the response.
 
         :meth:`process_spider_output` must return an iterable of
-        :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item`
-        objects.
+        :class:`~scrapy.http.Request` objects and :ref:`item object
+        <topics-items>`.
 
         :param response: the response which generated this output from the
           spider
         :type response: :class:`~scrapy.http.Response` object
 
         :param result: the result returned by the spider
-        :type result: an iterable of :class:`~scrapy.http.Request`, dict
-          or :class:`~scrapy.item.Item` objects
+        :type result: an iterable of :class:`~scrapy.http.Request` objects and
+          :ref:`item object <topics-items>`
 
         :param spider: the spider whose result is being processed
         :type spider: :class:`~scrapy.spiders.Spider` object
 
-
     .. method:: process_spider_exception(response, exception, spider)
 
         This method is called when a spider or :meth:`process_spider_output`
         method (from a previous spider middleware) raises an exception.
 
         :meth:`process_spider_exception` should return either ``None`` or an
-        iterable of :class:`~scrapy.http.Request`, dict or
-        :class:`~scrapy.item.Item` objects.
+        iterable of :class:`~scrapy.http.Request` objects and :ref:`item object
+        <topics-items>`.
 
         If it returns ``None``, Scrapy will continue processing this exception,
         executing any other :meth:`process_spider_exception` in the following
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 8ff5e7292..d4d6e2ea0 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -23,8 +23,8 @@ For spiders, the scraping cycle goes through something like this:
    :attr:`~scrapy.spiders.Spider.parse` method as callback function for the
    Requests.
 
-2. In the callback function, you parse the response (web page) and return either
-   dicts with extracted data, :class:`~scrapy.item.Item` objects,
+2. In the callback function, you parse the response (web page) and return
+   :ref:`item objects <topics-items>`,
    :class:`~scrapy.http.Request` objects, or an iterable of these objects.
    Those Requests will also contain a callback (maybe
    the same) and will then be downloaded by Scrapy and then their
@@ -179,8 +179,8 @@ scrapy.Spider
        the same requirements as the :class:`Spider` class.
 
        This method, as well as any other Request callback, must return an
-       iterable of :class:`~scrapy.http.Request` and/or
-       dicts or :class:`~scrapy.item.Item` objects.
+       iterable of :class:`~scrapy.http.Request` and/or :ref:`item objects
+       <topics-items>`.
 
        :param response: the response to parse
        :type response: :class:`~scrapy.http.Response`
@@ -234,7 +234,7 @@ Return multiple Requests and items from a single callback::
                 yield scrapy.Request(response.urljoin(href), self.parse)
 
 Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly;
-to give data more structure you can use :ref:`topics-items`::
+to give data more structure you can use :class:`~scrapy.item.Item` objects::
 
     import scrapy
     from myproject.items import MyItem
@@ -364,7 +364,7 @@ CrawlSpider
 
       This method is called for the start_urls responses. It allows to parse
       the initial responses and must return either an
-      :class:`~scrapy.item.Item` object, a :class:`~scrapy.http.Request`
+      :ref:`item object <topics-items>`, a :class:`~scrapy.http.Request`
       object, or an iterable containing any of them.
 
 Crawling rules
@@ -383,7 +383,7 @@ Crawling rules
    object with that name will be used) to be called for each link extracted with
    the specified link extractor. This callback receives a :class:`~scrapy.http.Response`
    as its first argument and must return either a single instance or an iterable of
-   :class:`~scrapy.item.Item`, ``dict`` and/or :class:`~scrapy.http.Request` objects
+   :ref:`item objects <topics-items>` and/or :class:`~scrapy.http.Request` objects
    (or any subclass of them). As mentioned above, the received :class:`~scrapy.http.Response`
    object will contain the text of the link that produced the :class:`~scrapy.http.Request`
    in its ``meta`` dictionary (under the ``link_text`` key)
@@ -531,7 +531,7 @@ XMLFeedSpider
         (``itertag``).  Receives the response and an
         :class:`~scrapy.selector.Selector` for each node.  Overriding this
         method is mandatory. Otherwise, you spider won't work.  This method
-        must return either a :class:`~scrapy.item.Item` object, a
+        must return an :ref:`item object <topics-items>`, a
         :class:`~scrapy.http.Request` object, or an iterable containing any of
         them.
 
@@ -541,7 +541,7 @@ XMLFeedSpider
         spider, and it's intended to perform any last time processing required
         before returning the results to the framework core, for example setting the
         item IDs. It receives a list of results and the response which originated
-        those results. It must return a list of results (Items or Requests).
+        those results. It must return a list of results (items or requests).
 
 
 XMLFeedSpider example
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 580fd2828..8b7fa8b58 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -1,11 +1,11 @@
 import json
 import logging
 
+from itemadapter import is_item, ItemAdapter
 from w3lib.url import is_url
 
 from scrapy.commands import ScrapyCommand
 from scrapy.http import Request
-from scrapy.item import _BaseItem
 from scrapy.utils import display
 from scrapy.utils.conf import arglist_to_dict
 from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
@@ -81,7 +81,7 @@ class Command(ScrapyCommand):
             items = self.items.get(lvl, [])
 
         print("# Scraped Items ", "-" * 60)
-        display.pprint([dict(x) for x in items], colorize=colour)
+        display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour)
 
     def print_requests(self, lvl=None, colour=True):
         if lvl is None:
@@ -117,7 +117,7 @@ class Command(ScrapyCommand):
         items, requests = [], []
 
         for x in iterate_spider_output(callback(response, **cb_kwargs)):
-            if isinstance(x, (_BaseItem, dict)):
+            if is_item(x):
                 items.append(x)
             elif isinstance(x, Request):
                 requests.append(x)
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index cdc2bac15..34f0d36d4 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -1,10 +1,10 @@
 import json
 
-from scrapy.item import _BaseItem
-from scrapy.http import Request
-from scrapy.exceptions import ContractFail
+from itemadapter import is_item, ItemAdapter
 
 from scrapy.contracts import Contract
+from scrapy.exceptions import ContractFail
+from scrapy.http import Request
 
 
 # contracts
@@ -48,11 +48,11 @@ class ReturnsContract(Contract):
     """
 
     name = 'returns'
-    objects = {
-        'request': Request,
-        'requests': Request,
-        'item': (_BaseItem, dict),
-        'items': (_BaseItem, dict),
+    object_type_verifiers = {
+        'request': lambda x: isinstance(x, Request),
+        'requests': lambda x: isinstance(x, Request),
+        'item': is_item,
+        'items': is_item,
     }
 
     def __init__(self, *args, **kwargs):
@@ -64,7 +64,7 @@ class ReturnsContract(Contract):
                 % len(self.args)
             )
         self.obj_name = self.args[0] or None
-        self.obj_type = self.objects[self.obj_name]
+        self.obj_type_verifier = self.object_type_verifiers[self.obj_name]
 
         try:
             self.min_bound = int(self.args[1])
@@ -79,7 +79,7 @@ class ReturnsContract(Contract):
     def post_process(self, output):
         occurrences = 0
         for x in output:
-            if isinstance(x, self.obj_type):
+            if self.obj_type_verifier(x):
                 occurrences += 1
 
         assertion = (self.min_bound <= occurrences <= self.max_bound)
@@ -103,8 +103,8 @@ class ScrapesContract(Contract):
 
     def post_process(self, output):
         for x in output:
-            if isinstance(x, (_BaseItem, dict)):
-                missing = [arg for arg in self.args if arg not in x]
+            if is_item(x):
+                missing = [arg for arg in self.args if arg not in ItemAdapter(x)]
                 if missing:
-                    raise ContractFail(
-                        "Missing fields: %s" % ", ".join(missing))
+                    missing_str = ", ".join(missing)
+                    raise ContractFail("Missing fields: %s" % missing_str)
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 6785e103d..d07c7aa62 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -4,18 +4,18 @@ extracts information from them"""
 import logging
 from collections import deque
 
-from twisted.python.failure import Failure
+from itemadapter import is_item
 from twisted.internet import defer
+from twisted.python.failure import Failure
 
-from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errback
-from scrapy.utils.spider import iterate_spider_output
-from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
-from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
-from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
 from scrapy import signals
-from scrapy.http import Request, Response
-from scrapy.item import _BaseItem
 from scrapy.core.spidermw import SpiderMiddlewareManager
+from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
+from scrapy.http import Request, Response
+from scrapy.utils.defer import defer_result, defer_succeed, iter_errback, parallel
+from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
+from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
+from scrapy.utils.spider import iterate_spider_output
 
 
 logger = logging.getLogger(__name__)
@@ -191,7 +191,7 @@ class Scraper:
         """
         if isinstance(output, Request):
             self.crawler.engine.crawl(request=output, spider=spider)
-        elif isinstance(output, (_BaseItem, dict)):
+        elif is_item(output):
             self.slot.itemproc_size += 1
             dfd = self.itemproc.process_item(output, spider)
             dfd.addBoth(self._itemproc_finished, output, response, spider)
@@ -200,10 +200,11 @@ class Scraper:
             pass
         else:
             typename = type(output).__name__
-            logger.error('Spider must return Request, BaseItem, dict or None, '
-                         'got %(typename)r in %(request)s',
-                         {'request': request, 'typename': typename},
-                         extra={'spider': spider})
+            logger.error(
+                'Spider must return request, item, or None, got %(typename)r in %(request)s',
+                {'request': request, 'typename': typename},
+                extra={'spider': spider},
+            )
 
     def _log_download_errors(self, spider_failure, download_failure, request, spider):
         """Log and silence errors that come from the engine (typically download
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index de009082a..712572673 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -4,16 +4,18 @@ Item Exporters are used to export/serialize items into different formats.
 
 import csv
 import io
-import pprint
 import marshal
-import warnings
 import pickle
+import pprint
+import warnings
 from xml.sax.saxutils import XMLGenerator
 
-from scrapy.utils.serialize import ScrapyJSONEncoder
-from scrapy.utils.python import to_bytes, to_unicode, is_listlike
-from scrapy.item import _BaseItem
+from itemadapter import is_item, ItemAdapter
+
 from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.item import _BaseItem
+from scrapy.utils.python import is_listlike, to_bytes, to_unicode
+from scrapy.utils.serialize import ScrapyJSONEncoder
 
 
 __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
@@ -56,11 +58,14 @@ class BaseItemExporter:
         """Return the fields to export as an iterable of tuples
         (name, serialized_value)
         """
+        item = ItemAdapter(item)
+
         if include_empty is None:
             include_empty = self.export_empty_fields
+
         if self.fields_to_export is None:
-            if include_empty and not isinstance(item, dict):
-                field_iter = item.fields.keys()
+            if include_empty:
+                field_iter = item.field_names()
             else:
                 field_iter = item.keys()
         else:
@@ -71,8 +76,8 @@ class BaseItemExporter:
 
         for field_name in field_iter:
             if field_name in item:
-                field = {} if isinstance(item, dict) else item.fields[field_name]
-                value = self.serialize_field(field, field_name, item[field_name])
+                field_meta = item.get_field_meta(field_name)
+                value = self.serialize_field(field_meta, field_name, item[field_name])
             else:
                 value = default_value
 
@@ -297,6 +302,7 @@ class PythonItemExporter(BaseItemExporter):
 
     .. _msgpack: https://pypi.org/project/msgpack/
     """
+
     def _configure(self, options, dont_fail=False):
         self.binary = options.pop('binary', True)
         super(PythonItemExporter, self)._configure(options, dont_fail)
@@ -314,22 +320,22 @@ class PythonItemExporter(BaseItemExporter):
     def _serialize_value(self, value):
         if isinstance(value, _BaseItem):
             return self.export_item(value)
-        if isinstance(value, dict):
-            return dict(self._serialize_dict(value))
-        if is_listlike(value):
+        elif is_item(value):
+            return dict(self._serialize_item(value))
+        elif is_listlike(value):
             return [self._serialize_value(v) for v in value]
         encode_func = to_bytes if self.binary else to_unicode
         if isinstance(value, (str, bytes)):
             return encode_func(value, encoding=self.encoding)
         return value
 
-    def _serialize_dict(self, value):
-        for key, val in value.items():
+    def _serialize_item(self, item):
+        for key, value in ItemAdapter(item).items():
             key = to_bytes(key) if self.binary else key
-            yield key, self._serialize_value(val)
+            yield key, self._serialize_value(value)
 
     def export_item(self, item):
         result = dict(self._get_serialized_fields(item))
         if self.binary:
-            result = dict(self._serialize_dict(result))
+            result = dict(self._serialize_item(result))
         return result
diff --git a/scrapy/item.py b/scrapy/item.py
index 97dfed976..4ab83d1a0 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -36,7 +36,7 @@ class BaseItem(_BaseItem, metaclass=_BaseItemMeta):
     """
 
     def __new__(cls, *args, **kwargs):
-        if issubclass(cls, BaseItem) and not (issubclass(cls, Item) or issubclass(cls, DictItem)):
+        if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)):
             warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
                  ScrapyDeprecationWarning, stacklevel=2)
         return super(BaseItem, cls).__new__(cls, *args, **kwargs)
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 21c4fb376..18f57945f 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -6,6 +6,8 @@ See documentation in docs/topics/loaders.rst
 from collections import defaultdict
 from contextlib import suppress
 
+from itemadapter import ItemAdapter
+
 from scrapy.item import Item
 from scrapy.loader.common import wrap_loader_context
 from scrapy.loader.processors import Identity
@@ -44,7 +46,7 @@ class ItemLoader:
         self._local_item = context['item'] = item
         self._local_values = defaultdict(list)
         # values from initial item
-        for field_name, value in item.items():
+        for field_name, value in ItemAdapter(item).items():
             self._values[field_name] += arg_to_iter(value)
 
     @property
@@ -127,13 +129,12 @@ class ItemLoader:
         return value
 
     def load_item(self):
-        item = self.item
+        adapter = ItemAdapter(self.item)
         for field_name in tuple(self._values):
             value = self.get_output_value(field_name)
             if value is not None:
-                item[field_name] = value
-
-        return item
+                adapter[field_name] = value
+        return adapter.item
 
     def get_output_value(self, field_name):
         proc = self.get_output_processor(field_name)
@@ -174,11 +175,8 @@ class ItemLoader:
                                     value, type(e).__name__, str(e)))
 
     def _get_item_field_attr(self, field_name, key, default=None):
-        if isinstance(self.item, Item):
-            value = self.item.fields[field_name].get(key, default)
-        else:
-            value = default
-        return value
+        field_meta = ItemAdapter(self.item).get_field_meta(field_name)
+        return field_meta.get(key, default)
 
     def _check_selector_method(self):
         if self.selector is None:
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 7d86d0d56..487382a38 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -10,24 +10,26 @@ import mimetypes
 import os
 import time
 from collections import defaultdict
-from email.utils import parsedate_tz, mktime_tz
+from contextlib import suppress
+from email.utils import mktime_tz, parsedate_tz
 from ftplib import FTP
 from io import BytesIO
 from urllib.parse import urlparse
 
+from itemadapter import ItemAdapter
 from twisted.internet import defer, threads
 
+from scrapy.exceptions import IgnoreRequest, NotConfigured
+from scrapy.http import Request
 from scrapy.pipelines.media import MediaPipeline
 from scrapy.settings import Settings
-from scrapy.exceptions import NotConfigured, IgnoreRequest
-from scrapy.http import Request
-from scrapy.utils.misc import md5sum
-from scrapy.utils.log import failure_to_exc_info
-from scrapy.utils.python import to_bytes
-from scrapy.utils.request import referer_str
 from scrapy.utils.boto import is_botocore
 from scrapy.utils.datatypes import CaselessDict
 from scrapy.utils.ftp import ftp_store_file
+from scrapy.utils.log import failure_to_exc_info
+from scrapy.utils.misc import md5sum
+from scrapy.utils.python import to_bytes
+from scrapy.utils.request import referer_str
 
 
 logger = logging.getLogger(__name__)
@@ -517,7 +519,8 @@ class FilesPipeline(MediaPipeline):
 
     # Overridable Interface
     def get_media_requests(self, item, info):
-        return [Request(x) for x in item.get(self.files_urls_field, [])]
+        urls = ItemAdapter(item).get(self.files_urls_field, [])
+        return [Request(u) for u in urls]
 
     def file_downloaded(self, response, request, info):
         path = self.file_path(request, response=response, info=info)
@@ -528,8 +531,8 @@ class FilesPipeline(MediaPipeline):
         return checksum
 
     def item_completed(self, results, item, info):
-        if isinstance(item, dict) or self.files_result_field in item.fields:
-            item[self.files_result_field] = [x for ok, x in results if ok]
+        with suppress(KeyError):
+            ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok]
         return item
 
     def file_path(self, request, response=None, info=None):
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index aeb520442..46f2bfb58 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -5,17 +5,19 @@ See documentation in topics/media-pipeline.rst
 """
 import functools
 import hashlib
+from contextlib import suppress
 from io import BytesIO
 
+from itemadapter import ItemAdapter
 from PIL import Image
 
+from scrapy.exceptions import DropItem
+from scrapy.http import Request
+from scrapy.pipelines.files import FileException, FilesPipeline
+# TODO: from scrapy.pipelines.media import MediaPipeline
+from scrapy.settings import Settings
 from scrapy.utils.misc import md5sum
 from scrapy.utils.python import to_bytes
-from scrapy.http import Request
-from scrapy.settings import Settings
-from scrapy.exceptions import DropItem
-# TODO: from scrapy.pipelines.media import MediaPipeline
-from scrapy.pipelines.files import FileException, FilesPipeline
 
 
 class NoimagesDrop(DropItem):
@@ -157,11 +159,12 @@ class ImagesPipeline(FilesPipeline):
         return image, buf
 
     def get_media_requests(self, item, info):
-        return [Request(x) for x in item.get(self.images_urls_field, [])]
+        urls = ItemAdapter(item).get(self.images_urls_field, [])
+        return [Request(u) for u in urls]
 
     def item_completed(self, results, item, info):
-        if isinstance(item, dict) or self.images_result_field in item.fields:
-            item[self.images_result_field] = [x for ok, x in results if ok]
+        with suppress(KeyError):
+            ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok]
         return item
 
     def file_path(self, request, response=None, info=None):
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 3ff5a8ad8..10de119ce 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -6,6 +6,7 @@ See documentation in docs/topics/shell.rst
 import os
 import signal
 
+from itemadapter import is_item
 from twisted.internet import threads, defer
 from twisted.python import threadable
 from w3lib.url import any_to_uri
@@ -13,20 +14,18 @@ from w3lib.url import any_to_uri
 from scrapy.crawler import Crawler
 from scrapy.exceptions import IgnoreRequest
 from scrapy.http import Request, Response
-from scrapy.item import _BaseItem
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
-from scrapy.utils.console import start_python_console
+from scrapy.utils.conf import get_config
+from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.misc import load_object
 from scrapy.utils.response import open_in_browser
-from scrapy.utils.conf import get_config
-from scrapy.utils.console import DEFAULT_PYTHON_SHELLS
 
 
 class Shell:
 
-    relevant_classes = (Crawler, Spider, Request, Response, _BaseItem, Settings)
+    relevant_classes = (Crawler, Spider, Request, Response, Settings)
 
     def __init__(self, crawler, update_vars=None, code=None):
         self.crawler = crawler
@@ -154,7 +153,7 @@ class Shell:
         return "\n".join("[s] %s" % line for line in b)
 
     def _is_relevant(self, value):
-        return isinstance(value, self.relevant_classes)
+        return isinstance(value, self.relevant_classes) or is_item(value)
 
 
 def inspect_response(response, spider):
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index a4ff8010d..5aad7398a 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -31,7 +31,7 @@ class XMLFeedSpider(Spider):
         processing required before returning the results to the framework core,
         for example setting the item GUIDs. It receives a list of results and
         the response which originated that results. It must return a list of
-        results (Items or Requests).
+        results (items or requests).
         """
         return results
 
diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl
index 6490f52a7..bd09890fe 100644
--- a/scrapy/templates/project/module/middlewares.py.tmpl
+++ b/scrapy/templates/project/module/middlewares.py.tmpl
@@ -5,6 +5,9 @@
 
 from scrapy import signals
 
+# useful for handling different item types with a single interface
+from itemadapter import is_item, ItemAdapter
+
 
 class ${ProjectName}SpiderMiddleware:
     # Not all methods need to be defined. If a method is not defined,
@@ -29,7 +32,7 @@ class ${ProjectName}SpiderMiddleware:
         # Called with the results returned from the Spider, after
         # it has processed the response.
 
-        # Must return an iterable of Request, dict or Item objects.
+        # Must return an iterable of Request, or item objects.
         for i in result:
             yield i
 
@@ -37,8 +40,7 @@ class ${ProjectName}SpiderMiddleware:
         # Called when a spider or process_spider_input() method
         # (from other spider middleware) raises an exception.
 
-        # Should return either None or an iterable of Request, dict
-        # or Item objects.
+        # Should return either None or an iterable of Request or item objects.
         pass
 
     def process_start_requests(self, start_requests, spider):
diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl
index ce0edd335..e845f43e9 100644
--- a/scrapy/templates/project/module/pipelines.py.tmpl
+++ b/scrapy/templates/project/module/pipelines.py.tmpl
@@ -4,6 +4,10 @@
 # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
 
 
+# useful for handling different item types with a single interface
+from itemadapter import ItemAdapter
+
+
 class ${ProjectName}Pipeline:
     def process_item(self, item, spider):
         return item
diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py
index bf73dfa18..dc9604578 100644
--- a/scrapy/utils/serialize.py
+++ b/scrapy/utils/serialize.py
@@ -2,10 +2,10 @@ import json
 import datetime
 import decimal
 
+from itemadapter import is_item, ItemAdapter
 from twisted.internet import defer
 
 from scrapy.http import Request, Response
-from scrapy.item import _BaseItem
 
 
 class ScrapyJSONEncoder(json.JSONEncoder):
@@ -26,8 +26,8 @@ class ScrapyJSONEncoder(json.JSONEncoder):
             return str(o)
         elif isinstance(o, defer.Deferred):
             return str(o)
-        elif isinstance(o, _BaseItem):
-            return dict(o)
+        elif is_item(o):
+            return ItemAdapter(o).asdict()
         elif isinstance(o, Request):
             return "<%s %s %s>" % (type(o).__name__, o.method, o.url)
         elif isinstance(o, Response):
diff --git a/setup.py b/setup.py
index 71dc3232d..5a99fd1bf 100644
--- a/setup.py
+++ b/setup.py
@@ -80,6 +80,7 @@ setup(
         'w3lib>=1.17.0',
         'zope.interface>=4.1.3',
         'protego>=0.1.15',
+        'itemadapter>=0.1.0',
     ],
     extras_require=extras_require,
 )
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 91fa1c5b5..dacb86e56 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,4 +1,6 @@
 # Tests requirements
+attrs
+dataclasses; python_version == '3.6'
 jmespath
 mitmproxy; python_version >= '3.6'
 mitmproxy<4.0.0; python_version < '3.6'
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 6696ee52e..1b848ac72 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -16,9 +16,11 @@ import sys
 from collections import defaultdict
 from urllib.parse import urlparse
 
+import attr
+from itemadapter import ItemAdapter
 from pydispatch import dispatcher
 from testfixtures import LogCapture
-from twisted.internet import reactor, defer
+from twisted.internet import defer, reactor
 from twisted.trial import unittest
 from twisted.web import server, static, util
 
@@ -32,7 +34,7 @@ from scrapy.spiders import Spider
 from scrapy.utils.signal import disconnect_all
 from scrapy.utils.test import get_crawler
 
-from tests import tests_datadir, get_testdata
+from tests import get_testdata, tests_datadir
 
 
 class TestItem(Item):
@@ -41,6 +43,13 @@ class TestItem(Item):
     price = Field()
 
 
+@attr.s
+class AttrsItem:
+    name = attr.ib(default="")
+    url = attr.ib(default="")
+    price = attr.ib(default=0)
+
+
 class TestSpider(Spider):
     name = "scrapytest.org"
     allowed_domains = ["scrapytest.org", "localhost"]
@@ -79,6 +88,27 @@ class DictItemsSpider(TestSpider):
     item_cls = dict
 
 
+class AttrsItemsSpider(TestSpider):
+    item_class = AttrsItem
+
+
+try:
+    from dataclasses import make_dataclass
+except ImportError:
+    DataClassItemsSpider = None
+else:
+    TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)])
+
+    class DataClassItemsSpider(DictItemsSpider):
+        def parse_item(self, response):
+            item = super().parse_item(response)
+            return TestDataClass(
+                name=item.get('name'),
+                url=item.get('url'),
+                price=item.get('price'),
+            )
+
+
 class ItemZeroDivisionErrorSpider(TestSpider):
     custom_settings = {
         "ITEM_PIPELINES": {
@@ -204,7 +234,10 @@ class EngineTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_crawler(self):
-        for spider in TestSpider, DictItemsSpider:
+
+        for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
+            if spider is None:
+                continue
             self.run = CrawlerRun(spider)
             yield self.run.run()
             self._assert_visited_urls()
@@ -281,6 +314,7 @@ class EngineTest(unittest.TestCase):
     def _assert_scraped_items(self):
         self.assertEqual(2, len(self.run.itemresp))
         for item, response in self.run.itemresp:
+            item = ItemAdapter(item)
             self.assertEqual(item['url'], response.url)
             if 'item1.html' in item['url']:
                 self.assertEqual('Item 1 name', item['name'])
diff --git a/tests/test_loader.py b/tests/test_loader.py
index f14714c75..8a9c6fca9 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -1,6 +1,9 @@
 from functools import partial
 import unittest
 
+import attr
+from itemadapter import ItemAdapter
+
 from scrapy.http import HtmlResponse
 from scrapy.item import Item, Field
 from scrapy.loader import ItemLoader
@@ -9,6 +12,13 @@ from scrapy.loader.processors import (Compose, Identity, Join,
 from scrapy.selector import Selector
 
 
+try:
+    from dataclasses import make_dataclass, field as dataclass_field
+except ImportError:
+    make_dataclass = None
+    dataclass_field = None
+
+
 # test items
 class NameItem(Item):
     name = Field()
@@ -28,6 +38,11 @@ class TestNestedItem(Item):
     image = Field()
 
 
+@attr.s
+class AttrsNameItem:
+    name = attr.ib(default="")
+
+
 # test item loaders
 class NameItemLoader(ItemLoader):
     default_item_class = TestItem
@@ -466,7 +481,7 @@ class InitializationTestMixin:
         il = ItemLoader(item=input_item)
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(dict(loaded_item), {'name': ['foo']})
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo']})
 
     def test_keep_list(self):
         """Loaded item should contain values from the initial item"""
@@ -474,7 +489,7 @@ class InitializationTestMixin:
         il = ItemLoader(item=input_item)
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']})
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar']})
 
     def test_add_value_singlevalue_singlevalue(self):
         """Values added after initialization should be appended"""
@@ -483,7 +498,7 @@ class InitializationTestMixin:
         il.add_value('name', 'bar')
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']})
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar']})
 
     def test_add_value_singlevalue_list(self):
         """Values added after initialization should be appended"""
@@ -492,7 +507,7 @@ class InitializationTestMixin:
         il.add_value('name', ['item', 'loader'])
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(dict(loaded_item), {'name': ['foo', 'item', 'loader']})
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'item', 'loader']})
 
     def test_add_value_list_singlevalue(self):
         """Values added after initialization should be appended"""
@@ -501,7 +516,7 @@ class InitializationTestMixin:
         il.add_value('name', 'qwerty')
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'qwerty']})
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar', 'qwerty']})
 
     def test_add_value_list_list(self):
         """Values added after initialization should be appended"""
@@ -510,7 +525,7 @@ class InitializationTestMixin:
         il.add_value('name', ['item', 'loader'])
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'item', 'loader']})
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar', 'item', 'loader']})
 
     def test_get_output_value_singlevalue(self):
         """Getting output value must not remove value from item"""
@@ -519,7 +534,7 @@ class InitializationTestMixin:
         self.assertEqual(il.get_output_value('name'), ['foo'])
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(loaded_item, dict({'name': ['foo']}))
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({'name': ['foo']}))
 
     def test_get_output_value_list(self):
         """Getting output value must not remove value from item"""
@@ -528,7 +543,7 @@ class InitializationTestMixin:
         self.assertEqual(il.get_output_value('name'), ['foo', 'bar'])
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(loaded_item, dict({'name': ['foo', 'bar']}))
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({'name': ['foo', 'bar']}))
 
     def test_values_single(self):
         """Values from initial item must be added to loader._values"""
@@ -551,6 +566,22 @@ class InitializationFromItemTest(InitializationTestMixin, unittest.TestCase):
     item_class = NameItem
 
 
+class InitializationFromAttrsItemTest(InitializationTestMixin, unittest.TestCase):
+    item_class = AttrsNameItem
+
+
+@unittest.skipIf(not make_dataclass, "dataclasses module is not available")
+class InitializationFromDataClassTest(InitializationTestMixin, unittest.TestCase):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if make_dataclass:
+            self.item_class = make_dataclass(
+                "TestDataClass",
+                [("name", list, dataclass_field(default_factory=list))],
+            )
+
+
 class BaseNoInputReprocessingLoader(ItemLoader):
     title_in = MapCompose(str.upper)
     title_out = TakeFirst()
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 6bbcbc2e9..a023dfcc8 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -2,22 +2,41 @@ import os
 import random
 import time
 from io import BytesIO
-from tempfile import mkdtemp
 from shutil import rmtree
-from unittest import mock
+from tempfile import mkdtemp
+from unittest import mock, skipIf
 from urllib.parse import urlparse
 
-from twisted.trial import unittest
+import attr
+from itemadapter import ItemAdapter
 from twisted.internet import defer
+from twisted.trial import unittest
 
-from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GCSFilesStore, FTPFilesStore
-from scrapy.item import Item, Field
 from scrapy.http import Request, Response
+from scrapy.item import Field, Item
+from scrapy.pipelines.files import (
+    FilesPipeline,
+    FSFilesStore,
+    FTPFilesStore,
+    GCSFilesStore,
+    S3FilesStore,
+)
 from scrapy.settings import Settings
-from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
-from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete
-from scrapy.utils.test import get_ftp_content_and_delete
 from scrapy.utils.boto import is_botocore
+from scrapy.utils.test import (
+    assert_aws_environ,
+    assert_gcs_environ,
+    get_ftp_content_and_delete,
+    get_gcs_content_and_delete,
+    get_s3_content_and_delete,
+)
+
+
+try:
+    from dataclasses import make_dataclass, field as dataclass_field
+except ImportError:
+    make_dataclass = None
+    dataclass_field = None
 
 
 def _mocked_download_func(request, info):
@@ -143,43 +162,88 @@ class FilesPipelineTestCase(unittest.TestCase):
             p.stop()
 
 
-class FilesPipelineTestCaseFields(unittest.TestCase):
+class FilesPipelineTestCaseFieldsMixin:
 
     def test_item_fields_default(self):
-        class TestItem(Item):
-            name = Field()
-            file_urls = Field()
-            files = Field()
-
-        for cls in TestItem, dict:
-            url = 'http://www.example.com/files/1.txt'
-            item = cls({'name': 'item1', 'file_urls': [url]})
-            pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': 's3://example/files/'}))
-            requests = list(pipeline.get_media_requests(item, None))
-            self.assertEqual(requests[0].url, url)
-            results = [(True, {'url': url})]
-            pipeline.item_completed(results, item, None)
-            self.assertEqual(item['files'], [results[0][1]])
+        url = 'http://www.example.com/files/1.txt'
+        item = self.item_class(name='item1', file_urls=[url])
+        pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': 's3://example/files/'}))
+        requests = list(pipeline.get_media_requests(item, None))
+        self.assertEqual(requests[0].url, url)
+        results = [(True, {'url': url})]
+        item = pipeline.item_completed(results, item, None)
+        files = ItemAdapter(item).get("files")
+        self.assertEqual(files, [results[0][1]])
+        self.assertIsInstance(item, self.item_class)
 
     def test_item_fields_override_settings(self):
-        class TestItem(Item):
-            name = Field()
-            files = Field()
-            stored_file = Field()
+        url = 'http://www.example.com/files/1.txt'
+        item = self.item_class(name='item1', custom_file_urls=[url])
+        pipeline = FilesPipeline.from_settings(Settings({
+            'FILES_STORE': 's3://example/files/',
+            'FILES_URLS_FIELD': 'custom_file_urls',
+            'FILES_RESULT_FIELD': 'custom_files'
+        }))
+        requests = list(pipeline.get_media_requests(item, None))
+        self.assertEqual(requests[0].url, url)
+        results = [(True, {'url': url})]
+        item = pipeline.item_completed(results, item, None)
+        custom_files = ItemAdapter(item).get("custom_files")
+        self.assertEqual(custom_files, [results[0][1]])
+        self.assertIsInstance(item, self.item_class)
 
-        for cls in TestItem, dict:
-            url = 'http://www.example.com/files/1.txt'
-            item = cls({'name': 'item1', 'files': [url]})
-            pipeline = FilesPipeline.from_settings(Settings({
-                'FILES_STORE': 's3://example/files/',
-                'FILES_URLS_FIELD': 'files',
-                'FILES_RESULT_FIELD': 'stored_file'
-            }))
-            requests = list(pipeline.get_media_requests(item, None))
-            self.assertEqual(requests[0].url, url)
-            results = [(True, {'url': url})]
-            pipeline.item_completed(results, item, None)
-            self.assertEqual(item['stored_file'], [results[0][1]])
+
+class FilesPipelineTestCaseFieldsDict(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
+    item_class = dict
+
+
+class FilesPipelineTestItem(Item):
+    name = Field()
+    # default fields
+    file_urls = Field()
+    files = Field()
+    # overridden fields
+    custom_file_urls = Field()
+    custom_files = Field()
+
+
+class FilesPipelineTestCaseFieldsItem(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
+    item_class = FilesPipelineTestItem
+
+
+@skipIf(not make_dataclass, "dataclasses module is not available")
+class FilesPipelineTestCaseFieldsDataClass(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if make_dataclass:
+            self.item_class = make_dataclass(
+                "FilesPipelineTestDataClass",
+                [
+                    ("name", str),
+                    # default fields
+                    ("file_urls", list, dataclass_field(default_factory=list)),
+                    ("files", list, dataclass_field(default_factory=list)),
+                    # overridden fields
+                    ("custom_file_urls", list, dataclass_field(default_factory=list)),
+                    ("custom_files", list, dataclass_field(default_factory=list)),
+                ],
+            )
+
+
+@attr.s
+class FilesPipelineTestAttrsItem:
+    name = attr.ib(default="")
+    # default fields
+    file_urls = attr.ib(default=lambda: [])
+    files = attr.ib(default=lambda: [])
+    # overridden fields
+    custom_file_urls = attr.ib(default=lambda: [])
+    custom_files = attr.ib(default=lambda: [])
+
+
+class FilesPipelineTestCaseFieldsAttrsItem(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
+    item_class = FilesPipelineTestAttrsItem
 
 
 class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 8ef27fce7..082e9ee21 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -1,17 +1,28 @@
-import io
 import hashlib
+import io
 import random
-from tempfile import mkdtemp
 from shutil import rmtree
+from tempfile import mkdtemp
+from unittest import skipIf
 
+import attr
+from itemadapter import ItemAdapter
 from twisted.trial import unittest
 
-from scrapy.item import Item, Field
 from scrapy.http import Request, Response
-from scrapy.settings import Settings
+from scrapy.item import Field, Item
 from scrapy.pipelines.images import ImagesPipeline
+from scrapy.settings import Settings
 from scrapy.utils.python import to_bytes
 
+
+try:
+    from dataclasses import make_dataclass, field as dataclass_field
+except ImportError:
+    make_dataclass = None
+    dataclass_field = None
+
+
 skip = False
 try:
     from PIL import Image
@@ -124,43 +135,89 @@ class DeprecatedImagesPipeline(ImagesPipeline):
         return 'thumbsup/%s/%s.jpg' % (thumb_id, thumb_guid)
 
 
-class ImagesPipelineTestCaseFields(unittest.TestCase):
+class ImagesPipelineTestCaseFieldsMixin:
 
     def test_item_fields_default(self):
-        class TestItem(Item):
-            name = Field()
-            image_urls = Field()
-            images = Field()
-
-        for cls in TestItem, dict:
-            url = 'http://www.example.com/images/1.jpg'
-            item = cls({'name': 'item1', 'image_urls': [url]})
-            pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': 's3://example/images/'}))
-            requests = list(pipeline.get_media_requests(item, None))
-            self.assertEqual(requests[0].url, url)
-            results = [(True, {'url': url})]
-            pipeline.item_completed(results, item, None)
-            self.assertEqual(item['images'], [results[0][1]])
+        url = 'http://www.example.com/images/1.jpg'
+        item = self.item_class(name='item1', image_urls=[url])
+        pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': 's3://example/images/'}))
+        requests = list(pipeline.get_media_requests(item, None))
+        self.assertEqual(requests[0].url, url)
+        results = [(True, {'url': url})]
+        item = pipeline.item_completed(results, item, None)
+        images = ItemAdapter(item).get("images")
+        self.assertEqual(images, [results[0][1]])
+        self.assertIsInstance(item, self.item_class)
 
     def test_item_fields_override_settings(self):
-        class TestItem(Item):
-            name = Field()
-            image = Field()
-            stored_image = Field()
+        url = 'http://www.example.com/images/1.jpg'
+        item = self.item_class(name='item1', custom_image_urls=[url])
+        pipeline = ImagesPipeline.from_settings(Settings({
+            'IMAGES_STORE': 's3://example/images/',
+            'IMAGES_URLS_FIELD': 'custom_image_urls',
+            'IMAGES_RESULT_FIELD': 'custom_images'
+        }))
+        requests = list(pipeline.get_media_requests(item, None))
+        self.assertEqual(requests[0].url, url)
+        results = [(True, {'url': url})]
+        item = pipeline.item_completed(results, item, None)
+        custom_images = ItemAdapter(item).get("custom_images")
+        self.assertEqual(custom_images, [results[0][1]])
+        self.assertIsInstance(item, self.item_class)
 
-        for cls in TestItem, dict:
-            url = 'http://www.example.com/images/1.jpg'
-            item = cls({'name': 'item1', 'image': [url]})
-            pipeline = ImagesPipeline.from_settings(Settings({
-                'IMAGES_STORE': 's3://example/images/',
-                'IMAGES_URLS_FIELD': 'image',
-                'IMAGES_RESULT_FIELD': 'stored_image'
-            }))
-            requests = list(pipeline.get_media_requests(item, None))
-            self.assertEqual(requests[0].url, url)
-            results = [(True, {'url': url})]
-            pipeline.item_completed(results, item, None)
-            self.assertEqual(item['stored_image'], [results[0][1]])
+
+class ImagesPipelineTestCaseFieldsDict(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
+    item_class = dict
+
+
+class ImagesPipelineTestItem(Item):
+    name = Field()
+    # default fields
+    image_urls = Field()
+    images = Field()
+    # overridden fields
+    custom_image_urls = Field()
+    custom_images = Field()
+
+
+class ImagesPipelineTestCaseFieldsItem(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
+    item_class = ImagesPipelineTestItem
+
+
+@skipIf(not make_dataclass, "dataclasses module is not available")
+class ImagesPipelineTestCaseFieldsDataClass(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
+    item_class = None
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if make_dataclass:
+            self.item_class = make_dataclass(
+                "FilesPipelineTestDataClass",
+                [
+                    ("name", str),
+                    # default fields
+                    ("image_urls", list, dataclass_field(default_factory=list)),
+                    ("images", list, dataclass_field(default_factory=list)),
+                    # overridden fields
+                    ("custom_image_urls", list, dataclass_field(default_factory=list)),
+                    ("custom_images", list, dataclass_field(default_factory=list)),
+                ],
+            )
+
+
+@attr.s
+class ImagesPipelineTestAttrsItem:
+    name = attr.ib(default="")
+    # default fields
+    image_urls = attr.ib(default=lambda: [])
+    images = attr.ib(default=lambda: [])
+    # overridden fields
+    custom_image_urls = attr.ib(default=lambda: [])
+    custom_images = attr.ib(default=lambda: [])
+
+
+class ImagesPipelineTestCaseFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
+    item_class = ImagesPipelineTestAttrsItem
 
 
 class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py
index 6dc117779..daf022aee 100644
--- a/tests/test_utils_serialize.py
+++ b/tests/test_utils_serialize.py
@@ -1,18 +1,25 @@
+import datetime
 import json
 import unittest
-import datetime
 from decimal import Decimal
 
+import attr
 from twisted.internet import defer
 
-from scrapy.utils.serialize import ScrapyJSONEncoder
 from scrapy.http import Request, Response
+from scrapy.utils.serialize import ScrapyJSONEncoder
+
+
+try:
+    from dataclasses import make_dataclass
+except ImportError:
+    make_dataclass = None
 
 
 class JsonEncoderTestCase(unittest.TestCase):
 
     def setUp(self):
-        self.encoder = ScrapyJSONEncoder()
+        self.encoder = ScrapyJSONEncoder(sort_keys=True)
 
     def test_encode_decode(self):
         dt = datetime.datetime(2010, 1, 2, 10, 11, 12)
@@ -31,7 +38,8 @@ class JsonEncoderTestCase(unittest.TestCase):
         for input, output in [('foo', 'foo'), (d, ds), (t, ts), (dt, dts),
                               (dec, decs), (['foo', d], ['foo', ds]), (s, ss),
                               (dt_set, dt_sets)]:
-            self.assertEqual(self.encoder.encode(input), json.dumps(output))
+            self.assertEqual(self.encoder.encode(input),
+                             json.dumps(output, sort_keys=True))
 
     def test_encode_deferred(self):
         self.assertIn('Deferred', self.encoder.encode(defer.Deferred()))
@@ -47,3 +55,30 @@ class JsonEncoderTestCase(unittest.TestCase):
         rs = self.encoder.encode(r)
         self.assertIn(r.url, rs)
         self.assertIn(str(r.status), rs)
+
+    @unittest.skipIf(not make_dataclass, "No dataclass support")
+    def test_encode_dataclass_item(self):
+        TestDataClass = make_dataclass(
+            "TestDataClass",
+            [("name", str), ("url", str), ("price", int)],
+        )
+        item = TestDataClass(name="Product", url="http://product.org", price=1)
+        encoded = self.encoder.encode(item)
+        self.assertEqual(
+            encoded,
+            '{"name": "Product", "price": 1, "url": "http://product.org"}'
+        )
+
+    def test_encode_attrs_item(self):
+        @attr.s
+        class AttrsItem:
+            name = attr.ib(type=str)
+            url = attr.ib(type=str)
+            price = attr.ib(type=int)
+
+        item = AttrsItem(name="Product", url="http://product.org", price=1)
+        encoded = self.encoder.encode(item)
+        self.assertEqual(
+            encoded,
+            '{"name": "Product", "price": 1, "url": "http://product.org"}'
+        )
diff --git a/tox.ini b/tox.ini
index 69b1bdfdd..4c790158d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -37,7 +37,7 @@ deps =
     pytest-flake8
 commands =
     py.test --flake8 {posargs:docs scrapy tests}
-    
+
 [testenv:pylint]
 basepython = python3
 deps =
@@ -62,6 +62,7 @@ deps =
     -ctests/constraints.txt
     cryptography==2.0
     cssselect==0.9.1
+    itemadapter==0.1.0
     lxml==3.5.0
     parsel==1.5.0
     Protego==0.1.15

From 214da8e5235c9676bc4b0427c8bf58f328fe6570 Mon Sep 17 00:00:00 2001
From: Ram Rachum <ram@rachum.com>
Date: Wed, 17 Jun 2020 13:50:54 +0300
Subject: [PATCH 173/243] Use chain.from_iterable in python.py

---
 scrapy/utils/python.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 9c1f3c2fe..afa8a8135 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -334,10 +334,10 @@ class MutableChain:
     """
 
     def __init__(self, *args):
-        self.data = chain(*args)
+        self.data = chain.from_iterable(args)
 
     def extend(self, *iterables):
-        self.data = chain(self.data, *iterables)
+        self.data = chain(self.data, chain.from_iterable(iterables))
 
     def __iter__(self):
         return self

From 3d027fb578532d504b3dbfaa77a06c3560f85d3c Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Wed, 17 Jun 2020 18:08:14 +0300
Subject: [PATCH 174/243] Fix missing storage.store calls in
 FeedExporter.close_spider (#4626)

---
 scrapy/extensions/feedexport.py |  4 ++-
 tests/test_feedexport.py        | 49 ++++++++++++++++++++++++++++++++-
 2 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 998d2a5d1..30e6349d6 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -270,7 +270,9 @@ class FeedExporter:
             if not slot.itemcount and not slot.store_empty:
                 # We need to call slot.storage.store nonetheless to get the file
                 # properly closed.
-                return defer.maybeDeferred(slot.storage.store, slot.file)
+                d = defer.maybeDeferred(slot.storage.store, slot.file)
+                deferred_list.append(d)
+                continue
             slot.finish_exporting()
             logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
             log_args = {'format': slot.format,
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 8eeb29b6d..f7013bc44 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -7,6 +7,7 @@ import string
 import tempfile
 import warnings
 from io import BytesIO
+from logging import getLogger
 from pathlib import Path
 from string import ascii_letters, digits
 from unittest import mock
@@ -14,9 +15,11 @@ from urllib.parse import urljoin, urlparse, quote
 from urllib.request import pathname2url
 
 import lxml.etree
+from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial import unittest
-from w3lib.url import path_to_file_uri
+from w3lib.url import file_uri_to_path, path_to_file_uri
+from zope.interface import implementer
 from zope.interface.verify import verifyObject
 
 import scrapy
@@ -390,6 +393,25 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):
     pass
 
 
+@implementer(IFeedStorage)
+class LogOnStoreFileStorage:
+    """
+    This storage logs inside `store` method.
+    It can be used to make sure `store` method is invoked.
+    """
+
+    def __init__(self, uri):
+        self.path = file_uri_to_path(uri)
+        self.logger = getLogger()
+
+    def open(self, spider):
+        return tempfile.NamedTemporaryFile(prefix='feed-')
+
+    def store(self, file):
+        self.logger.info('Storage.store is called')
+        file.close()
+
+
 class FeedExportTest(unittest.TestCase):
 
     class MyItem(scrapy.Item):
@@ -426,11 +448,17 @@ class FeedExportTest(unittest.TestCase):
                 yield runner.crawl(spider_cls)
 
             for file_path, feed in FEEDS.items():
+                if not os.path.exists(str(file_path)):
+                    continue
+
                 with open(str(file_path), 'rb') as f:
                     content[feed['format']] = f.read()
 
         finally:
             for file_path in FEEDS.keys():
+                if not os.path.exists(str(file_path)):
+                    continue
+
                 os.remove(str(file_path))
 
         return content
@@ -623,6 +651,25 @@ class FeedExportTest(unittest.TestCase):
             data = yield self.exported_no_data(settings)
             self.assertEqual(data[fmt], expctd)
 
+    @defer.inlineCallbacks
+    def test_export_no_items_multiple_feeds(self):
+        """ Make sure that `storage.store` is called for every feed. """
+        settings = {
+            'FEEDS': {
+                self._random_temp_filename(): {'format': 'json'},
+                self._random_temp_filename(): {'format': 'xml'},
+                self._random_temp_filename(): {'format': 'csv'},
+            },
+            'FEED_STORAGES': {'file': 'tests.test_feedexport.LogOnStoreFileStorage'},
+            'FEED_STORE_EMPTY': False
+        }
+
+        with LogCapture() as log:
+            yield self.exported_no_data(settings)
+
+        print(log)
+        self.assertEqual(str(log).count('Storage.store is called'), 3)
+
     @defer.inlineCallbacks
     def test_export_multiple_item_classes(self):
 

From 5d541731870eaaa3dd658673b82a7d0ca056f689 Mon Sep 17 00:00:00 2001
From: Devi Sandeep <sandeep0138@gmail.com>
Date: Thu, 18 Jun 2020 05:01:38 -0500
Subject: [PATCH 175/243] Update docs on accessing callback arguments in
 errback (#4634)

---
 docs/topics/request-response.rst | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index bbd715766..d88d40b00 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -189,6 +189,10 @@ Request objects
         cloned using the ``copy()`` or ``replace()`` methods, and can also be
         accessed, in your spider, from the ``response.cb_kwargs`` attribute.
 
+        In case of a failure to process the request, this dict can be accessed as
+        ``failure.request.cb_kwargs`` in the request's errback. For more information,
+        see :ref:`topics-request-response-ref-accessing-callback-arguments-in-errback`.
+
     .. method:: Request.copy()
 
        Return a new Request which is a copy of this Request. See also:
@@ -312,6 +316,31 @@ errors if needed::
                 request = failure.request
                 self.logger.error('TimeoutError on %s', request.url)
 
+.. _topics-request-response-ref-accessing-callback-arguments-in-errback:
+
+Accessing additional data in errback functions
+----------------------------------------------
+
+In case of a failure to process the request, you may be interested in
+accessing arguments to the callback functions so you can process further
+based on the arguments in the errback. The following example shows how to
+achieve this by using ``Failure.request.cb_kwargs``::
+
+    def parse(self, response):
+        request = scrapy.Request('http://www.example.com/index.html',
+                                 callback=self.parse_page2,
+                                 errback=self.errback_page2,
+                                 cb_kwargs=dict(main_url=response.url))
+        yield request
+
+    def parse_page2(self, response, main_url):
+        pass
+
+    def errback_page2(self, failure):
+        yield dict(
+            main_url=failure.request.cb_kwargs['main_url'],
+        )
+
 .. _topics-request-meta:
 
 Request.meta special keys

From 7babf359e0221613b872b5b204e7c523b7b84486 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 18 Jun 2020 13:52:04 -0300
Subject: [PATCH 176/243] Typing: Tox env, CI job

---
 .gitignore  |   1 +
 .travis.yml |   2 +
 setup.cfg   | 171 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 tox.ini     |   7 +++
 4 files changed, 181 insertions(+)

diff --git a/.gitignore b/.gitignore
index ff6e2ea65..83a2569dd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,7 @@ htmlcov/
 .pytest_cache/
 .coverage.*
 .cache/
+.mypy_cache/
 
 # Windows
 Thumbs.db
diff --git a/.travis.yml b/.travis.yml
index e44f85237..b403ac54c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,6 +15,8 @@ matrix:
       python: 3.8
     - env: TOXENV=docs
       python: 3.7  # Keep in sync with .readthedocs.yml
+    - env: TOXENV=typing
+      python: 3.8
 
     - env: TOXENV=pypy3
     - env: TOXENV=pinned
diff --git a/setup.cfg b/setup.cfg
index 2296a1052..a9138c1c0 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -3,3 +3,174 @@ doc_files = docs AUTHORS INSTALL LICENSE README.rst
 
 [bdist_wheel]
 universal=1
+
+[mypy]
+ignore_missing_imports = true
+follow_imports = skip
+
+# FIXME: remove the following sections once the issues are solved
+
+[mypy-scrapy]
+ignore_errors = True
+
+[mypy-scrapy._monkeypatches]
+ignore_errors = True
+
+[mypy-scrapy.commands]
+ignore_errors = True
+
+[mypy-scrapy.commands.bench]
+ignore_errors = True
+
+[mypy-scrapy.commands.parse]
+ignore_errors = True
+
+[mypy-scrapy.downloadermiddlewares.httpproxy]
+ignore_errors = True
+
+[mypy-scrapy.contracts]
+ignore_errors = True
+
+[mypy-scrapy.core.spidermw]
+ignore_errors = True
+
+[mypy-scrapy.interfaces]
+ignore_errors = True
+
+[mypy-scrapy.item]
+ignore_errors = True
+
+[mypy-scrapy.http.cookies]
+ignore_errors = True
+
+[mypy-scrapy.mail]
+ignore_errors = True
+
+[mypy-scrapy.pipelines.images]
+ignore_errors = True
+
+[mypy-scrapy.settings.default_settings]
+ignore_errors = True
+
+[mypy-scrapy.spidermiddlewares.referer]
+ignore_errors = True
+
+[mypy-scrapy.utils.httpobj]
+ignore_errors = True
+
+[mypy-scrapy.utils.request]
+ignore_errors = True
+
+[mypy-scrapy.utils.response]
+ignore_errors = True
+
+[mypy-scrapy.utils.spider]
+ignore_errors = True
+
+[mypy-scrapy.utils.trackref]
+ignore_errors = True
+
+[mypy-tests.mocks.dummydbm]
+ignore_errors = True
+
+[mypy-tests.spiders]
+ignore_errors = True
+
+[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.exception]
+ignore_errors = True
+
+[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.normal]
+ignore_errors = True
+
+[mypy-tests.test_command_fetch]
+ignore_errors = True
+
+[mypy-tests.test_command_parse]
+ignore_errors = True
+
+[mypy-tests.test_command_shell]
+ignore_errors = True
+
+[mypy-tests.test_command_version]
+ignore_errors = True
+
+[mypy-tests.test_contracts]
+ignore_errors = True
+
+[mypy-tests.test_crawler]
+ignore_errors = True
+
+[mypy-tests.test_downloader_handlers]
+ignore_errors = True
+
+[mypy-tests.test_engine]
+ignore_errors = True
+
+[mypy-tests.test_exporters]
+ignore_errors = True
+
+[mypy-tests.test_http_request]
+ignore_errors = True
+
+[mypy-tests.test_linkextractors]
+ignore_errors = True
+
+[mypy-tests.test_loader]
+ignore_errors = True
+
+[mypy-tests.test_pipeline_crawl]
+ignore_errors = True
+
+[mypy-tests.test_pipeline_files]
+ignore_errors = True
+
+[mypy-tests.test_pipeline_images]
+ignore_errors = True
+
+[mypy-tests.test_pipelines]
+ignore_errors = True
+
+[mypy-tests.test_request_cb_kwargs]
+ignore_errors = True
+
+[mypy-tests.test_request_left]
+ignore_errors = True
+
+[mypy-tests.test_scheduler]
+ignore_errors = True
+
+[mypy-tests.test_signals]
+ignore_errors = True
+
+[mypy-tests.test_spiderloader.test_spiders.nested.spider4]
+ignore_errors = True
+
+[mypy-tests.test_spiderloader.test_spiders.spider1]
+ignore_errors = True
+
+[mypy-tests.test_spiderloader.test_spiders.spider2]
+ignore_errors = True
+
+[mypy-tests.test_spiderloader.test_spiders.spider3]
+ignore_errors = True
+
+[mypy-tests.test_spidermiddleware_httperror]
+ignore_errors = True
+
+[mypy-tests.test_spidermiddleware_output_chain]
+ignore_errors = True
+
+[mypy-tests.test_spidermiddleware_referer]
+ignore_errors = True
+
+[mypy-tests.test_utils_reqser]
+ignore_errors = True
+
+[mypy-tests.test_utils_serialize]
+ignore_errors = True
+
+[mypy-tests.test_utils_spider]
+ignore_errors = True
+
+[mypy-tests.test_utils_url]
+ignore_errors = True
diff --git a/tox.ini b/tox.ini
index 4c790158d..27d21ade2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -23,6 +23,13 @@ passenv =
 commands =
     py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests}
 
+[testenv:typing]
+basepython = python3
+deps =
+    mypy==0.780
+commands =
+    mypy {posargs: scrapy tests}
+
 [testenv:security]
 basepython = python3
 deps =

From a4bfd5ab6fd75c4badac1c5d9b40706181c41bd9 Mon Sep 17 00:00:00 2001
From: Stanislau Hluboki <glubokiy.stas@gmail.com>
Date: Sat, 13 Jun 2020 18:04:38 +0300
Subject: [PATCH 177/243] Fix duplicated feed logs

---
 scrapy/extensions/feedexport.py | 19 +++++++---
 tests/test_feedexport.py        | 63 +++++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+), 5 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 30e6349d6..61dad8726 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -279,11 +279,20 @@ class FeedExporter:
                         'itemcount': slot.itemcount,
                         'uri': slot.uri}
             d = defer.maybeDeferred(slot.storage.store, slot.file)
-            d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args,
-                                                extra={'spider': spider}))
-            d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args,
-                                                exc_info=failure_to_exc_info(f),
-                                                extra={'spider': spider}))
+
+            # Use `largs=log_args` to copy log_args into function's scope
+            # instead of using `log_args` from the outer scope
+            d.addCallback(
+                lambda _, largs=log_args: logger.info(
+                    logfmt % "Stored", largs, extra={'spider': spider}
+                )
+            )
+            d.addErrback(
+                lambda f, largs=log_args: logger.error(
+                    logfmt % "Error storing", largs,
+                    exc_info=failure_to_exc_info(f), extra={'spider': spider}
+                )
+            )
             deferred_list.append(d)
         return defer.DeferredList(deferred_list) if deferred_list else None
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index f7013bc44..e38644214 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -393,6 +393,27 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):
     pass
 
 
+class DummyBlockingFeedStorage(BlockingFeedStorage):
+
+    def __init__(self, uri):
+        self.path = file_uri_to_path(uri)
+
+    def _store_in_thread(self, file):
+        dirname = os.path.dirname(self.path)
+        if dirname and not os.path.exists(dirname):
+            os.makedirs(dirname)
+        with open(self.path, 'ab') as output_file:
+            output_file.write(file.read())
+
+        file.close()
+
+
+class FailingBlockingFeedStorage(DummyBlockingFeedStorage):
+
+    def _store_in_thread(self, file):
+        raise OSError('Cannot store')
+
+
 @implementer(IFeedStorage)
 class LogOnStoreFileStorage:
     """
@@ -1025,3 +1046,45 @@ class FeedExportTest(unittest.TestCase):
         }
         data = yield self.exported_no_data(settings)
         self.assertEqual(data['csv'], b'')
+
+    @defer.inlineCallbacks
+    def test_multiple_feeds_success_logs_blocking_feed_storage(self):
+        settings = {
+            'FEEDS': {
+                self._random_temp_filename(): {'format': 'json'},
+                self._random_temp_filename(): {'format': 'xml'},
+                self._random_temp_filename(): {'format': 'csv'},
+            },
+            'FEED_STORAGES': {'file': 'tests.test_feedexport.DummyBlockingFeedStorage'},
+        }
+        items = [
+            {'foo': 'bar1', 'baz': ''},
+            {'foo': 'bar2', 'baz': 'quux'},
+        ]
+        with LogCapture() as log:
+            yield self.exported_data(items, settings)
+
+        print(log)
+        for fmt in ['json', 'xml', 'csv']:
+            self.assertIn('Stored %s feed (2 items)' % fmt, str(log))
+
+    @defer.inlineCallbacks
+    def test_multiple_feeds_failing_logs_blocking_feed_storage(self):
+        settings = {
+            'FEEDS': {
+                self._random_temp_filename(): {'format': 'json'},
+                self._random_temp_filename(): {'format': 'xml'},
+                self._random_temp_filename(): {'format': 'csv'},
+            },
+            'FEED_STORAGES': {'file': 'tests.test_feedexport.FailingBlockingFeedStorage'},
+        }
+        items = [
+            {'foo': 'bar1', 'baz': ''},
+            {'foo': 'bar2', 'baz': 'quux'},
+        ]
+        with LogCapture() as log:
+            yield self.exported_data(items, settings)
+
+        print(log)
+        for fmt in ['json', 'xml', 'csv']:
+            self.assertIn('Error storing %s feed (2 items)' % fmt, str(log))

From b99fe4aa4c35ff8f5c2355a09988ddb1e90d6b70 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 19 Jun 2020 21:41:15 -0300
Subject: [PATCH 178/243] Add google-cloud-storage to the 'pinned' tox
 environment

---
 tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tox.ini b/tox.ini
index 4c790158d..d28eab131 100644
--- a/tox.ini
+++ b/tox.ini
@@ -76,6 +76,7 @@ deps =
     -rtests/requirements-py3.txt
     # Extras
     botocore==1.3.23
+    google-cloud-storage==1.29.0
     Pillow==3.4.2
 
 [testenv:extra-deps]

From 3efea98e0518cd0d3b92b06f6b44b701dff1e53d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 22 Jun 2020 12:41:14 -0300
Subject: [PATCH 179/243] Docs: add note about dataclass items and loaders

---
 docs/topics/loaders.rst | 37 ++++++++++++++++++++++++++++++++++++-
 1 file changed, 36 insertions(+), 1 deletion(-)

diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 6645bf123..d70e03ad4 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -26,7 +26,7 @@ Using Item Loaders to populate items
 To use an Item Loader, you must first instantiate it. You can either
 instantiate it with an :ref:`item object <topics-items>` or without one, in which
 case an instance of :class:`~scrapy.item.Item` is automatically created in the
-Item Loader ``__init__`` method using the :class:`~scrapy.item.Item` subclass
+Item Loader ``__init__`` method using the :ref:`item <topics-items>` class
 specified in the :attr:`ItemLoader.default_item_class` attribute.
 
 Then, you start collecting values into the Item Loader, typically using
@@ -76,6 +76,41 @@ called which actually returns the item populated with the data
 previously extracted and collected with the :meth:`~ItemLoader.add_xpath`,
 :meth:`~ItemLoader.add_css`, and :meth:`~ItemLoader.add_value` calls.
 
+
+.. _topics-loaders-dataclass:
+
+Working with dataclass items
+============================
+
+By default, :ref:`dataclass items <dataclass-items>` require all fields to be
+passed when created. This could be an issue when using dataclass items with
+item loaders, since fields could be populated incrementally.
+
+Given the way that item loaders store data internally, the recommended approach
+to overcome this is to define items using the :func:`~dataclasses.field`
+function, with ``list`` as the ``default_factory`` argument::
+
+    from dataclasses import dataclass, field
+
+    @dataclass
+    class InventoryItem:
+        name: str = field(default_factory=list)
+        price: float = field(default_factory=list)
+        stock: int = field(default_factory=list)
+
+Note that in order to keep the example simple, the types do not match
+completely. A more accurate but verbose definition would be::
+
+    from dataclasses import dataclass, field
+    from typing import List, Union
+
+    @dataclass
+    class InventoryItem:
+        name: Union[str, List[str]] = field(default_factory=list)
+        price: Union[float, List[float]] = field(default_factory=list)
+        stock: Union[int, List[int]] = field(default_factory=list)
+
+
 .. _topics-loaders-processors:
 
 Input and Output processors

From 1335d9053e08c0321927728060d4eba0b6da687b Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Mon, 22 Jun 2020 14:05:44 -0300
Subject: [PATCH 180/243] Update docs/topics/loaders.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/loaders.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index d70e03ad4..c5f121dbf 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -25,7 +25,7 @@ Using Item Loaders to populate items
 
 To use an Item Loader, you must first instantiate it. You can either
 instantiate it with an :ref:`item object <topics-items>` or without one, in which
-case an instance of :class:`~scrapy.item.Item` is automatically created in the
+case an :ref:`item object <topics-items>` is automatically created in the
 Item Loader ``__init__`` method using the :ref:`item <topics-items>` class
 specified in the :attr:`ItemLoader.default_item_class` attribute.
 

From 73b6ce8cb560da71b082efdba44b90e1ea932b17 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 22 Jun 2020 14:13:37 -0300
Subject: [PATCH 181/243] Update docs about dataclass items and loaders

---
 docs/topics/loaders.rst | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index c5f121dbf..e921395d2 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -84,9 +84,11 @@ Working with dataclass items
 
 By default, :ref:`dataclass items <dataclass-items>` require all fields to be
 passed when created. This could be an issue when using dataclass items with
-item loaders, since fields could be populated incrementally.
+item loaders: unless a pre-populated item is passed to the loader, fields
+will be populated incrementally using the loader's :meth:`~ItemLoader.add_xpath`,
+:meth:`~ItemLoader.add_css` and :meth:`~ItemLoader.add_value` methods.
 
-Given the way that item loaders store data internally, the recommended approach
+Given the way that item loaders store data internally, one approach
 to overcome this is to define items using the :func:`~dataclasses.field`
 function, with ``list`` as the ``default_factory`` argument::
 

From cfd039aeb6d46f7c120edd82ce06d6b9f4f8e7db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 22 Jun 2020 19:28:33 +0200
Subject: [PATCH 182/243] Remove a duplicate GCS_PROJECT_ID reference target

---
 docs/topics/media-pipeline.rst | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 01de3dedb..096618648 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -204,7 +204,6 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
 Google Cloud Storage
 ---------------------
 
-.. setting:: GCS_PROJECT_ID
 .. setting:: FILES_STORE_GCS_ACL
 .. setting:: IMAGES_STORE_GCS_ACL
 

From 3672f5f988cbb29c6bebb7ed535c0d5ad941d868 Mon Sep 17 00:00:00 2001
From: Lukas Anzinger <lukas@lukasanzinger.at>
Date: Tue, 23 Jun 2020 14:51:21 +0200
Subject: [PATCH 183/243] Spider constructor expects name as argument, not
 start_urls.

Fixes #4644
---
 tests/test_scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 930a5dd99..2b6cb0902 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -296,7 +296,7 @@ class StartUrlsSpider(Spider):
 
     def __init__(self, start_urls):
         self.start_urls = start_urls
-        super(StartUrlsSpider, self).__init__(start_urls)
+        super(StartUrlsSpider, self).__init__(name='StartUrlsSpider')
 
     def parse(self, response):
         pass

From 6f4c964aa4949584cfd251cf2aa2c0c7de6cf251 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 24 Jun 2020 12:26:38 +0200
Subject: [PATCH 184/243] Cover Scrapy 2.2.0 in the release notes (#4630)

---
 docs/contributing.rst            |   5 +-
 docs/news.rst                    | 195 +++++++++++++++++++++++++++++++
 docs/topics/media-pipeline.rst   |   9 +-
 docs/topics/request-response.rst |   4 +-
 scrapy/http/response/text.py     |   2 +
 scrapy/utils/misc.py             |   5 +-
 6 files changed, 214 insertions(+), 6 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index aed5ab92e..7b901dd00 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -155,6 +155,9 @@ Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports
 removal, etc) in separate commits from functional changes. This will make pull
 requests easier to review and more likely to get merged.
 
+
+.. _coding-style:
+
 Coding style
 ============
 
@@ -163,7 +166,7 @@ Scrapy:
 
 * Unless otherwise specified, follow :pep:`8`.
 
-* It's OK to use lines longer than 80 chars if it improves the code
+* It's OK to use lines longer than 79 chars if it improves the code
   readability.
 
 * Don't put your name in the code you contribute; git provides enough
diff --git a/docs/news.rst b/docs/news.rst
index a158246eb..80d130e4a 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,201 @@
 Release notes
 =============
 
+.. _release-2.2.0:
+
+Scrapy 2.2.0 (2020-06-24)
+-------------------------
+
+Highlights:
+
+* Python 3.5.2+ is required now
+* :ref:`dataclass objects <dataclass-items>` and
+  :ref:`attrs objects <attrs-items>` are now valid :ref:`item types
+  <item-types>`
+* New :meth:`TextResponse.json <scrapy.http.TextResponse.json>` method
+* New :signal:`bytes_received` signal that allows canceling response download
+* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` fixes
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+*   Support for Python 3.5.0 and 3.5.1 has been dropped; Scrapy now refuses to
+    run with a Python version lower than 3.5.2, which introduced
+    :class:`typing.Type` (:issue:`4615`)
+
+
+Deprecations
+~~~~~~~~~~~~
+
+*   :meth:`TextResponse.body_as_unicode
+    <scrapy.http.TextResponse.body_as_unicode>` is now deprecated, use
+    :attr:`TextResponse.text <scrapy.http.TextResponse.text>` instead
+    (:issue:`4546`, :issue:`4555`, :issue:`4579`)
+
+*   :class:`scrapy.item.BaseItem` is now deprecated, use
+    :class:`scrapy.item.Item` instead (:issue:`4534`)
+
+
+New features
+~~~~~~~~~~~~
+
+*   :ref:`dataclass objects <dataclass-items>` and
+    :ref:`attrs objects <attrs-items>` are now valid :ref:`item types
+    <item-types>`, and a new itemadapter_ library makes it easy to
+    write code that :ref:`supports any item type <supporting-item-types>`
+    (:issue:`2749`, :issue:`2807`, :issue:`3761`, :issue:`3881`, :issue:`4642`)
+
+*   A new :meth:`TextResponse.json <scrapy.http.TextResponse.json>` method
+    allows to deserialize JSON responses (:issue:`2444`, :issue:`4460`,
+    :issue:`4574`)
+
+*   A new :signal:`bytes_received` signal allows monitoring response download
+    progress and :ref:`stopping downloads <topics-stop-response-download>`
+    (:issue:`4205`, :issue:`4559`)
+
+*   The dictionaries in the result list of a :ref:`media pipeline
+    <topics-media-pipeline>` now include a new key, ``status``, which indicates
+    if the file was downloaded or, if the file was not downloaded, why it was
+    not downloaded; see :meth:`FilesPipeline.get_media_requests
+    <scrapy.pipelines.files.FilesPipeline.get_media_requests>` for more
+    information (:issue:`2893`, :issue:`4486`)
+
+*   When using :ref:`Google Cloud Storage <media-pipeline-gcs>` for
+    a :ref:`media pipeline <topics-media-pipeline>`, a warning is now logged if
+    the configured credentials do not grant the required permissions
+    (:issue:`4346`, :issue:`4508`)
+
+*   :ref:`Link extractors <topics-link-extractors>` are now serializable,
+    as long as you do not use :ref:`lambdas <lambda>` for parameters; for
+    example, you can now pass link extractors in :attr:`Request.cb_kwargs
+    <scrapy.http.Request.cb_kwargs>` or
+    :attr:`Request.meta <scrapy.http.Request.meta>` when :ref:`persisting
+    scheduled requests <topics-jobs>` (:issue:`4554`)
+
+*   Upgraded the :ref:`pickle protocol <pickle-protocols>` that Scrapy uses
+    from protocol 2 to protocol 4, improving serialization capabilities and
+    performance (:issue:`4135`, :issue:`4541`)
+
+*   :func:`scrapy.utils.misc.create_instance` now raises a :exc:`TypeError`
+    exception if the resulting instance is ``None`` (:issue:`4528`,
+    :issue:`4532`)
+
+.. _itemadapter: https://github.com/scrapy/itemadapter
+
+
+Bug fixes
+~~~~~~~~~
+
+*   :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer
+    discards cookies defined in :attr:`Request.headers
+    <scrapy.http.Request.headers>` (:issue:`1992`, :issue:`2400`)
+
+*   :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer
+    re-encodes cookies defined as :class:`bytes` in the ``cookies`` parameter
+    of the ``__init__`` method of :class:`~scrapy.http.Request`
+    (:issue:`2400`, :issue:`3575`)
+
+*   When :setting:`FEEDS` defines multiple URIs, :setting:`FEED_STORE_EMPTY` is
+    ``False`` and the crawl yields no items, Scrapy no longer stops feed
+    exports after the first URI (:issue:`4621`, :issue:`4626`)
+
+*   :class:`~scrapy.spiders.Spider` callbacks defined using :doc:`coroutine
+    syntax <topics/coroutines>` no longer need to return an iterable, and may
+    instead return a :class:`~scrapy.http.Request` object, an
+    :ref:`item <topics-items>`, or ``None`` (:issue:`4609`)
+
+*   The :command:`startproject` command now ensures that the generated project
+    folders and files have the right permissions (:issue:`4604`)
+
+*   Fix a :exc:`KeyError` exception being sometimes raised from
+    :class:`scrapy.utils.datatypes.LocalWeakReferencedCache` (:issue:`4597`,
+    :issue:`4599`)
+
+*   When :setting:`FEEDS` defines multiple URIs, log messages about items being
+    stored now contain information from the corresponding feed, instead of
+    always containing information about only one of the feeds (:issue:`4619`,
+    :issue:`4629`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+*   Added a new section about :ref:`accessing cb_kwargs from errbacks
+    <errback-cb_kwargs>` (:issue:`4598`, :issue:`4634`)
+
+*   Covered chompjs_ in :ref:`topics-parsing-javascript` (:issue:`4556`,
+    :issue:`4562`)
+
+*   Removed from :doc:`topics/coroutines` the warning about the API being
+    experimental (:issue:`4511`, :issue:`4513`)
+
+*   Removed references to unsupported versions of :doc:`Twisted
+    <twisted:index>` (:issue:`4533`)
+
+*   Updated the description of the :ref:`screenshot pipeline example
+    <ScreenshotPipeline>`, which now uses :doc:`coroutine syntax
+    <topics/coroutines>` instead of returning a
+    :class:`~twisted.internet.defer.Deferred` (:issue:`4514`, :issue:`4593`)
+
+*   Removed a misleading import line from the
+    :func:`scrapy.utils.log.configure_logging` code example (:issue:`4510`,
+    :issue:`4587`)
+
+*   The display-on-hover behavior of internal documentation references now also
+    covers links to :ref:`commands <topics-commands>`, :attr:`Request.meta
+    <scrapy.http.Request.meta>` keys, :ref:`settings <topics-settings>` and
+    :ref:`signals <topics-signals>` (:issue:`4495`, :issue:`4563`)
+
+*   It is again possible to download the documentation for offline reading
+    (:issue:`4578`, :issue:`4585`)
+
+*   Removed backslashes preceding ``*args`` and ``**kwargs`` in some function
+    and method signatures (:issue:`4592`, :issue:`4596`)
+
+.. _chompjs: https://github.com/Nykakin/chompjs
+
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+*   Adjusted the code base further to our :ref:`style guidelines
+    <coding-style>` (:issue:`4237`, :issue:`4525`, :issue:`4538`,
+    :issue:`4539`, :issue:`4540`, :issue:`4542`, :issue:`4543`, :issue:`4544`,
+    :issue:`4545`, :issue:`4557`, :issue:`4558`, :issue:`4566`, :issue:`4568`,
+    :issue:`4572`)
+
+*   Removed remnants of Python 2 support (:issue:`4550`, :issue:`4553`,
+    :issue:`4568`)
+
+*   Improved code sharing between the :command:`crawl` and :command:`runspider`
+    commands (:issue:`4548`, :issue:`4552`)
+
+*   Replaced ``chain(*iterable)`` with ``chain.from_iterable(iterable)``
+    (:issue:`4635`)
+
+*   You may now run the :mod:`asyncio` tests with Tox on any Python version
+    (:issue:`4521`)
+
+*   Updated test requirements to reflect an incompatibility with pytest 5.4 and
+    5.4.1 (:issue:`4588`)
+
+*   Improved :class:`~scrapy.spiderloader.SpiderLoader` test coverage for
+    scenarios involving duplicate spider names (:issue:`4549`, :issue:`4560`)
+
+*   Configured Travis CI to also run the tests with Python 3.5.2
+    (:issue:`4518`, :issue:`4615`)
+
+*   Added a `Pylint <https://www.pylint.org/>`_ job to Travis CI
+    (:issue:`3727`)
+
+*   Added a `Mypy <http://mypy-lang.org/>`_ job to Travis CI (:issue:`4637`)
+
+*   Made use of set literals in tests (:issue:`4573`)
+
+*   Cleaned up the Travis CI configuration (:issue:`4517`, :issue:`4519`,
+    :issue:`4522`, :issue:`4537`)
+
+
 .. _release-2.1.0:
 
 Scrapy 2.1.0 (2020-04-24)
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 01de3dedb..9f2a06dd7 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -201,6 +201,9 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
 .. _s3.scality: https://s3.scality.com/
 .. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
+
+.. _media-pipeline-gcs:
+
 Google Cloud Storage
 ---------------------
 
@@ -475,7 +478,11 @@ See here the methods that you can override in your custom Files Pipeline:
 
         * ``checksum`` - a `MD5 hash`_ of the image contents
 
-        * ``status`` - the file status indication. It can be one of the following:
+        * ``status`` - the file status indication.
+
+          .. versionadded:: 2.2
+
+          It can be one of the following:
 
           * ``downloaded`` - file was downloaded.
           * ``uptodate`` - file was not downloaded, as it was downloaded recently,
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index d88d40b00..fbd8e4b73 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -191,7 +191,7 @@ Request objects
 
         In case of a failure to process the request, this dict can be accessed as
         ``failure.request.cb_kwargs`` in the request's errback. For more information,
-        see :ref:`topics-request-response-ref-accessing-callback-arguments-in-errback`.
+        see :ref:`errback-cb_kwargs`.
 
     .. method:: Request.copy()
 
@@ -316,7 +316,7 @@ errors if needed::
                 request = failure.request
                 self.logger.error('TimeoutError on %s', request.url)
 
-.. _topics-request-response-ref-accessing-callback-arguments-in-errback:
+.. _errback-cb_kwargs:
 
 Accessing additional data in errback functions
 ----------------------------------------------
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 40cf3f483..b43fe5c19 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -74,6 +74,8 @@ class TextResponse(Response):
 
     def json(self):
         """
+        .. versionadded:: 2.2
+
         Deserialize a JSON document to a Python object.
         """
         if self._cached_decoded_json is _NONE:
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index a7808cb2c..8e5fde246 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -138,8 +138,9 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
 
     Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
 
-    Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
-    extension has not been implemented correctly).
+    .. versionchanged:: 2.2
+       Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
+       extension has not been implemented correctly).
     """
     if settings is None:
         if crawler is None:

From 9f60481360628bafe467f59b2144d69204b8b4e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 24 Jun 2020 12:27:39 +0200
Subject: [PATCH 185/243] =?UTF-8?q?Bump=20version:=202.1.0=20=E2=86=92=202?=
 =?UTF-8?q?.2.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index de22a2783..8d4d74bc5 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.1.0
+current_version = 2.2.0
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 7ec1d6db4..ccbccc3dc 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-2.1.0
+2.2.0

From c3cee74fd401e6a6307b5eb1786e532bb2cd5aa8 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Fri, 26 Jun 2020 18:45:21 +0300
Subject: [PATCH 186/243] Change default value of FEED_STORAGE_BATCH_ITEM_COUNT
 to 0

---
 docs/topics/feed-exports.rst        |  2 +-
 scrapy/extensions/feedexport.py     |  2 +-
 scrapy/settings/default_settings.py |  2 +-
 tests/test_feedexport.py            | 20 ++++++++++----------
 4 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 866ce78eb..0b37e9a7d 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -435,7 +435,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 
 FEED_STORAGE_BATCH_ITEM_COUNT
 -----------------------------
-Default: ``None``
+Default: ``0``
 
 If assigned an integer number higher than ``0``, Scrapy generates multiple output files
 storing up to the specified number of items in each output file.
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 1331782e3..e06116acd 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -243,7 +243,7 @@ class FeedExporter:
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
-        self.storage_batch_item_count = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None)
+        self.storage_batch_item_count = self.settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 5a7dc533e..810acd5a3 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -146,7 +146,7 @@ FEED_STORAGES_BASE = {
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
 }
-FEED_STORAGE_BATCH_ITEM_COUNT = None
+FEED_STORAGE_BATCH_ITEM_COUNT = 0
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
     'json': 'scrapy.exporters.JsonItemExporter',
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 1a6a5624b..578cd396b 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1144,7 +1144,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['jl']:
@@ -1160,7 +1160,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         data = yield self.exported_data(items, settings)
         for batch in data['csv']:
             got_batch = csv.DictReader(to_unicode(batch).splitlines())
@@ -1176,7 +1176,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['xml']:
@@ -1194,7 +1194,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         # XML
@@ -1219,7 +1219,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import pickle
@@ -1236,7 +1236,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import marshal
@@ -1262,7 +1262,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
             'FEED_STORAGE_BATCH_ITEM_COUNT': 2
         }
         header = self.MyItem.fields.keys()
-        yield self.assertExported(items, header, rows, settings=settings)
+        yield self.assertExported(items, header, rows, settings=Settings(settings))
 
     def test_wrong_path(self):
         """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """
@@ -1412,14 +1412,14 @@ class BatchDeliveriesTest(FeedExportTestBase):
             bucket_name=s3_test_bucket_name, prefix=prefix
         )
         storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key)
-        settings = {
+        settings = Settings({
             'FEEDS': {
                 s3_test_file_uri: {
                     'format': 'json',
                 },
             },
             'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
-        }
+        })
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
             self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
@@ -1436,7 +1436,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
         s3 = boto3.resource('s3')
         my_bucket = s3.Bucket(s3_test_bucket_name)
-        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
 
         with MockServer() as s:
             runner = CrawlerRunner(Settings(settings))

From 88a52198b90faa0129c8e05072197cdffbb9653b Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Sat, 27 Jun 2020 11:50:26 +0300
Subject: [PATCH 187/243] Add batch_item_count support in FEEDS setting

---
 scrapy/extensions/feedexport.py |  5 +++--
 tests/test_feedexport.py        | 39 ++++++++++++++++++++++++++++++---
 2 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index e06116acd..2312c994e 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -25,7 +25,6 @@ from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import without_none_values
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -337,7 +336,9 @@ class FeedExporter:
             slot.exporter.export_item(item)
             slot.itemcount += 1
             # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
-            if self.storage_batch_item_count and slot.itemcount == self.storage_batch_item_count:
+            if self.feeds[slot.uri_template].get('batch_item_count', self.storage_batch_item_count) \
+                    and slot.itemcount == self.feeds[slot.uri_template].get('batch_item_count',
+                                                                            self.storage_batch_item_count):
                 uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
                 self._close_slot(slot, spider)
                 slots.append(self._start_new_batch(
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 578cd396b..3bc0c083c 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1327,8 +1327,9 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     '<items>\n  <item>\n    <foo>FOO1</foo>\n  </item>\n</items>'
                 ).encode('latin-1')
             ],
-            'csv': ['bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
-                    'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')],
+            'csv': ['foo,bar\r\nFOO,BAR\r\n'.encode('utf-8'),
+                    'foo,bar\r\nFOO1,BAR1\r\n'.encode('utf-8')],
+            'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')],
         }
 
         settings = {
@@ -1348,9 +1349,16 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {
                     'format': 'csv',
                     'indent': None,
-                    'fields': ['bar', 'foo'],
+                    'fields': ['foo', 'bar'],
                     'encoding': 'utf-8',
                 },
+                os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {
+                    'format': 'jsonlines',
+                    'indent': None,
+                    'fields': ['foo', 'bar'],
+                    'encoding': 'utf-8',
+                    'batch_item_count': 0,
+                },
             },
             'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
         }
@@ -1359,6 +1367,31 @@ class BatchDeliveriesTest(FeedExportTestBase):
             for expected_batch, got_batch in zip(expected, data[fmt]):
                 self.assertEqual(expected_batch, got_batch)
 
+    @defer.inlineCallbacks
+    def test_batch_item_count_feeds_setting(self):
+        items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]
+
+        formats = {
+            'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n'.encode('utf-8'),
+                          '{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')],
+        }
+
+        settings = {
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): {
+                    'format': 'jsonlines',
+                    'indent': None,
+                    'fields': ['foo', 'bar'],
+                    'encoding': 'utf-8',
+                    'batch_item_count': 1,
+                },
+            },
+        }
+        data = yield self.exported_data(items, settings)
+        for fmt, expected in formats.items():
+            for expected_batch, got_batch in zip(expected, data[fmt]):
+                self.assertEqual(expected_batch, got_batch)
+
     @defer.inlineCallbacks
     def test_batch_path_differ(self):
         """

From 05c2587c6a32b84a94463f2b1187e49f94957aa2 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Sun, 28 Jun 2020 09:45:45 +0300
Subject: [PATCH 188/243] Docs update and tiny fixes

---
 docs/topics/feed-exports.rst | 1 +
 tests/test_feedexport.py     | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 0b37e9a7d..3da56821e 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -272,6 +272,7 @@ as a fallback value if that key is not provided for a specific feed definition.
 * ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
 * ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
 * ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
+* ``batch_item_count``: falls back to :setting:`FEED_STORAGE_BATCH_ITEM_COUNT`
 
 .. setting:: FEED_EXPORT_ENCODING
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 3bc0c083c..542cce70f 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1352,7 +1352,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     'fields': ['foo', 'bar'],
                     'encoding': 'utf-8',
                 },
-                os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {
+                os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): {
                     'format': 'jsonlines',
                     'indent': None,
                     'fields': ['foo', 'bar'],
@@ -1423,8 +1423,8 @@ class BatchDeliveriesTest(FeedExportTestBase):
         [testenv]
         setenv =
             AWS_SECRET_ACCESS_KEY = ABCD
-            AWS_ACCESS_KEY_ID = ABCD
-            S3_TEST_BUCKET_NAME = ABCD
+            AWS_ACCESS_KEY_ID = EFGH
+            S3_TEST_BUCKET_NAME = IJKL
         """
         try:
             import boto3

From 464f24f8c160466f93eb8ebb4ec8b84d1824eaa0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com>
Date: Mon, 29 Jun 2020 14:20:29 +0200
Subject: [PATCH 189/243] Add --data-raw to utils.curl and fix missing method
 with data (#4612)

---
 scrapy/utils/curl.py     | 12 +++++++++---
 tests/test_utils_curl.py | 25 +++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index 16639356e..67b22dbc5 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -17,8 +17,8 @@ class CurlParser(argparse.ArgumentParser):
 curl_parser = CurlParser()
 curl_parser.add_argument('url')
 curl_parser.add_argument('-H', '--header', dest='headers', action='append')
-curl_parser.add_argument('-X', '--request', dest='method', default='get')
-curl_parser.add_argument('-d', '--data', dest='data')
+curl_parser.add_argument('-X', '--request', dest='method')
+curl_parser.add_argument('-d', '--data', '--data-raw', dest='data')
 curl_parser.add_argument('-u', '--user', dest='auth')
 
 
@@ -66,7 +66,9 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
     if not parsed_url.scheme:
         url = 'http://' + url
 
-    result = {'method': parsed_args.method.upper(), 'url': url}
+    method = parsed_args.method or 'GET'
+
+    result = {'method': method.upper(), 'url': url}
 
     headers = []
     cookies = {}
@@ -90,5 +92,9 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
         result['cookies'] = cookies
     if parsed_args.data:
         result['body'] = parsed_args.data
+        if not parsed_args.method:
+            # if the "data" is specified but the "method" is not specified,
+            # the default method is 'POST'
+            result['method'] = 'POST'
 
     return result
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index 50e1bfd5f..299a51efe 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -141,6 +141,31 @@ class CurlToRequestKwargsTest(unittest.TestCase):
         }
         self._test_command(curl_command, expected_result)
 
+    def test_post_data_raw(self):
+        curl_command = (
+            "curl 'https://www.example.org/' --data-raw 'excerptLength=200&ena"
+            "bleDidYouMean=true&sortCriteria=ffirstz32xnamez32x201740686%20asc"
+            "ending&queryFunctions=%5B%5D&rankingFunctions=%5B%5D'"
+        )
+        expected_result = {
+            "method": "POST",
+            "url": "https://www.example.org/",
+            "body": (
+                "excerptLength=200&enableDidYouMean=true&sortCriteria=ffirstz3"
+                "2xnamez32x201740686%20ascending&queryFunctions=%5B%5D&ranking"
+                "Functions=%5B%5D")
+        }
+        self._test_command(curl_command, expected_result)
+
+    def test_explicit_get_with_data(self):
+        curl_command = 'curl httpbin.org/anything -X GET --data asdf'
+        expected_result = {
+            "method": "GET",
+            "url": "http://httpbin.org/anything",
+            "body": "asdf"
+        }
+        self._test_command(curl_command, expected_result)
+
     def test_patch(self):
         curl_command = (
             'curl "https://example.com/api/fake" -u "username:password" -H "Ac'

From e46b47c365b27e3f3f383a102122361db0a86a42 Mon Sep 17 00:00:00 2001
From: Aditya Kumar <k.aditya00@gmail.com>
Date: Mon, 29 Jun 2020 18:35:13 +0530
Subject: [PATCH 190/243] Renew the localhost certificate for tests (#4650)

Validity
  Not Before: Jun 28 12:54:15 2020 GMT
  Not After : Jun 28 12:54:15 2021 GMT
Subject: C = IE, O = Scrapy, CN = localhost
---
 tests/keys/localhost.crt | 36 ++++++++++++++--------------
 tests/keys/localhost.key | 52 ++++++++++++++++++++--------------------
 2 files changed, 44 insertions(+), 44 deletions(-)

diff --git a/tests/keys/localhost.crt b/tests/keys/localhost.crt
index 13c5b5bd6..0cf5256d8 100644
--- a/tests/keys/localhost.crt
+++ b/tests/keys/localhost.crt
@@ -1,20 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIIDNzCCAh+gAwIBAgIJANWqWyPdTY8CMA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV
-BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0x
-NzA0MjcxNzQxNTdaFw0xODA0MjcxNzQxNTdaMDIxCzAJBgNVBAYTAklFMQ8wDQYD
-VQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB
-BQADggEPADCCAQoCggEBAK1jcwlJ+bpr63lmK1mSk83nduF+27EPTU3RyteoPM2K
-o/RqZnr/mR29U6Pu42YuhLvBUu7rQxGi+rgkwno6lMFP4y5glxRygIlPsP4WQO3Y
-njmysWfYxQoIml2A+tiLewrMZocHI2cNgrO8Fd0u7KMiLlvUCN0pVyOwZ/ym9rPY
-ObfquG/xYTFzgYD/wy1n4AXE4ve3uZPfB3ZGtB3fUmuowg5KZ1L3uWpviyqr1qB/
-8NXcORLegAPsquLA05gnDPOuMs7dSMeKMphvpbSerRXLGxLIfWOZ0rs8oV96Re52
-gSEg/kIIS+ts37sJofcEnx9C4FkTR8zXin9eZhgCYs0CAwEAAaNQME4wHQYDVR0O
-BBYEFOoYbg0MvcnbTN0jxISsP2ctMbjpMB8GA1UdIwQYMBaAFOoYbg0MvcnbTN0j
-xISsP2ctMbjpMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAF/JlzES
-9Z3Azaj60gvJHyPJsPSM4tUfnWoFfFrui3oPG5TJPxWqrLBsTEachUTKOd5+XR2i
-jxUuREMkcRjbc0jjsqhsxPvfgrUrbIvKjEFLfAPvvLvcQIMUJf09SEjaaMkUAYd+
-TJaxFn5kd9Q6HbkD/fEN+lKhNZI40IJvfu7u4emUj3uKy9zrw576/T8aDYUl/own
-tqqfXh/jN8wnKCQwma7gaPmMOMqBt6zCsrN9/eKnMBpdULkUtjJD4NDg03XUFLlM
-am/oQ+MnasCcctkaXKbTGx3WfBVmkGj4b3Au18CVZkRWN2QsMdBC8JLRTICKse8U
-Mjybr/hQK3mnVdE=
+MIIDRTCCAi2gAwIBAgIUGoISfeW3LwSWHC52ORXdZY9pNLswDQYJKoZIhvcNAQEL
+BQAwMjELMAkGA1UEBhMCSUUxDzANBgNVBAoMBlNjcmFweTESMBAGA1UEAwwJbG9j
+YWxob3N0MB4XDTIwMDYyODEyNTQxNVoXDTIxMDYyODEyNTQxNVowMjELMAkGA1UE
+BhMCSUUxDzANBgNVBAoMBlNjcmFweTESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjAN
+BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvCLxfTEQuIdf8JhiHrbVkGHYrNSK
+2XD2TCPaSIpJ2KKlFUrIz3A9tWlOfLnWabS5od89yOebhYj4DN/Qm2TViGg1mtWe
+pD1K2YWd1Af+hhAw5D+TpW2RH9TVhX7Ey5osWcl+0uy+RlKZE8qum72xi1vxWOmH
+wYw06iN8klQ3JfP2/eLRXBQjsh7WW0dbJ7yLvG6UFz1RbhFTtlxeIMenzNsHaMg7
+56Ru57/MMbaBwdBttXVzJDQ7imo8njuxDMszliC/QgIdBUBFzA2LB5qpr+v+laDN
+cN9t9Q9stsu446dFnRoofxJjMFW7lLu6h/lwP5r0kfeUkMDhXJ4mb6KwfwIDAQAB
+o1MwUTAdBgNVHQ4EFgQUVEdXn8ha2FA73zcy1Ia0FQMzMEYwHwYDVR0jBBgwFoAU
+VEdXn8ha2FA73zcy1Ia0FQMzMEYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B
+AQsFAAOCAQEAZpGBPsexMD+IwcMNIgc7FiaJsb8E30C9vWxgdnkpapi9zLJ4yiHQ
+VxkV9RTezUEADkaDj+2qFveamWTzJLnphgaaUpVeMcYACPhRVOYXidNrZyTmHIsX
+FwaTzAggW6CP7JxAcpxH0f9+NWFCZI36FihRdwuWyvrUl7rsXaexu0SOI/Ck0oWf
+2IW+jo67TSmcbte+J8wq77DX32mVLb/2nqpItH4T2Di+XjVBARACVOSdgdlo7lZE
+W8mSEXqP2BVx8JGG8X1znNLHcmjVj4EtkpH0wkYzpC4cvGkTsUcU7CU7ZyVUp+Bb
+dPMVxyRKWfAjRJc8o5Ot1mgHrx5coOtzAA==
 -----END CERTIFICATE-----
diff --git a/tests/keys/localhost.key b/tests/keys/localhost.key
index da975e6d3..8fc373bdd 100644
--- a/tests/keys/localhost.key
+++ b/tests/keys/localhost.key
@@ -1,28 +1,28 @@
 -----BEGIN PRIVATE KEY-----
-MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtY3MJSfm6a+t5
-ZitZkpPN53bhftuxD01N0crXqDzNiqP0amZ6/5kdvVOj7uNmLoS7wVLu60MRovq4
-JMJ6OpTBT+MuYJcUcoCJT7D+FkDt2J45srFn2MUKCJpdgPrYi3sKzGaHByNnDYKz
-vBXdLuyjIi5b1AjdKVcjsGf8pvaz2Dm36rhv8WExc4GA/8MtZ+AFxOL3t7mT3wd2
-RrQd31JrqMIOSmdS97lqb4sqq9agf/DV3DkS3oAD7KriwNOYJwzzrjLO3UjHijKY
-b6W0nq0VyxsSyH1jmdK7PKFfekXudoEhIP5CCEvrbN+7CaH3BJ8fQuBZE0fM14p/
-XmYYAmLNAgMBAAECggEAQKY4GlqO1seugRFrUHaqzbdkSCf42kgOVtnGfCqqoSj0
-gQm7NFlhSglxykokV9E4hJlMxvDJjSXrvgVWziRRmtKiroQtUN5wtsIUCGlbxFNk
-i7bpFwNoVJlolTymS1+WfSxBfk9XD/GlrkaPEG2SpjD0gCDLPUtQxmncHARVMDDu
-Eysk3njGghsTF7XMh8ljTE3CqqNSx9BkeWQr6EYfXcgaQ2jp9E+FspB5+KWeO4ss
-ELVHgtwmYSRPAEuz4XHz87RLuakqafko6ftvh3upVQwm0VXuwM+lEUYZrzoU2JQ4
-hePKHRaWQC4tawV6FyVHK4X0MuKP4uESr7YHbJ03sQKBgQDV4CyQU6xccW6hMxlD
-7hvrGcPQEPg6M4rX2uqWpB6RCh6stZEydYeh5S+A6ltml/2csw9Bl8nZM6KbArZa
-EKrZcOn7JgFyPpiDHqgEIx+9XL/mnsKMSkBKTFcvucVgjIWE8GT7jfAqMkcSysWf
-uRyUvtNpshmRLcdNhEjrr3vcwwKBgQDPid6sxBVcoyvrYUsRRVpXATJ9tsmU93LG
-HMHDlXkZ2CMfEuA0xLK+B9iyHMhh8NwYFjcG5oeVyVjE8SbifX4Sg49hde8ykXSR
-UBSNt22/JaWgreL95LEC/y9q+G4osli7NwRW1x6tB5cN1mE0hZI8Z0ETvyr3DoWO
-j/dbdFYJLwKBgDjVLCJiCbA6+EHfuTwC3upXW2BD0iJtJdz8MFA9Zl32SXZtfRri
-fls38qqYHBekFeF493nfouSTwwbb7qb6PNwxFAwH6mR4W8Cj+dO3nayNI/VdhKcQ
-6AqWRKjK/bcNQEG2O69Y5VPhLl/BAEjUQNMJ7lXs3LxmZMqld1cht5FPAoGBAJbI
-xXbiU97lUmCGZKLcr4EtBoEdz6GiksnrVMAEFmM3jHTkIu9TxcWZL9BgZxn5g/8g
-DMS/styZ2BvmVWkS4gkTepXFuI8V7Qoyk2xPS7Yn5QkzrQroH89clhfy/R4mTZ9f
-npB1ZP0z2YSdMCyXqyKlpjtxlga/jzt/z6irgmLTAoGAPrmudajtSBq534Ql2lPM
-8U6baRSAMMzV7MXcR8F1CRewQiYOzlgsB8toELNtjg1IGPqmoiNDDKmkHs3R2mO6
-J45kDPLFe9DTyZLZj0pWWK6yRLc/BA/gGzKFpMkNcyzLlQjNPqY/9mrrYea4J9Cj
-Z+pMCFLbwAbFZ9Qb/NFlUv0=
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC8IvF9MRC4h1/w
+mGIettWQYdis1IrZcPZMI9pIiknYoqUVSsjPcD21aU58udZptLmh3z3I55uFiPgM
+39CbZNWIaDWa1Z6kPUrZhZ3UB/6GEDDkP5OlbZEf1NWFfsTLmixZyX7S7L5GUpkT
+yq6bvbGLW/FY6YfBjDTqI3ySVDcl8/b94tFcFCOyHtZbR1snvIu8bpQXPVFuEVO2
+XF4gx6fM2wdoyDvnpG7nv8wxtoHB0G21dXMkNDuKajyeO7EMyzOWIL9CAh0FQEXM
+DYsHmqmv6/6VoM1w3231D2y2y7jjp0WdGih/EmMwVbuUu7qH+XA/mvSR95SQwOFc
+niZvorB/AgMBAAECggEAHVpSVRb/pdqxNEeCH4qlHWa2uJhcpXpDYzPAzcqNpPgT
+S5QkaoD3j8NDVKBl/I4O3FuJNzwzfo0VLmUJFgWQbzzbCDJGExfhArkfG8K3ilEi
+X6ovrgK/PrklKzPRHncKbmPKnrwDH9OpQHZB8diRx81rhVTCModehh1NRUNQa2I1
+QzFC7uyXx3duoIsI5QXVeEGuwHZfqIY/z+9SscdVFL6elXTPFUzBzcmAqQgdgWKN
+HXgX22LE0rAu8NnRvOZZWt4/nOjvlCFCPTB11NgthmKlVnsx4H7gpQ2OPh4bZ+0W
+birVEtZ3E1jxoGvw1FzxyqqpGkcanRMa8QWzK4JwuQKBgQDrgclpkqZrgHB/TC1p
+hLvsdflGI2SGs+c/mYR3GEjf0kJtI88WL5fj1QezdkDyOpwxFvnLslswfzdtzvis
+vksGysV35vhMPQUcmWhvzA7Pdxdv4BZr+ckER0SAYBBxg9KYZyxewGb5XzB8Cz2o
+8V+YpwrMAOYGuXHTfafv4CKlTQKBgQDMgetvV9/E3HNtKsATiPIwT3e1MzyPXigq
+12NkHSZa6s4yqm/h/fSUn54sJbhx+OtRRhktOo0aB34tcogtrJyClvCPdRAP/4Qi
+M43FjKo2cWiubWvtWlOZU04bpClG324q420rK7dCA2stID/Fa0sMQgAAyPH8TGMo
+gbvyrk4W+wKBgQDMIOnYZTF0epaH8BponJFaqwMOhTzr+OGW4dTMebMotZG4EdK8
+kzIfW5XaOsSecKjTb+vCYGzkA1CjEEPBTwuu7nDstblAM5/Lozi/tmqb7sjUwrIM
+kyxmVfONJjb6fV07lioCUtiui5B15DRkzBqlMRyNqLW43GJKA19d7rN4/QKBgCzy
+kRBTu/bEjQn9T2H7w18i2CiXLkREaYeg91NVpMxutwsjspt0+YCA5H7He5ZxIycl
+xPrP15tU8kKC3bNMMMny6sRc8j7R5fSuaAZ3OCHnIx7TJdlw9NbKHGyu0/Ojv87l
+VWUbopd7sN6mK930CvaSuvVxNN5C27hXazuXW8ppAoGBANcWsenNKpCJgF0cNPHX
+abPaWfcs5FKMNz8gEdGk3B1z/KBpYz59smPwurYVCXaWE6iv99sDOP7CVneF02sV
+SqyNzVhcVSG788uB3CwnpEvm7ydoH89L5dvYekAHP8RJulhWCK45lXkHLiYGKvhv
+PWuPk5VX+qF78JhUhPO3nfnu
 -----END PRIVATE KEY-----

From 5b88c522ac1b1a9ba1588573d90cf3bc01339282 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 30 Jun 2020 12:18:21 -0300
Subject: [PATCH 191/243] Simplify dataclass example in item loader docs

---
 docs/topics/loaders.rst | 24 ++++++------------------
 1 file changed, 6 insertions(+), 18 deletions(-)

diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index e921395d2..9c82bb4d9 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -88,29 +88,17 @@ item loaders: unless a pre-populated item is passed to the loader, fields
 will be populated incrementally using the loader's :meth:`~ItemLoader.add_xpath`,
 :meth:`~ItemLoader.add_css` and :meth:`~ItemLoader.add_value` methods.
 
-Given the way that item loaders store data internally, one approach
-to overcome this is to define items using the :func:`~dataclasses.field`
-function, with ``list`` as the ``default_factory`` argument::
+One approach to overcome this is to define items using the
+:func:`~dataclasses.field` function, with a ``default`` argument::
 
     from dataclasses import dataclass, field
+    from typing import Optional
 
     @dataclass
     class InventoryItem:
-        name: str = field(default_factory=list)
-        price: float = field(default_factory=list)
-        stock: int = field(default_factory=list)
-
-Note that in order to keep the example simple, the types do not match
-completely. A more accurate but verbose definition would be::
-
-    from dataclasses import dataclass, field
-    from typing import List, Union
-
-    @dataclass
-    class InventoryItem:
-        name: Union[str, List[str]] = field(default_factory=list)
-        price: Union[float, List[float]] = field(default_factory=list)
-        stock: Union[int, List[int]] = field(default_factory=list)
+        name: Optional[str] = field(default=None)
+        price: Optional[float] = field(default=None)
+        stock: Optional[int] = field(default=None)
 
 
 .. _topics-loaders-processors:

From 7b1d3c35ea3bfde2ac7fc69a2a26bbcb94aec1bf Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Wed, 1 Jul 2020 11:54:39 +0300
Subject: [PATCH 192/243] Minor updates

---
 docs/topics/feed-exports.rst    |  4 ++--
 scrapy/extensions/feedexport.py | 34 +++++++++++++++++----------------
 scrapy/utils/conf.py            |  4 ++++
 tests/test_feedexport.py        | 23 ++++++----------------
 tests/test_utils_conf.py        |  4 ++++
 5 files changed, 34 insertions(+), 35 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 3da56821e..0b659f30e 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -442,7 +442,7 @@ If assigned an integer number higher than ``0``, Scrapy generates multiple outpu
 storing up to the specified number of items in each output file.
 
 When generating multiple output files, you must use at least one of the following
-placeholders in :setting:`FEED_URI` to indicate how the different output file names are
+placeholders in the feed URI to indicate how the different output file names are
 generated:
 
 * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
@@ -457,7 +457,7 @@ For instance, if your settings include::
 
 And your :command:`crawl` command line is::
 
-  scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
+    scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
 
 The command line above can generate a directory tree like::
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 2312c994e..5908987a3 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -242,7 +242,6 @@ class FeedExporter:
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
-        self.storage_batch_item_count = self.settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
@@ -253,7 +252,7 @@ class FeedExporter:
 
     def open_spider(self, spider):
         for uri, feed in self.feeds.items():
-            uri_params = self._get_uri_params(spider, feed['uri_params'], None)
+            uri_params = self._get_uri_params(spider, feed['uri_params'])
             self.slots.append(self._start_new_batch(
                 batch_id=1,
                 uri=uri % uri_params,
@@ -299,7 +298,7 @@ class FeedExporter:
     def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
         """
         Redirect the output data stream to a new file.
-        Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified.
+        Execute multiple times if FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
         :param batch_id: sequence number of current batch
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
@@ -331,14 +330,15 @@ class FeedExporter:
 
     def item_scraped(self, item, spider):
         slots = []
-        for idx, slot in enumerate(self.slots):
+        for slot in self.slots:
             slot.start_exporting()
             slot.exporter.export_item(item)
             slot.itemcount += 1
             # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
-            if self.feeds[slot.uri_template].get('batch_item_count', self.storage_batch_item_count) \
-                    and slot.itemcount == self.feeds[slot.uri_template].get('batch_item_count',
-                                                                            self.storage_batch_item_count):
+            if (
+                    self.feeds[slot.uri_template]['batch_item_count']
+                    and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count']
+            ):
                 uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
                 self._close_slot(slot, spider)
                 slots.append(self._start_new_batch(
@@ -369,15 +369,17 @@ class FeedExporter:
 
     def _settings_are_valid(self, uri):
         """
-        If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s
-        to distinguish different files of partial output
+        If FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
+        %(batch_time)s or %(batch_id)s to distinguish different files of partial output
         """
-        if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
-            return True
-        logger.error(
-            '%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified'
-        )
-        return False
+        for uri_template, values in self.feeds.items():
+            if values['batch_item_count'] and not any(s in uri_template for s in ['%(batch_time)s', '%(batch_id)s']):
+                logger.error(
+                    '%(batch_time)s or %(batch_id)s must be in uri({}) if FEED_STORAGE_BATCH_ITEM_COUNT setting '
+                    'or FEEDS.batch_item_count is specified and greater than 0.'.format(uri_template)
+                )
+                return False
+        return True
 
     def _storage_supported(self, uri):
         scheme = urlparse(uri).scheme
@@ -404,7 +406,7 @@ class FeedExporter:
     def _get_storage(self, uri):
         return self._get_instance(self.storages[urlparse(uri).scheme], uri)
 
-    def _get_uri_params(self, spider, uri_params, slot):
+    def _get_uri_params(self, spider, uri_params, slot=None):
         params = {}
         for k in dir(spider):
             params[k] = getattr(spider, k)
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 5921f82bf..0e02f0f28 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -115,6 +115,10 @@ def feed_complete_default_values_from_settings(feed, settings):
     out = feed.copy()
     out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
     out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
+    out.setdefault(
+        "batch_item_count",
+        out.get('batch_item_count', settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT'))
+    )
     out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
     out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
     if settings["FEED_EXPORT_INDENT"] is None:
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 542cce70f..db14b20b9 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1265,7 +1265,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
         yield self.assertExported(items, header, rows, settings=Settings(settings))
 
     def test_wrong_path(self):
-        """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """
+        """ If path is without %(batch_time)s and %(batch_id)s an exception must be raised """
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},
@@ -1329,7 +1329,6 @@ class BatchDeliveriesTest(FeedExportTestBase):
             ],
             'csv': ['foo,bar\r\nFOO,BAR\r\n'.encode('utf-8'),
                     'foo,bar\r\nFOO1,BAR1\r\n'.encode('utf-8')],
-            'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')],
         }
 
         settings = {
@@ -1352,13 +1351,6 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     'fields': ['foo', 'bar'],
                     'encoding': 'utf-8',
                 },
-                os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): {
-                    'format': 'jsonlines',
-                    'indent': None,
-                    'fields': ['foo', 'bar'],
-                    'encoding': 'utf-8',
-                    'batch_item_count': 0,
-                },
             },
             'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
         }
@@ -1369,19 +1361,16 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_batch_item_count_feeds_setting(self):
-        items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]
-
+        items = [dict({'foo': u'FOO'}), dict({'foo': u'FOO1'})]
         formats = {
-            'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n'.encode('utf-8'),
-                          '{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')],
+            'json': ['[{"foo": "FOO"}]'.encode('utf-8'),
+                     '[{"foo": "FOO1"}]'.encode('utf-8')],
         }
-
         settings = {
             'FEEDS': {
-                os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): {
-                    'format': 'jsonlines',
+                os.path.join(self._random_temp_filename(), 'json', self._file_mark): {
+                    'format': 'json',
                     'indent': None,
-                    'fields': ['foo', 'bar'],
                     'encoding': 'utf-8',
                     'batch_item_count': 1,
                 },
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index e5d3ef582..95ec2b64a 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -149,6 +149,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "FEED_EXPORT_INDENT": 42,
             "FEED_STORE_EMPTY": True,
             "FEED_URI_PARAMS": (1, 2, 3, 4),
+            "FEED_STORAGE_BATCH_ITEM_COUNT": 2,
         })
         new_feed = feed_complete_default_values_from_settings(feed, settings)
         self.assertEqual(new_feed, {
@@ -157,6 +158,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "indent": 42,
             "store_empty": True,
             "uri_params": (1, 2, 3, 4),
+            "batch_item_count": 2,
         })
 
     def test_feed_complete_default_values_from_settings_non_empty(self):
@@ -169,6 +171,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],
             "FEED_EXPORT_INDENT": 42,
             "FEED_STORE_EMPTY": True,
+            "FEED_STORAGE_BATCH_ITEM_COUNT": 2,
         })
         new_feed = feed_complete_default_values_from_settings(feed, settings)
         self.assertEqual(new_feed, {
@@ -177,6 +180,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "indent": 42,
             "store_empty": True,
             "uri_params": None,
+            "batch_item_count": 2,
         })
 
 

From af55d23167f9cec22f815bc9f9884b10a9a35f5b Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Wed, 1 Jul 2020 19:46:54 +0500
Subject: [PATCH 193/243] Update the OpenSSL cipher list format link

OpenSSL `ciphers(1)` is now almost empty: https://www.openssl.org/docs/manmaster/man1/ciphers.html

Alternative would be linking to 1.1.1 docs specifically.
---
 docs/topics/settings.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 5178f272f..8cc8806a5 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -469,7 +469,7 @@ necessary to access certain HTTPS websites: for example, you may need to use
 ``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a
 specific cipher that is not included in ``DEFAULT`` if a website requires it.
 
-.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/ciphers.html#CIPHER-LIST-FORMAT
+.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT
 
 .. setting:: DOWNLOADER_CLIENT_TLS_METHOD
 

From 1e245046ed8ac3d9f89860501c2da95b69aaabf6 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 2 Jul 2020 12:38:08 +0300
Subject: [PATCH 194/243] Change setting name. Add leading zeroes to batch_id.
 Minor fixes.

---
 docs/topics/feed-exports.rst        | 19 +++++++++--------
 scrapy/extensions/feedexport.py     | 24 +++++++++++++---------
 scrapy/settings/default_settings.py |  2 +-
 scrapy/utils/conf.py                |  5 +----
 tests/test_feedexport.py            | 32 ++++++++++++++---------------
 tests/test_utils_conf.py            |  4 ++--
 6 files changed, 45 insertions(+), 41 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 0b659f30e..56efa80a7 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -220,7 +220,7 @@ These are the settings used for configuring the feed exports:
  * :setting:`FEED_STORAGE_FTP_ACTIVE`
  * :setting:`FEED_STORAGE_S3_ACL`
  * :setting:`FEED_EXPORTERS`
- * :setting:`FEED_STORAGE_BATCH_ITEM_COUNT`
+ * :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
 
 .. currentmodule:: scrapy.extensions.feedexport
 
@@ -272,7 +272,7 @@ as a fallback value if that key is not provided for a specific feed definition.
 * ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
 * ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
 * ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
-* ``batch_item_count``: falls back to :setting:`FEED_STORAGE_BATCH_ITEM_COUNT`
+* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
 
 .. setting:: FEED_EXPORT_ENCODING
 
@@ -432,9 +432,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 .. _botocore: https://github.com/boto/botocore
 .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
-.. setting:: FEED_STORAGE_BATCH_ITEM_COUNT
+.. setting:: FEED_EXPORT_BATCH_ITEM_COUNT
 
-FEED_STORAGE_BATCH_ITEM_COUNT
+FEED_EXPORT_BATCH_ITEM_COUNT
 -----------------------------
 Default: ``0``
 
@@ -448,16 +448,19 @@ generated:
 * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
   (e.g. ``2020-03-28T14-45-08.237134``)
 
-* ``%(batch_id)s`` - gets replaced by the batch  sequence number of batch
-  (e.g. ``2`` for the second file)
+* ``%(batch_id)0xd`` - gets replaced by the sequence number of the batch.
+By replacing ``x`` with an integer you set the number of leading zeroes to prevent
+inappropriate sorting like this: [``'1'``, ``'10'``, ``'2'``]. Here are some examples:
+    ``%(batch_id)01d`` for the second batch gets replaced by ``2``
+    ``%(batch_id)05d`` for the third batch gets replaced by ``00003``
 
 For instance, if your settings include::
 
-    FEED_STORAGE_BATCH_ITEM_COUNT=100
+    FEED_EXPORT_BATCH_ITEM_COUNT=100
 
 And your :command:`crawl` command line is::
 
-    scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
+    scrapy crawl spidername -o dirname/%(batch_id)d-filename%(batch_time)s.json
 
 The command line above can generate a directory tree like::
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 5908987a3..adb6ea2e4 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -6,6 +6,7 @@ See documentation in docs/topics/feed-exports.rst
 
 import logging
 import os
+import re
 import sys
 import warnings
 from datetime import datetime
@@ -25,6 +26,7 @@ from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import without_none_values
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -245,7 +247,7 @@ class FeedExporter:
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
-            if not self._settings_are_valid(uri):
+            if not self._settings_are_valid():
                 raise NotConfigured
             if not self._exporter_supported(feed['format']):
                 raise NotConfigured
@@ -298,7 +300,7 @@ class FeedExporter:
     def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
         """
         Redirect the output data stream to a new file.
-        Execute multiple times if FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
+        Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
         :param batch_id: sequence number of current batch
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
@@ -334,10 +336,10 @@ class FeedExporter:
             slot.start_exporting()
             slot.exporter.export_item(item)
             slot.itemcount += 1
-            # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
+            # create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one
             if (
-                    self.feeds[slot.uri_template]['batch_item_count']
-                    and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count']
+                self.feeds[slot.uri_template]['batch_item_count']
+                and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count']
             ):
                 uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
                 self._close_slot(slot, spider)
@@ -367,16 +369,18 @@ class FeedExporter:
             return True
         logger.error("Unknown feed format: %(format)s", {'format': format})
 
-    def _settings_are_valid(self, uri):
+    def _settings_are_valid(self):
         """
-        If FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
+        If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
         %(batch_time)s or %(batch_id)s to distinguish different files of partial output
         """
         for uri_template, values in self.feeds.items():
-            if values['batch_item_count'] and not any(s in uri_template for s in ['%(batch_time)s', '%(batch_id)s']):
+            if values['batch_item_count'] and not re.findall(r'(%\(batch_time\)s|(%\(batch_id\)0\d*d))', uri_template):
                 logger.error(
-                    '%(batch_time)s or %(batch_id)s must be in uri({}) if FEED_STORAGE_BATCH_ITEM_COUNT setting '
-                    'or FEEDS.batch_item_count is specified and greater than 0.'.format(uri_template)
+                    '%(batch_time)s or %(batch_id)0xd must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting '
+                    'or FEEDS.batch_item_count is specified and greater than 0. For more info see:'
+                    'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'
+                    ''.format(uri_template)
                 )
                 return False
         return True
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 810acd5a3..0016bbe1b 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -146,7 +146,7 @@ FEED_STORAGES_BASE = {
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
 }
-FEED_STORAGE_BATCH_ITEM_COUNT = 0
+FEED_EXPORT_BATCH_ITEM_COUNT = 0
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
     'json': 'scrapy.exporters.JsonItemExporter',
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 0e02f0f28..64f9c824b 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -113,12 +113,9 @@ def get_sources(use_closest=True):
 
 def feed_complete_default_values_from_settings(feed, settings):
     out = feed.copy()
+    out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT'))
     out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
     out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
-    out.setdefault(
-        "batch_item_count",
-        out.get('batch_item_count', settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT'))
-    )
     out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
     out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
     if settings["FEED_EXPORT_INDENT"] is None:
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index db14b20b9..d20b40e2f 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1108,7 +1108,7 @@ class FeedExportTest(FeedExportTestBase):
 
 class BatchDeliveriesTest(FeedExportTestBase):
     __test__ = True
-    _file_mark = '_%(batch_time)s_#%(batch_id)s_'
+    _file_mark = '_%(batch_time)s_#%(batch_id)02d_'
 
     @defer.inlineCallbacks
     def run_and_export(self, spider_cls, settings):
@@ -1144,7 +1144,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
             },
         })
-        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
+        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['jl']:
@@ -1160,7 +1160,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
             },
         })
-        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
+        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         data = yield self.exported_data(items, settings)
         for batch in data['csv']:
             got_batch = csv.DictReader(to_unicode(batch).splitlines())
@@ -1176,7 +1176,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
             },
         })
-        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
+        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['xml']:
@@ -1194,7 +1194,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
             },
         })
-        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
+        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         # XML
@@ -1219,7 +1219,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
             },
         })
-        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
+        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import pickle
@@ -1236,7 +1236,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
             },
         })
-        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
+        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import marshal
@@ -1259,18 +1259,18 @@ class BatchDeliveriesTest(FeedExportTestBase):
             {'foo': 'bar3', 'baz': 'quux3', 'egg': ''}
         ]
         settings = {
-            'FEED_STORAGE_BATCH_ITEM_COUNT': 2
+            'FEED_EXPORT_BATCH_ITEM_COUNT': 2
         }
         header = self.MyItem.fields.keys()
         yield self.assertExported(items, header, rows, settings=Settings(settings))
 
     def test_wrong_path(self):
-        """ If path is without %(batch_time)s and %(batch_id)s an exception must be raised """
+        """ If path is without %(batch_time)s and %(batch_id)0xd an exception must be raised """
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},
             },
-            'FEED_STORAGE_BATCH_ITEM_COUNT': 1
+            'FEED_EXPORT_BATCH_ITEM_COUNT': 1
         }
         crawler = get_crawler(settings_dict=settings)
         self.assertRaises(NotConfigured, FeedExporter, crawler)
@@ -1282,7 +1282,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 'FEEDS': {
                     os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
                 },
-                'FEED_STORAGE_BATCH_ITEM_COUNT': 1
+                'FEED_EXPORT_BATCH_ITEM_COUNT': 1
             }
             data = yield self.exported_no_data(settings)
             data = dict(data)
@@ -1304,7 +1304,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 },
                 'FEED_STORE_EMPTY': True,
                 'FEED_EXPORT_INDENT': None,
-                'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
+                'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
             }
             data = yield self.exported_no_data(settings)
             data = dict(data)
@@ -1352,7 +1352,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     'encoding': 'utf-8',
                 },
             },
-            'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
+            'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
         }
         data = yield self.exported_data(items, settings)
         for fmt, expected in formats.items():
@@ -1398,7 +1398,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     'format': 'json',
                 },
             },
-            'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
+            'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
         }
         data = yield self.exported_data(items, settings)
         self.assertEqual(len(items) + 1, len(data['json']))
@@ -1440,7 +1440,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     'format': 'json',
                 },
             },
-            'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
+            'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
         })
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
@@ -1458,7 +1458,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
         s3 = boto3.resource('s3')
         my_bucket = s3.Bucket(s3_test_bucket_name)
-        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
+        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
 
         with MockServer() as s:
             runner = CrawlerRunner(Settings(settings))
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index 95ec2b64a..f3ef36127 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -149,7 +149,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "FEED_EXPORT_INDENT": 42,
             "FEED_STORE_EMPTY": True,
             "FEED_URI_PARAMS": (1, 2, 3, 4),
-            "FEED_STORAGE_BATCH_ITEM_COUNT": 2,
+            "FEED_EXPORT_BATCH_ITEM_COUNT": 2,
         })
         new_feed = feed_complete_default_values_from_settings(feed, settings)
         self.assertEqual(new_feed, {
@@ -171,7 +171,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],
             "FEED_EXPORT_INDENT": 42,
             "FEED_STORE_EMPTY": True,
-            "FEED_STORAGE_BATCH_ITEM_COUNT": 2,
+            "FEED_EXPORT_BATCH_ITEM_COUNT": 2,
         })
         new_feed = feed_complete_default_values_from_settings(feed, settings)
         self.assertEqual(new_feed, {

From 3199048520ebe3798c14cfa7362612b51d156b55 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 2 Jul 2020 20:10:08 +0200
Subject: [PATCH 195/243] Complete Azure Pipelines CI setup

---
 azure-pipelines.yml                 | 18 ++++++------------
 tests/CrawlerRunner/ip_address.py   | 15 ++++++++++++++-
 tests/mockserver.py                 |  5 ++---
 tests/test_commands.py              |  5 +++++
 tests/test_crawler.py               | 13 +++++++++++++
 tests/test_feedexport.py            |  8 +++++++-
 tests/test_proxy_connect.py         |  3 +++
 tests/test_spiderloader/__init__.py | 23 +++++++++++++++--------
 tests/test_utils_asyncio.py         |  7 ++++++-
 tox.ini                             | 18 +++++++++++++++---
 10 files changed, 86 insertions(+), 29 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index ffc4d549b..710e42090 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -1,29 +1,23 @@
-# Python package
-# Create and test a Python package on multiple Python versions.
-# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
-# https://docs.microsoft.com/azure/devops/pipelines/languages/python
-
-
+variables:
+  TOXENV: py
 pool:
-  vmImage: 'windows-2019'
+  vmImage: 'windows-latest'
 strategy:
   matrix:
     Python35:
       python.version: '3.5'
-      TOXENV: py35
+      TOXENV: windows-pinned
     Python36:
       python.version: '3.6'
-      TOXENV: py36
     Python37:
       python.version: '3.7'
-      TOXENV: py37
-
+    Python38:
+      python.version: '3.8'
 steps:
 - task: UsePythonVersion@0
   inputs:
     versionSpec: '$(python.version)'
   displayName: 'Use Python $(python.version)'
-
 - script: |
     pip install -U tox twine wheel codecov
     tox
diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py
index 826374cd4..ea75bc3c9 100644
--- a/tests/CrawlerRunner/ip_address.py
+++ b/tests/CrawlerRunner/ip_address.py
@@ -1,7 +1,9 @@
 from urllib.parse import urlparse
 
 from twisted.internet import reactor
-from twisted.names.client import createResolver
+from twisted.names import cache, hosts as hostsModule, resolve
+from twisted.names.client import Resolver
+from twisted.python.runtime import platform
 
 from scrapy import Spider, Request
 from scrapy.crawler import CrawlerRunner
@@ -10,6 +12,17 @@ from scrapy.utils.log import configure_logging
 from tests.mockserver import MockServer, MockDNSServer
 
 
+# https://stackoverflow.com/a/32784190
+def createResolver(servers=None, resolvconf=None, hosts=None):
+    if hosts is None:
+        hosts = (b'/etc/hosts' if platform.getType() == 'posix'
+                 else r'c:\windows\hosts')
+    theResolver = Resolver(resolvconf, servers)
+    hostResolver = hostsModule.Resolver(hosts)
+    L = [hostResolver, cache.CacheResolver(), theResolver]
+    return resolve.ResolverChain(L)
+
+
 class LocalhostSpider(Spider):
     name = "localhost_spider"
 
diff --git a/tests/mockserver.py b/tests/mockserver.py
index df30feab6..1f40473ba 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -247,9 +247,8 @@ class MockDNSServer:
     def __enter__(self):
         self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'],
                           stdout=PIPE, env=get_testenv())
-        host, port = self.proc.stdout.readline().strip().decode('ascii').split(":")
-        self.host = host
-        self.port = int(port)
+        self.host = '127.0.0.1'
+        self.port = int(self.proc.stdout.readline().strip().decode('ascii').split(":")[1])
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 24a341759..ee0e4511a 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -2,6 +2,7 @@ import inspect
 import json
 import optparse
 import os
+import platform
 import subprocess
 import sys
 import tempfile
@@ -10,6 +11,7 @@ from os.path import exists, join, abspath
 from shutil import rmtree, copytree
 from tempfile import mkdtemp
 from threading import Timer
+from unittest import skipIf
 
 from twisted.trial import unittest
 
@@ -319,6 +321,9 @@ class BadSpider(scrapy.Spider):
         self.assertIn("start_requests", log)
         self.assertIn("badspider.py", log)
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_asyncio_enabled_true(self):
         log = self.get_log(self.debug_log_spider, args=[
             '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor'
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 78704fb2c..1a4cfe813 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -4,6 +4,7 @@ import platform
 import subprocess
 import sys
 import warnings
+from unittest import skipIf
 
 from pytest import raises, mark
 from testfixtures import LogCapture
@@ -252,6 +253,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
                 })
 
     @defer.inlineCallbacks
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_crawler_process_asyncio_enabled_true(self):
         with LogCapture(level=logging.DEBUG) as log:
             if self.reactor_pytest == 'asyncio':
@@ -293,11 +297,17 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn('Spider closed (finished)', log)
         self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_asyncio_enabled_no_reactor(self):
         log = self.run_script('asyncio_enabled_no_reactor.py')
         self.assertIn('Spider closed (finished)', log)
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_asyncio_enabled_reactor(self):
         log = self.run_script('asyncio_enabled_reactor.py')
         self.assertIn('Spider closed (finished)', log)
@@ -327,6 +337,9 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log)
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_reactor_asyncio(self):
         log = self.run_script("twisted_reactor_asyncio.py")
         self.assertIn("Spider closed (finished)", log)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index e38644214..f7b997560 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -455,9 +455,15 @@ class FeedExportTest(unittest.TestCase):
     def run_and_export(self, spider_cls, settings):
         """ Run spider with specified settings; return exported data. """
 
+        def path_to_url(path):
+            return urljoin('file:', pathname2url(str(path)))
+
+        def printf_escape(string):
+            return string.replace('%', '%%')
+
         FEEDS = settings.get('FEEDS') or {}
         settings['FEEDS'] = {
-            urljoin('file:', pathname2url(str(file_path))): feed
+            printf_escape(path_to_url(file_path)): feed
             for file_path, feed in FEEDS.items()
         }
 
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index eb4ecc91d..fc5658ae7 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -1,5 +1,6 @@
 import json
 import os
+import platform
 import re
 import sys
 from subprocess import Popen, PIPE
@@ -59,6 +60,8 @@ def _wrong_credentials(proxy_url):
 
 @skipIf(sys.version_info < (3, 5, 4),
         "requires mitmproxy < 3.0.0, which these tests do not support")
+@skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7),
+        "mitmproxy does not support Windows when running Python < 3.7")
 class ProxyConnectTestCase(TestCase):
 
     def setUp(self):
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index d922c6059..4929f1e3e 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -20,13 +20,20 @@ from scrapy.crawler import CrawlerRunner
 module_dir = os.path.dirname(os.path.abspath(__file__))
 
 
+def _copytree(source, target):
+    try:
+        shutil.copytree(source, target)
+    except shutil.Error:
+        pass
+
+
 class SpiderLoaderTest(unittest.TestCase):
 
     def setUp(self):
         orig_spiders_dir = os.path.join(module_dir, 'test_spiders')
         self.tmpdir = tempfile.mkdtemp()
         self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
-        shutil.copytree(orig_spiders_dir, self.spiders_dir)
+        _copytree(orig_spiders_dir, self.spiders_dir)
         sys.path.append(self.tmpdir)
         settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
         self.spider_loader = SpiderLoader.from_settings(settings)
@@ -124,7 +131,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
         self.tmpdir = self.mktemp()
         os.mkdir(self.tmpdir)
         self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
-        shutil.copytree(orig_spiders_dir, self.spiders_dir)
+        _copytree(orig_spiders_dir, self.spiders_dir)
         sys.path.append(self.tmpdir)
         self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
 
@@ -134,8 +141,8 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
 
     def test_dupename_warning(self):
         # copy 1 spider module so as to have duplicate spider name
-        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider3.py'),
-                        os.path.join(self.tmpdir, 'test_spiders_xxx/spider3dupe.py'))
+        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3.py'),
+                        os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3dupe.py'))
 
         with warnings.catch_warnings(record=True) as w:
             spider_loader = SpiderLoader.from_settings(self.settings)
@@ -156,10 +163,10 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
     def test_multiple_dupename_warning(self):
         # copy 2 spider modules so as to have duplicate spider name
         # This should issue 2 warning, 1 for each duplicate spider name
-        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider1.py'),
-                        os.path.join(self.tmpdir, 'test_spiders_xxx/spider1dupe.py'))
-        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider2.py'),
-                        os.path.join(self.tmpdir, 'test_spiders_xxx/spider2dupe.py'))
+        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1.py'),
+                        os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1dupe.py'))
+        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2.py'),
+                        os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2dupe.py'))
 
         with warnings.catch_warnings(record=True) as w:
             spider_loader = SpiderLoader.from_settings(self.settings)
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index 295323e4d..a2114bd18 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -1,4 +1,6 @@
-from unittest import TestCase
+import platform
+import sys
+from unittest import skipIf, TestCase
 
 from pytest import mark
 
@@ -12,6 +14,9 @@ class AsyncioTest(TestCase):
         # the result should depend only on the pytest --reactor argument
         self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio')
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_install_asyncio_reactor(self):
         # this should do nothing
         install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
diff --git a/tox.ini b/tox.ini
index 27d21ade2..f729327ca 100644
--- a/tox.ini
+++ b/tox.ini
@@ -63,14 +63,12 @@ basepython = pypy3
 commands =
     py.test {posargs:--durations=10 docs scrapy tests}
 
-[testenv:pinned]
-basepython = python3
+[pinned]
 deps =
     -ctests/constraints.txt
     cryptography==2.0
     cssselect==0.9.1
     itemadapter==0.1.0
-    lxml==3.5.0
     parsel==1.5.0
     Protego==0.1.15
     PyDispatcher==2.0.5
@@ -85,6 +83,20 @@ deps =
     botocore==1.3.23
     Pillow==3.4.2
 
+[testenv:pinned]
+basepython = python3
+deps =
+    {[pinned]deps}
+    lxml==3.5.0
+
+[testenv:windows-pinned]
+basepython = python3
+deps =
+    {[pinned]deps}
+    # First lxml version that includes a Windows wheel for Python 3.5, so we do
+    # not need to build lxml from sources in a CI Windows job:
+    lxml==3.8.0
+
 [testenv:extra-deps]
 deps =
     {[testenv]deps}

From eb937742566105f3525a9f76e4ae68cc18e9fd8d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 3 Jul 2020 01:41:47 +0200
Subject: [PATCH 196/243] TrackrefTestCase.test_get_oldest: protect from lack
 of precision

---
 tests/test_utils_trackref.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index 16e02f919..b8e8c3130 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -1,7 +1,10 @@
 import unittest
 from io import StringIO
+from time import sleep, time
 from unittest import mock
 
+from twisted.trial.unittest import SkipTest
+
 from scrapy.utils import trackref
 
 
@@ -55,7 +58,18 @@ Foo                                 1   oldest: 0s ago\n\n''')
 
     def test_get_oldest(self):
         o1 = Foo()  # NOQA
+
+        o1_time = time()
+
         o2 = Bar()  # NOQA
+
+        o3_time = time()
+        if o3_time <= o1_time:
+            sleep(0.01)
+            o3_time = time()
+        if o3_time <= o1_time:
+            raise SkipTest('time.time is not precise enough')
+
         o3 = Foo()  # NOQA
         self.assertIs(trackref.get_oldest('Foo'), o1)
         self.assertIs(trackref.get_oldest('Bar'), o2)

From 6454d456d2bcab0828aba6d81d98f7393ab7e04d Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Fri, 3 Jul 2020 08:29:54 +0300
Subject: [PATCH 197/243] Make check of placeholder less strict

---
 docs/topics/feed-exports.rst    | 11 ++++++-----
 scrapy/extensions/feedexport.py |  4 ++--
 tests/test_feedexport.py        |  2 +-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 56efa80a7..0bb5f1733 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -448,11 +448,12 @@ generated:
 * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
   (e.g. ``2020-03-28T14-45-08.237134``)
 
-* ``%(batch_id)0xd`` - gets replaced by the sequence number of the batch.
-By replacing ``x`` with an integer you set the number of leading zeroes to prevent
-inappropriate sorting like this: [``'1'``, ``'10'``, ``'2'``]. Here are some examples:
-    ``%(batch_id)01d`` for the second batch gets replaced by ``2``
-    ``%(batch_id)05d`` for the third batch gets replaced by ``00003``
+* ``%(batch_id)d`` - gets replaced by the sequence number of the batch.
+
+  Use :ref:`printf-style string formatting <python:old-string-formatting>` to
+  alter the number format. For example, to make the batch ID a 5-digit
+  number by introducing leading zeroes as needed, use ``%(batch_id)05d``
+  (e.g. ``3`` becomes ``00003``, ``123`` becomes ``00123``).
 
 For instance, if your settings include::
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index adb6ea2e4..e15c1a09c 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -375,9 +375,9 @@ class FeedExporter:
         %(batch_time)s or %(batch_id)s to distinguish different files of partial output
         """
         for uri_template, values in self.feeds.items():
-            if values['batch_item_count'] and not re.findall(r'(%\(batch_time\)s|(%\(batch_id\)0\d*d))', uri_template):
+            if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template):
                 logger.error(
-                    '%(batch_time)s or %(batch_id)0xd must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting '
+                    '%(batch_time)s or %(batch_id) must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting '
                     'or FEEDS.batch_item_count is specified and greater than 0. For more info see:'
                     'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'
                     ''.format(uri_template)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index d20b40e2f..4e0b867a4 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1265,7 +1265,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
         yield self.assertExported(items, header, rows, settings=Settings(settings))
 
     def test_wrong_path(self):
-        """ If path is without %(batch_time)s and %(batch_id)0xd an exception must be raised """
+        """ If path is without %(batch_time)s and %(batch_id) an exception must be raised """
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},

From ec06cf79a6a7264ec3e32d7de8c4e305c2afa05e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 6 Jul 2020 10:47:11 +0200
Subject: [PATCH 198/243] Update tests/CrawlerRunner/ip_address.py

Co-authored-by: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
---
 tests/CrawlerRunner/ip_address.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py
index ea75bc3c9..b8254afdf 100644
--- a/tests/CrawlerRunner/ip_address.py
+++ b/tests/CrawlerRunner/ip_address.py
@@ -15,8 +15,7 @@ from tests.mockserver import MockServer, MockDNSServer
 # https://stackoverflow.com/a/32784190
 def createResolver(servers=None, resolvconf=None, hosts=None):
     if hosts is None:
-        hosts = (b'/etc/hosts' if platform.getType() == 'posix'
-                 else r'c:\windows\hosts')
+        hosts = b'/etc/hosts' if platform.getType() == 'posix' else r'c:\windows\hosts'
     theResolver = Resolver(resolvconf, servers)
     hostResolver = hostsModule.Resolver(hosts)
     L = [hostResolver, cache.CacheResolver(), theResolver]

From 17aec5944cab33b3cdcd497d2362cacbf7773e47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 6 Jul 2020 10:47:25 +0200
Subject: [PATCH 199/243] Update tests/CrawlerRunner/ip_address.py

Co-authored-by: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
---
 tests/CrawlerRunner/ip_address.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py
index b8254afdf..3f9738798 100644
--- a/tests/CrawlerRunner/ip_address.py
+++ b/tests/CrawlerRunner/ip_address.py
@@ -18,8 +18,8 @@ def createResolver(servers=None, resolvconf=None, hosts=None):
         hosts = b'/etc/hosts' if platform.getType() == 'posix' else r'c:\windows\hosts'
     theResolver = Resolver(resolvconf, servers)
     hostResolver = hostsModule.Resolver(hosts)
-    L = [hostResolver, cache.CacheResolver(), theResolver]
-    return resolve.ResolverChain(L)
+    chain = [hostResolver, cache.CacheResolver(), theResolver]
+    return resolve.ResolverChain(chain)
 
 
 class LocalhostSpider(Spider):

From f1020e0e6af064ab31b812c25bda6b0f08827222 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Mon, 6 Jul 2020 15:40:53 +0300
Subject: [PATCH 200/243] Tiny changes

---
 scrapy/extensions/feedexport.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index e15c1a09c..21177b1b0 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -305,7 +305,7 @@ class FeedExporter:
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
         :param spider: user spider
-        :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)s to create new uri
+        :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri
         """
         storage = self._get_storage(uri)
         file = storage.open(spider)
@@ -372,13 +372,13 @@ class FeedExporter:
     def _settings_are_valid(self):
         """
         If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
-        %(batch_time)s or %(batch_id)s to distinguish different files of partial output
+        %(batch_time)s or %(batch_id)d to distinguish different files of partial output
         """
         for uri_template, values in self.feeds.items():
             if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template):
                 logger.error(
-                    '%(batch_time)s or %(batch_id) must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting '
-                    'or FEEDS.batch_item_count is specified and greater than 0. For more info see:'
+                    '%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT '
+                    'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: '
                     'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'
                     ''.format(uri_template)
                 )

From 770a8127e8e76d95243c1d586b4bb6113a38870a Mon Sep 17 00:00:00 2001
From: ajaymittur28 <ajay.cs18@bmsce.ac.in>
Date: Tue, 7 Jul 2020 15:23:29 +0530
Subject: [PATCH 201/243] Added basic `scrapy check` tests

---
 tests/test_command_check.py | 96 +++++++++++++++++++++++++++++++++++++
 1 file changed, 96 insertions(+)
 create mode 100644 tests/test_command_check.py

diff --git a/tests/test_command_check.py b/tests/test_command_check.py
new file mode 100644
index 000000000..52005a4c5
--- /dev/null
+++ b/tests/test_command_check.py
@@ -0,0 +1,96 @@
+from os.path import join, abspath
+
+from tests.test_commands import CommandTest
+
+
+class CheckCommandTest(CommandTest):
+
+    command = 'check'
+
+    def setUp(self):
+        super(CheckCommandTest, self).setUp()
+        self.spider_name = 'check_spider'
+        self.spider = abspath(join(self.proj_mod_path, 'spiders', 'checkspider.py'))
+
+    def _write_contract(self, contracts, parse_def):
+        with open(self.spider, 'w') as file:
+            file.write(f"""
+import scrapy
+
+class CheckSpider(scrapy.Spider):
+    name = '{self.spider_name}'
+    start_urls = ['http://example.com']
+
+    def parse(self, response, **cb_kwargs):
+        \"\"\"
+        @url http://www.amazon.com/s?field-keywords=selfish+gene
+        {contracts}
+        \"\"\"
+        {parse_def}
+            """)
+
+    def _test_contract(self, contracts='', parse_def='pass'):
+        self._write_contract(contracts, parse_def)
+        p, out, err = self.proc('check')
+        self.assertIn('OK', err)
+        self.assertEqual(p.returncode, 0)
+
+    def test_check_returns_requests_contract(self):
+        contracts = """
+        @returns requests 1
+        """
+        parse_def = """
+        yield scrapy.Request(url='http://next-url.com')
+        """
+        self._test_contract(contracts, parse_def)
+
+    def test_check_returns_items_contract(self):
+        contracts = """
+        @returns items 1
+        """
+        parse_def = """
+        yield {'key1': 'val1', 'key2': 'val2'}
+        """
+        self._test_contract(contracts, parse_def)
+
+    def test_check_cb_kwargs_contract(self):
+        contracts = """
+        @cb_kwargs {"arg1": "val1", "arg2": "val2"}
+        """
+        parse_def = """
+        if len(cb_kwargs.items()) == 0:
+            raise Exception("Callback args not set")
+        """
+        self._test_contract(contracts, parse_def)
+
+    def test_check_scrapes_contract(self):
+        contracts = """
+        @scrapes key1 key2
+        """
+        parse_def = """
+        yield {'key1': 'val1', 'key2': 'val2'}
+        """
+        self._test_contract(contracts, parse_def)
+
+    def test_check_all_default_contracts(self):
+        contracts = """
+        @returns items 1
+        @returns requests 1
+        @scrapes key1 key2
+        @cb_kwargs {"arg1": "val1", "arg2": "val2"}
+        """
+        parse_def = """
+        yield {'key1': 'val1', 'key2': 'val2'}
+        yield scrapy.Request(url='http://next-url.com')
+        if len(cb_kwargs.items()) == 0:
+            raise Exception("Callback args not set")
+        """
+        self._test_contract(contracts, parse_def)
+
+    def test_SCRAPY_CHECK_set(self):
+        parse_def = """
+        import os
+        if not os.environ.get('SCRAPY_CHECK'):
+            raise Exception('SCRAPY_CHECK not set')
+        """
+        self._test_contract(parse_def=parse_def)

From d014840672820b3970282f31a51b9ff24cd46bd3 Mon Sep 17 00:00:00 2001
From: ajaymittur28 <ajay.cs18@bmsce.ac.in>
Date: Tue, 7 Jul 2020 15:24:33 +0530
Subject: [PATCH 202/243] Ignore flake8 E501 for `scrapy check` tests`

---
 pytest.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pytest.ini b/pytest.ini
index bae68cd3a..97320a008 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -173,6 +173,7 @@ flake8-ignore =
     tests/pipelines.py F841 E226
     tests/spiders.py E501 E127
     tests/test_closespider.py E501 E127
+    tests/test_command_check.py E501
     tests/test_command_fetch.py E501
     tests/test_command_parse.py E501 E128 E303 E226
     tests/test_command_shell.py E501 E128

From 79b4dfc53e431bdad31925aaf16831a0dde536ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 7 Jul 2020 14:07:04 +0200
Subject: [PATCH 203/243] Fix permission handling on project generation from
 template files

---
 scrapy/commands/startproject.py               |  30 +---
 .../project/module/__init__.py                |   0
 .../project/module/items.py.tmpl              |  12 ++
 .../project/module/middlewares.py.tmpl        | 103 +++++++++++
 .../project/module/pipelines.py.tmpl          |  13 ++
 .../project/module/settings.py.tmpl           |  88 ++++++++++
 .../project/module/spiders/__init__.py        |   4 +
 .../read_only_templates/project/scrapy.cfg    |  11 ++
 .../read_only_templates/spiders/basic.tmpl    |  10 ++
 .../read_only_templates/spiders/crawl.tmpl    |  20 +++
 .../read_only_templates/spiders/csvfeed.tmpl  |  20 +++
 .../read_only_templates/spiders/xmlfeed.tmpl  |  16 ++
 tests/test_commands.py                        | 164 ++++++++++++++++++
 13 files changed, 467 insertions(+), 24 deletions(-)
 create mode 100644 tests/sample_data/read_only_templates/project/module/__init__.py
 create mode 100644 tests/sample_data/read_only_templates/project/module/items.py.tmpl
 create mode 100644 tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl
 create mode 100644 tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl
 create mode 100644 tests/sample_data/read_only_templates/project/module/settings.py.tmpl
 create mode 100644 tests/sample_data/read_only_templates/project/module/spiders/__init__.py
 create mode 100644 tests/sample_data/read_only_templates/project/scrapy.cfg
 create mode 100644 tests/sample_data/read_only_templates/spiders/basic.tmpl
 create mode 100644 tests/sample_data/read_only_templates/spiders/crawl.tmpl
 create mode 100644 tests/sample_data/read_only_templates/spiders/csvfeed.tmpl
 create mode 100644 tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 852281959..e702d7cdc 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -1,10 +1,10 @@
 import re
 import os
-import stat
 import string
 from importlib import import_module
 from os.path import join, exists, abspath
 from shutil import ignore_patterns, move, copy2, copystat
+from stat import S_IWUSR as OWNER_WRITE_PERMISSION
 
 import scrapy
 from scrapy.commands import ScrapyCommand
@@ -78,30 +78,12 @@ class Command(ScrapyCommand):
                 self._copytree(srcname, dstname)
             else:
                 copy2(srcname, dstname)
+                current_permissions = os.stat(dstname).st_mode
+                os.chmod(dstname, current_permissions | OWNER_WRITE_PERMISSION)
+
         copystat(src, dst)
-        self._set_rw_permissions(dst)
-
-    def _set_rw_permissions(self, path):
-        """
-        Sets permissions of a directory tree to +rw and +rwx for folders.
-        This is necessary if the start template files come without write
-        permissions.
-        """
-        mode_rw = (stat.S_IRUSR
-                   | stat.S_IWUSR
-                   | stat.S_IRGRP
-                   | stat.S_IROTH)
-
-        mode_x = (stat.S_IXUSR
-                  | stat.S_IXGRP
-                  | stat.S_IXOTH)
-
-        os.chmod(path, mode_rw | mode_x)
-        for root, dirs, files in os.walk(path):
-            for dir in dirs:
-                os.chmod(join(root, dir), mode_rw | mode_x)
-            for file in files:
-                os.chmod(join(root, file), mode_rw)
+        current_permissions = os.stat(dst).st_mode
+        os.chmod(dst, current_permissions | OWNER_WRITE_PERMISSION)
 
     def run(self, args, opts):
         if len(args) not in (1, 2):
diff --git a/tests/sample_data/read_only_templates/project/module/__init__.py b/tests/sample_data/read_only_templates/project/module/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/sample_data/read_only_templates/project/module/items.py.tmpl b/tests/sample_data/read_only_templates/project/module/items.py.tmpl
new file mode 100644
index 000000000..88a18331c
--- /dev/null
+++ b/tests/sample_data/read_only_templates/project/module/items.py.tmpl
@@ -0,0 +1,12 @@
+# Define here the models for your scraped items
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/items.html
+
+import scrapy
+
+
+class ${ProjectName}Item(scrapy.Item):
+    # define the fields for your item here like:
+    # name = scrapy.Field()
+    pass
diff --git a/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl b/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl
new file mode 100644
index 000000000..bd09890fe
--- /dev/null
+++ b/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl
@@ -0,0 +1,103 @@
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+
+# useful for handling different item types with a single interface
+from itemadapter import is_item, ItemAdapter
+
+
+class ${ProjectName}SpiderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, or item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Request or item objects.
+        pass
+
+    def process_start_requests(self, start_requests, spider):
+        # Called with the start requests of the spider, and works
+        # similarly to the process_spider_output() method, except
+        # that it doesn’t have a response associated.
+
+        # Must return only requests (not items).
+        for r in start_requests:
+            yield r
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)
+
+
+class ${ProjectName}DownloaderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)
diff --git a/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl b/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl
new file mode 100644
index 000000000..e845f43e9
--- /dev/null
+++ b/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl
@@ -0,0 +1,13 @@
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+
+
+# useful for handling different item types with a single interface
+from itemadapter import ItemAdapter
+
+
+class ${ProjectName}Pipeline:
+    def process_item(self, item, spider):
+        return item
diff --git a/tests/sample_data/read_only_templates/project/module/settings.py.tmpl b/tests/sample_data/read_only_templates/project/module/settings.py.tmpl
new file mode 100644
index 000000000..a414b5fde
--- /dev/null
+++ b/tests/sample_data/read_only_templates/project/module/settings.py.tmpl
@@ -0,0 +1,88 @@
+# Scrapy settings for $project_name project
+#
+# For simplicity, this file contains only settings considered important or
+# commonly used. You can find more settings consulting the documentation:
+#
+#     https://docs.scrapy.org/en/latest/topics/settings.html
+#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+BOT_NAME = '$project_name'
+
+SPIDER_MODULES = ['$project_name.spiders']
+NEWSPIDER_MODULE = '$project_name.spiders'
+
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+#USER_AGENT = '$project_name (+http://www.yourdomain.com)'
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = True
+
+# Configure maximum concurrent requests performed by Scrapy (default: 16)
+#CONCURRENT_REQUESTS = 32
+
+# Configure a delay for requests for the same website (default: 0)
+# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
+# See also autothrottle settings and docs
+#DOWNLOAD_DELAY = 3
+# The download delay setting will honor only one of:
+#CONCURRENT_REQUESTS_PER_DOMAIN = 16
+#CONCURRENT_REQUESTS_PER_IP = 16
+
+# Disable cookies (enabled by default)
+#COOKIES_ENABLED = False
+
+# Disable Telnet Console (enabled by default)
+#TELNETCONSOLE_ENABLED = False
+
+# Override the default request headers:
+#DEFAULT_REQUEST_HEADERS = {
+#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+#   'Accept-Language': 'en',
+#}
+
+# Enable or disable spider middlewares
+# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+#SPIDER_MIDDLEWARES = {
+#    '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543,
+#}
+
+# Enable or disable downloader middlewares
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#DOWNLOADER_MIDDLEWARES = {
+#    '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543,
+#}
+
+# Enable or disable extensions
+# See https://docs.scrapy.org/en/latest/topics/extensions.html
+#EXTENSIONS = {
+#    'scrapy.extensions.telnet.TelnetConsole': None,
+#}
+
+# Configure item pipelines
+# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+#ITEM_PIPELINES = {
+#    '$project_name.pipelines.${ProjectName}Pipeline': 300,
+#}
+
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
+#AUTOTHROTTLE_ENABLED = True
+# The initial download delay
+#AUTOTHROTTLE_START_DELAY = 5
+# The maximum download delay to be set in case of high latencies
+#AUTOTHROTTLE_MAX_DELAY = 60
+# The average number of requests Scrapy should be sending in parallel to
+# each remote server
+#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# Enable showing throttling stats for every response received:
+#AUTOTHROTTLE_DEBUG = False
+
+# Enable and configure HTTP caching (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+#HTTPCACHE_ENABLED = True
+#HTTPCACHE_EXPIRATION_SECS = 0
+#HTTPCACHE_DIR = 'httpcache'
+#HTTPCACHE_IGNORE_HTTP_CODES = []
+#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
diff --git a/tests/sample_data/read_only_templates/project/module/spiders/__init__.py b/tests/sample_data/read_only_templates/project/module/spiders/__init__.py
new file mode 100644
index 000000000..ebd689ac5
--- /dev/null
+++ b/tests/sample_data/read_only_templates/project/module/spiders/__init__.py
@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.
diff --git a/tests/sample_data/read_only_templates/project/scrapy.cfg b/tests/sample_data/read_only_templates/project/scrapy.cfg
new file mode 100644
index 000000000..1daeaa541
--- /dev/null
+++ b/tests/sample_data/read_only_templates/project/scrapy.cfg
@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = ${project_name}.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = ${project_name}
diff --git a/tests/sample_data/read_only_templates/spiders/basic.tmpl b/tests/sample_data/read_only_templates/spiders/basic.tmpl
new file mode 100644
index 000000000..e9112bc95
--- /dev/null
+++ b/tests/sample_data/read_only_templates/spiders/basic.tmpl
@@ -0,0 +1,10 @@
+import scrapy
+
+
+class $classname(scrapy.Spider):
+    name = '$name'
+    allowed_domains = ['$domain']
+    start_urls = ['http://$domain/']
+
+    def parse(self, response):
+        pass
diff --git a/tests/sample_data/read_only_templates/spiders/crawl.tmpl b/tests/sample_data/read_only_templates/spiders/crawl.tmpl
new file mode 100644
index 000000000..356496487
--- /dev/null
+++ b/tests/sample_data/read_only_templates/spiders/crawl.tmpl
@@ -0,0 +1,20 @@
+import scrapy
+from scrapy.linkextractors import LinkExtractor
+from scrapy.spiders import CrawlSpider, Rule
+
+
+class $classname(CrawlSpider):
+    name = '$name'
+    allowed_domains = ['$domain']
+    start_urls = ['http://$domain/']
+
+    rules = (
+        Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),
+    )
+
+    def parse_item(self, response):
+        item = {}
+        #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get()
+        #item['name'] = response.xpath('//div[@id="name"]').get()
+        #item['description'] = response.xpath('//div[@id="description"]').get()
+        return item
diff --git a/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl b/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl
new file mode 100644
index 000000000..cbcbe9e2c
--- /dev/null
+++ b/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl
@@ -0,0 +1,20 @@
+from scrapy.spiders import CSVFeedSpider
+
+
+class $classname(CSVFeedSpider):
+    name = '$name'
+    allowed_domains = ['$domain']
+    start_urls = ['http://$domain/feed.csv']
+    # headers = ['id', 'name', 'description', 'image_link']
+    # delimiter = '\t'
+
+    # Do any adaptations you need here
+    #def adapt_response(self, response):
+    #    return response
+
+    def parse_row(self, response, row):
+        i = {}
+        #i['url'] = row['url']
+        #i['name'] = row['name']
+        #i['description'] = row['description']
+        return i
diff --git a/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl b/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl
new file mode 100644
index 000000000..5aa2aa8b0
--- /dev/null
+++ b/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl
@@ -0,0 +1,16 @@
+from scrapy.spiders import XMLFeedSpider
+
+
+class $classname(XMLFeedSpider):
+    name = '$name'
+    allowed_domains = ['$domain']
+    start_urls = ['http://$domain/feed.xml']
+    iterator = 'iternodes' # you can change this; see the docs
+    itertag = 'item' # change it accordingly
+
+    def parse_node(self, response, selector):
+        item = {}
+        #item['url'] = selector.select('url').get()
+        #item['name'] = selector.select('name').get()
+        #item['description'] = selector.select('description').get()
+        return item
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 24a341759..8336c8759 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -6,7 +6,9 @@ import subprocess
 import sys
 import tempfile
 from contextlib import contextmanager
+from itertools import chain
 from os.path import exists, join, abspath
+from pathlib import Path
 from shutil import rmtree, copytree
 from tempfile import mkdtemp
 from threading import Timer
@@ -15,6 +17,7 @@ from twisted.trial import unittest
 
 import scrapy
 from scrapy.commands import ScrapyCommand
+from scrapy.commands.startproject import IGNORE
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_testenv
@@ -119,8 +122,34 @@ class StartprojectTest(ProjectTest):
         self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params'))
 
 
+def get_permissions_dict(path, renamings=None, ignore=None):
+    renamings = renamings or tuple()
+    permissions_dict = {
+        '.': os.stat(path).st_mode,
+    }
+    for root, dirs, files in os.walk(path):
+        nodes = list(chain(dirs, files))
+        if ignore:
+            ignored_names = ignore(root, nodes)
+            nodes = [node for node in nodes
+                        if node not in ignored_names]
+        for node in nodes:
+            absolute_path = os.path.join(root, node)
+            relative_path = os.path.relpath(absolute_path, path)
+            for search_string, replacement in renamings:
+                relative_path = relative_path.replace(
+                    search_string,
+                    replacement
+                )
+            permissions = os.stat(absolute_path).st_mode
+            permissions_dict[relative_path] = permissions
+    return permissions_dict
+
+
 class StartprojectTemplatesTest(ProjectTest):
 
+    maxDiff = None
+
     def setUp(self):
         super(StartprojectTemplatesTest, self).setUp()
         self.tmpl = join(self.temp_path, 'templates')
@@ -139,6 +168,141 @@ class StartprojectTemplatesTest(ProjectTest):
         self.assertIn(self.tmpl_proj, out)
         assert exists(join(self.proj_path, 'root_template'))
 
+    def test_startproject_permissions_from_writable(self):
+        """Check that generated files have the right permissions when the
+        template folder has the same permissions as in the project, i.e.
+        everything is writable."""
+        scrapy_path = scrapy.__path__[0]
+        templates_dir = os.path.join(scrapy_path, 'templates', 'project')
+        project_name = 'startproject1'
+        renamings = (
+            ('module', project_name),
+            ('.tmpl', ''),
+        )
+        expected_permissions = get_permissions_dict(
+            templates_dir,
+            renamings,
+            IGNORE,
+        )
+
+        destination = mkdtemp()
+        process = subprocess.Popen(
+            (
+                sys.executable,
+                '-m',
+                'scrapy.cmdline',
+                'startproject',
+                project_name,
+            ),
+            cwd=destination,
+            env=self.env,
+        )
+        process.wait()
+
+        project_dir = os.path.join(destination, project_name)
+        actual_permissions = get_permissions_dict(project_dir)
+
+        self.assertEqual(actual_permissions, expected_permissions)
+
+    def test_startproject_permissions_from_read_only(self):
+        """Check that generated files have the right permissions when the
+        template folder has been made read-only, which is something that some
+        systems do.
+
+        See https://github.com/scrapy/scrapy/pull/4604
+        """
+        scrapy_path = scrapy.__path__[0]
+        templates_dir = os.path.join(scrapy_path, 'templates', 'project')
+        project_name = 'startproject2'
+        renamings = (
+            ('module', project_name),
+            ('.tmpl', ''),
+        )
+        expected_permissions = get_permissions_dict(
+            templates_dir,
+            renamings,
+            IGNORE,
+        )
+
+        tests_path = os.path.dirname(__file__)
+        read_only_templates_dir = os.path.join(
+            tests_path, 'sample_data', 'read_only_templates'
+        )
+        destination = mkdtemp()
+        process = subprocess.Popen(
+            (
+                sys.executable,
+                '-m',
+                'scrapy.cmdline',
+                'startproject',
+                project_name,
+                '--set',
+                'TEMPLATES_DIR={}'.format(read_only_templates_dir),
+            ),
+            cwd=destination,
+            env=self.env,
+        )
+        process.wait()
+
+        project_dir = os.path.join(destination, project_name)
+        actual_permissions = get_permissions_dict(project_dir)
+
+        self.assertEqual(actual_permissions, expected_permissions)
+
+    def test_startproject_permissions_unchanged_in_destination(self):
+        """Check that pre-existing folders and files in the destination folder
+        do not see their permissions modified."""
+        scrapy_path = scrapy.__path__[0]
+        templates_dir = os.path.join(scrapy_path, 'templates', 'project')
+        project_name = 'startproject3'
+        renamings = (
+            ('module', project_name),
+            ('.tmpl', ''),
+        )
+        expected_permissions = get_permissions_dict(
+            templates_dir,
+            renamings,
+            IGNORE,
+        )
+
+        destination = mkdtemp()
+        project_dir = os.path.join(destination, project_name)
+
+        existing_nodes = {
+            oct(permissions)[2:] + extension: permissions
+            for extension in ('', '.d')
+            for permissions in (
+                0o444, 0o555, 0o644, 0o666, 0o755, 0o777,
+            )
+        }
+        os.mkdir(project_dir)
+        project_dir_path = Path(project_dir)
+        for node, permissions in existing_nodes.items():
+            path = project_dir_path / node
+            if node.endswith('.d'):
+                path.mkdir(mode=permissions)
+            else:
+                path.touch(mode=permissions)
+            expected_permissions[node] = path.stat().st_mode
+
+        process = subprocess.Popen(
+            (
+                sys.executable,
+                '-m',
+                'scrapy.cmdline',
+                'startproject',
+                project_name,
+                '.',
+            ),
+            cwd=project_dir,
+            env=self.env,
+        )
+        process.wait()
+
+        actual_permissions = get_permissions_dict(project_dir)
+
+        self.assertEqual(actual_permissions, expected_permissions)
+
 
 class CommandTest(ProjectTest):
 

From a3afff4a0e1b25903d1de5c5501846d1f1288d82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 7 Jul 2020 14:11:02 +0200
Subject: [PATCH 204/243] Fix style issue

---
 tests/test_commands.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index 8336c8759..bd799817d 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -131,8 +131,7 @@ def get_permissions_dict(path, renamings=None, ignore=None):
         nodes = list(chain(dirs, files))
         if ignore:
             ignored_names = ignore(root, nodes)
-            nodes = [node for node in nodes
-                        if node not in ignored_names]
+            nodes = [node for node in nodes if node not in ignored_names]
         for node in nodes:
             absolute_path = os.path.join(root, node)
             relative_path = os.path.relpath(absolute_path, path)

From e1450799ce2dfa32e248cb0b4069668ee5e6f4ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 7 Jul 2020 14:11:37 +0200
Subject: [PATCH 205/243] Remove debug test case variable

---
 tests/test_commands.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index bd799817d..c25495d16 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -147,8 +147,6 @@ def get_permissions_dict(path, renamings=None, ignore=None):
 
 class StartprojectTemplatesTest(ProjectTest):
 
-    maxDiff = None
-
     def setUp(self):
         super(StartprojectTemplatesTest, self).setUp()
         self.tmpl = join(self.temp_path, 'templates')

From ca77ca1f751a614b0c9394d1e8c6d0b9cfcdd957 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 7 Jul 2020 14:44:03 +0200
Subject: [PATCH 206/243] Generate read-only files on the fly

---
 scrapy/commands/startproject.py               |  13 ++-
 .../project/module/__init__.py                |   0
 .../project/module/items.py.tmpl              |  12 --
 .../project/module/middlewares.py.tmpl        | 103 ------------------
 .../project/module/pipelines.py.tmpl          |  13 ---
 .../project/module/settings.py.tmpl           |  88 ---------------
 .../project/module/spiders/__init__.py        |   4 -
 .../read_only_templates/project/scrapy.cfg    |  11 --
 .../read_only_templates/spiders/basic.tmpl    |  10 --
 .../read_only_templates/spiders/crawl.tmpl    |  20 ----
 .../read_only_templates/spiders/csvfeed.tmpl  |  20 ----
 .../read_only_templates/spiders/xmlfeed.tmpl  |  16 ---
 tests/test_commands.py                        |  31 ++++--
 13 files changed, 28 insertions(+), 313 deletions(-)
 delete mode 100644 tests/sample_data/read_only_templates/project/module/__init__.py
 delete mode 100644 tests/sample_data/read_only_templates/project/module/items.py.tmpl
 delete mode 100644 tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl
 delete mode 100644 tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl
 delete mode 100644 tests/sample_data/read_only_templates/project/module/settings.py.tmpl
 delete mode 100644 tests/sample_data/read_only_templates/project/module/spiders/__init__.py
 delete mode 100644 tests/sample_data/read_only_templates/project/scrapy.cfg
 delete mode 100644 tests/sample_data/read_only_templates/spiders/basic.tmpl
 delete mode 100644 tests/sample_data/read_only_templates/spiders/crawl.tmpl
 delete mode 100644 tests/sample_data/read_only_templates/spiders/csvfeed.tmpl
 delete mode 100644 tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index e702d7cdc..eccc2a3e1 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -20,7 +20,12 @@ TEMPLATES_TO_RENDER = (
     ('${project_name}', 'middlewares.py.tmpl'),
 )
 
-IGNORE = ignore_patterns('*.pyc', '.svn')
+IGNORE = ignore_patterns('*.pyc', '__pycache__', '.svn')
+
+
+def _make_writable(path):
+    current_permissions = os.stat(path).st_mode
+    os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION)
 
 
 class Command(ScrapyCommand):
@@ -78,12 +83,10 @@ class Command(ScrapyCommand):
                 self._copytree(srcname, dstname)
             else:
                 copy2(srcname, dstname)
-                current_permissions = os.stat(dstname).st_mode
-                os.chmod(dstname, current_permissions | OWNER_WRITE_PERMISSION)
+                _make_writable(dstname)
 
         copystat(src, dst)
-        current_permissions = os.stat(dst).st_mode
-        os.chmod(dst, current_permissions | OWNER_WRITE_PERMISSION)
+        _make_writable(dst)
 
     def run(self, args, opts):
         if len(args) not in (1, 2):
diff --git a/tests/sample_data/read_only_templates/project/module/__init__.py b/tests/sample_data/read_only_templates/project/module/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/tests/sample_data/read_only_templates/project/module/items.py.tmpl b/tests/sample_data/read_only_templates/project/module/items.py.tmpl
deleted file mode 100644
index 88a18331c..000000000
--- a/tests/sample_data/read_only_templates/project/module/items.py.tmpl
+++ /dev/null
@@ -1,12 +0,0 @@
-# Define here the models for your scraped items
-#
-# See documentation in:
-# https://docs.scrapy.org/en/latest/topics/items.html
-
-import scrapy
-
-
-class ${ProjectName}Item(scrapy.Item):
-    # define the fields for your item here like:
-    # name = scrapy.Field()
-    pass
diff --git a/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl b/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl
deleted file mode 100644
index bd09890fe..000000000
--- a/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl
+++ /dev/null
@@ -1,103 +0,0 @@
-# Define here the models for your spider middleware
-#
-# See documentation in:
-# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
-
-from scrapy import signals
-
-# useful for handling different item types with a single interface
-from itemadapter import is_item, ItemAdapter
-
-
-class ${ProjectName}SpiderMiddleware:
-    # Not all methods need to be defined. If a method is not defined,
-    # scrapy acts as if the spider middleware does not modify the
-    # passed objects.
-
-    @classmethod
-    def from_crawler(cls, crawler):
-        # This method is used by Scrapy to create your spiders.
-        s = cls()
-        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
-        return s
-
-    def process_spider_input(self, response, spider):
-        # Called for each response that goes through the spider
-        # middleware and into the spider.
-
-        # Should return None or raise an exception.
-        return None
-
-    def process_spider_output(self, response, result, spider):
-        # Called with the results returned from the Spider, after
-        # it has processed the response.
-
-        # Must return an iterable of Request, or item objects.
-        for i in result:
-            yield i
-
-    def process_spider_exception(self, response, exception, spider):
-        # Called when a spider or process_spider_input() method
-        # (from other spider middleware) raises an exception.
-
-        # Should return either None or an iterable of Request or item objects.
-        pass
-
-    def process_start_requests(self, start_requests, spider):
-        # Called with the start requests of the spider, and works
-        # similarly to the process_spider_output() method, except
-        # that it doesn’t have a response associated.
-
-        # Must return only requests (not items).
-        for r in start_requests:
-            yield r
-
-    def spider_opened(self, spider):
-        spider.logger.info('Spider opened: %s' % spider.name)
-
-
-class ${ProjectName}DownloaderMiddleware:
-    # Not all methods need to be defined. If a method is not defined,
-    # scrapy acts as if the downloader middleware does not modify the
-    # passed objects.
-
-    @classmethod
-    def from_crawler(cls, crawler):
-        # This method is used by Scrapy to create your spiders.
-        s = cls()
-        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
-        return s
-
-    def process_request(self, request, spider):
-        # Called for each request that goes through the downloader
-        # middleware.
-
-        # Must either:
-        # - return None: continue processing this request
-        # - or return a Response object
-        # - or return a Request object
-        # - or raise IgnoreRequest: process_exception() methods of
-        #   installed downloader middleware will be called
-        return None
-
-    def process_response(self, request, response, spider):
-        # Called with the response returned from the downloader.
-
-        # Must either;
-        # - return a Response object
-        # - return a Request object
-        # - or raise IgnoreRequest
-        return response
-
-    def process_exception(self, request, exception, spider):
-        # Called when a download handler or a process_request()
-        # (from other downloader middleware) raises an exception.
-
-        # Must either:
-        # - return None: continue processing this exception
-        # - return a Response object: stops process_exception() chain
-        # - return a Request object: stops process_exception() chain
-        pass
-
-    def spider_opened(self, spider):
-        spider.logger.info('Spider opened: %s' % spider.name)
diff --git a/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl b/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl
deleted file mode 100644
index e845f43e9..000000000
--- a/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl
+++ /dev/null
@@ -1,13 +0,0 @@
-# Define your item pipelines here
-#
-# Don't forget to add your pipeline to the ITEM_PIPELINES setting
-# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
-
-
-# useful for handling different item types with a single interface
-from itemadapter import ItemAdapter
-
-
-class ${ProjectName}Pipeline:
-    def process_item(self, item, spider):
-        return item
diff --git a/tests/sample_data/read_only_templates/project/module/settings.py.tmpl b/tests/sample_data/read_only_templates/project/module/settings.py.tmpl
deleted file mode 100644
index a414b5fde..000000000
--- a/tests/sample_data/read_only_templates/project/module/settings.py.tmpl
+++ /dev/null
@@ -1,88 +0,0 @@
-# Scrapy settings for $project_name project
-#
-# For simplicity, this file contains only settings considered important or
-# commonly used. You can find more settings consulting the documentation:
-#
-#     https://docs.scrapy.org/en/latest/topics/settings.html
-#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
-#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
-
-BOT_NAME = '$project_name'
-
-SPIDER_MODULES = ['$project_name.spiders']
-NEWSPIDER_MODULE = '$project_name.spiders'
-
-
-# Crawl responsibly by identifying yourself (and your website) on the user-agent
-#USER_AGENT = '$project_name (+http://www.yourdomain.com)'
-
-# Obey robots.txt rules
-ROBOTSTXT_OBEY = True
-
-# Configure maximum concurrent requests performed by Scrapy (default: 16)
-#CONCURRENT_REQUESTS = 32
-
-# Configure a delay for requests for the same website (default: 0)
-# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
-# See also autothrottle settings and docs
-#DOWNLOAD_DELAY = 3
-# The download delay setting will honor only one of:
-#CONCURRENT_REQUESTS_PER_DOMAIN = 16
-#CONCURRENT_REQUESTS_PER_IP = 16
-
-# Disable cookies (enabled by default)
-#COOKIES_ENABLED = False
-
-# Disable Telnet Console (enabled by default)
-#TELNETCONSOLE_ENABLED = False
-
-# Override the default request headers:
-#DEFAULT_REQUEST_HEADERS = {
-#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-#   'Accept-Language': 'en',
-#}
-
-# Enable or disable spider middlewares
-# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
-#SPIDER_MIDDLEWARES = {
-#    '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543,
-#}
-
-# Enable or disable downloader middlewares
-# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
-#DOWNLOADER_MIDDLEWARES = {
-#    '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543,
-#}
-
-# Enable or disable extensions
-# See https://docs.scrapy.org/en/latest/topics/extensions.html
-#EXTENSIONS = {
-#    'scrapy.extensions.telnet.TelnetConsole': None,
-#}
-
-# Configure item pipelines
-# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
-#ITEM_PIPELINES = {
-#    '$project_name.pipelines.${ProjectName}Pipeline': 300,
-#}
-
-# Enable and configure the AutoThrottle extension (disabled by default)
-# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
-#AUTOTHROTTLE_ENABLED = True
-# The initial download delay
-#AUTOTHROTTLE_START_DELAY = 5
-# The maximum download delay to be set in case of high latencies
-#AUTOTHROTTLE_MAX_DELAY = 60
-# The average number of requests Scrapy should be sending in parallel to
-# each remote server
-#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
-# Enable showing throttling stats for every response received:
-#AUTOTHROTTLE_DEBUG = False
-
-# Enable and configure HTTP caching (disabled by default)
-# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
-#HTTPCACHE_ENABLED = True
-#HTTPCACHE_EXPIRATION_SECS = 0
-#HTTPCACHE_DIR = 'httpcache'
-#HTTPCACHE_IGNORE_HTTP_CODES = []
-#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
diff --git a/tests/sample_data/read_only_templates/project/module/spiders/__init__.py b/tests/sample_data/read_only_templates/project/module/spiders/__init__.py
deleted file mode 100644
index ebd689ac5..000000000
--- a/tests/sample_data/read_only_templates/project/module/spiders/__init__.py
+++ /dev/null
@@ -1,4 +0,0 @@
-# This package will contain the spiders of your Scrapy project
-#
-# Please refer to the documentation for information on how to create and manage
-# your spiders.
diff --git a/tests/sample_data/read_only_templates/project/scrapy.cfg b/tests/sample_data/read_only_templates/project/scrapy.cfg
deleted file mode 100644
index 1daeaa541..000000000
--- a/tests/sample_data/read_only_templates/project/scrapy.cfg
+++ /dev/null
@@ -1,11 +0,0 @@
-# Automatically created by: scrapy startproject
-#
-# For more information about the [deploy] section see:
-# https://scrapyd.readthedocs.io/en/latest/deploy.html
-
-[settings]
-default = ${project_name}.settings
-
-[deploy]
-#url = http://localhost:6800/
-project = ${project_name}
diff --git a/tests/sample_data/read_only_templates/spiders/basic.tmpl b/tests/sample_data/read_only_templates/spiders/basic.tmpl
deleted file mode 100644
index e9112bc95..000000000
--- a/tests/sample_data/read_only_templates/spiders/basic.tmpl
+++ /dev/null
@@ -1,10 +0,0 @@
-import scrapy
-
-
-class $classname(scrapy.Spider):
-    name = '$name'
-    allowed_domains = ['$domain']
-    start_urls = ['http://$domain/']
-
-    def parse(self, response):
-        pass
diff --git a/tests/sample_data/read_only_templates/spiders/crawl.tmpl b/tests/sample_data/read_only_templates/spiders/crawl.tmpl
deleted file mode 100644
index 356496487..000000000
--- a/tests/sample_data/read_only_templates/spiders/crawl.tmpl
+++ /dev/null
@@ -1,20 +0,0 @@
-import scrapy
-from scrapy.linkextractors import LinkExtractor
-from scrapy.spiders import CrawlSpider, Rule
-
-
-class $classname(CrawlSpider):
-    name = '$name'
-    allowed_domains = ['$domain']
-    start_urls = ['http://$domain/']
-
-    rules = (
-        Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),
-    )
-
-    def parse_item(self, response):
-        item = {}
-        #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get()
-        #item['name'] = response.xpath('//div[@id="name"]').get()
-        #item['description'] = response.xpath('//div[@id="description"]').get()
-        return item
diff --git a/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl b/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl
deleted file mode 100644
index cbcbe9e2c..000000000
--- a/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl
+++ /dev/null
@@ -1,20 +0,0 @@
-from scrapy.spiders import CSVFeedSpider
-
-
-class $classname(CSVFeedSpider):
-    name = '$name'
-    allowed_domains = ['$domain']
-    start_urls = ['http://$domain/feed.csv']
-    # headers = ['id', 'name', 'description', 'image_link']
-    # delimiter = '\t'
-
-    # Do any adaptations you need here
-    #def adapt_response(self, response):
-    #    return response
-
-    def parse_row(self, response, row):
-        i = {}
-        #i['url'] = row['url']
-        #i['name'] = row['name']
-        #i['description'] = row['description']
-        return i
diff --git a/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl b/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl
deleted file mode 100644
index 5aa2aa8b0..000000000
--- a/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl
+++ /dev/null
@@ -1,16 +0,0 @@
-from scrapy.spiders import XMLFeedSpider
-
-
-class $classname(XMLFeedSpider):
-    name = '$name'
-    allowed_domains = ['$domain']
-    start_urls = ['http://$domain/feed.xml']
-    iterator = 'iternodes' # you can change this; see the docs
-    itertag = 'item' # change it accordingly
-
-    def parse_node(self, response, selector):
-        item = {}
-        #item['url'] = selector.select('url').get()
-        #item['name'] = selector.select('name').get()
-        #item['description'] = selector.select('description').get()
-        return item
diff --git a/tests/test_commands.py b/tests/test_commands.py
index c25495d16..f3fe45139 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -2,6 +2,7 @@ import inspect
 import json
 import optparse
 import os
+from stat import S_IWRITE as ANYONE_WRITE_PERMISSION
 import subprocess
 import sys
 import tempfile
@@ -17,7 +18,7 @@ from twisted.trial import unittest
 
 import scrapy
 from scrapy.commands import ScrapyCommand
-from scrapy.commands.startproject import IGNORE
+from scrapy.commands.startproject import IGNORE, _make_writable
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_testenv
@@ -170,14 +171,14 @@ class StartprojectTemplatesTest(ProjectTest):
         template folder has the same permissions as in the project, i.e.
         everything is writable."""
         scrapy_path = scrapy.__path__[0]
-        templates_dir = os.path.join(scrapy_path, 'templates', 'project')
+        project_template = os.path.join(scrapy_path, 'templates', 'project')
         project_name = 'startproject1'
         renamings = (
             ('module', project_name),
             ('.tmpl', ''),
         )
         expected_permissions = get_permissions_dict(
-            templates_dir,
+            project_template,
             renamings,
             IGNORE,
         )
@@ -209,22 +210,30 @@ class StartprojectTemplatesTest(ProjectTest):
         See https://github.com/scrapy/scrapy/pull/4604
         """
         scrapy_path = scrapy.__path__[0]
-        templates_dir = os.path.join(scrapy_path, 'templates', 'project')
+        templates_dir = os.path.join(scrapy_path, 'templates')
+        project_template = os.path.join(templates_dir, 'project')
         project_name = 'startproject2'
         renamings = (
             ('module', project_name),
             ('.tmpl', ''),
         )
         expected_permissions = get_permissions_dict(
-            templates_dir,
+            project_template,
             renamings,
             IGNORE,
         )
 
-        tests_path = os.path.dirname(__file__)
-        read_only_templates_dir = os.path.join(
-            tests_path, 'sample_data', 'read_only_templates'
-        )
+        def _make_read_only(path):
+            current_permissions = os.stat(path).st_mode
+            os.chmod(path, current_permissions & ~ANYONE_WRITE_PERMISSION)
+
+        read_only_templates_dir = str(Path(mkdtemp()) / 'templates')
+        copytree(templates_dir, read_only_templates_dir)
+
+        for root, dirs, files in os.walk(read_only_templates_dir):
+            for node in chain(dirs, files):
+                _make_read_only(os.path.join(root, node))
+
         destination = mkdtemp()
         process = subprocess.Popen(
             (
@@ -250,14 +259,14 @@ class StartprojectTemplatesTest(ProjectTest):
         """Check that pre-existing folders and files in the destination folder
         do not see their permissions modified."""
         scrapy_path = scrapy.__path__[0]
-        templates_dir = os.path.join(scrapy_path, 'templates', 'project')
+        project_template = os.path.join(scrapy_path, 'templates', 'project')
         project_name = 'startproject3'
         renamings = (
             ('module', project_name),
             ('.tmpl', ''),
         )
         expected_permissions = get_permissions_dict(
-            templates_dir,
+            project_template,
             renamings,
             IGNORE,
         )

From 7e386157033e69bccc56e3a34d5545daa0174d44 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 7 Jul 2020 15:30:19 +0200
Subject: [PATCH 207/243] Remove unused import

---
 tests/test_commands.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index f3fe45139..002237824 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -18,7 +18,7 @@ from twisted.trial import unittest
 
 import scrapy
 from scrapy.commands import ScrapyCommand
-from scrapy.commands.startproject import IGNORE, _make_writable
+from scrapy.commands.startproject import IGNORE
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_testenv

From 3e98ed24b6e9189d7fc7b4209d24971068274ddb Mon Sep 17 00:00:00 2001
From: ajaymittur28 <ajay.cs18@bmsce.ac.in>
Date: Wed, 8 Jul 2020 17:13:57 +0530
Subject: [PATCH 208/243] Convert f-string to .format()

---
 tests/test_command_check.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index 52005a4c5..72acd817c 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -14,20 +14,20 @@ class CheckCommandTest(CommandTest):
 
     def _write_contract(self, contracts, parse_def):
         with open(self.spider, 'w') as file:
-            file.write(f"""
+            file.write("""
 import scrapy
 
 class CheckSpider(scrapy.Spider):
-    name = '{self.spider_name}'
+    name = '{0}'
     start_urls = ['http://example.com']
 
     def parse(self, response, **cb_kwargs):
         \"\"\"
         @url http://www.amazon.com/s?field-keywords=selfish+gene
-        {contracts}
+        {1}
         \"\"\"
-        {parse_def}
-            """)
+        {2}
+            """.format(self.spider_name, contracts, parse_def))
 
     def _test_contract(self, contracts='', parse_def='pass'):
         self._write_contract(contracts, parse_def)

From 75bff7b6d33bdc74c1a8eb0e43e4b484473c3062 Mon Sep 17 00:00:00 2001
From: ajaymittur28 <ajay.cs18@bmsce.ac.in>
Date: Wed, 8 Jul 2020 19:48:42 +0530
Subject: [PATCH 209/243] Update url contract value

---
 tests/test_command_check.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index 72acd817c..f27f526a3 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -23,7 +23,7 @@ class CheckSpider(scrapy.Spider):
 
     def parse(self, response, **cb_kwargs):
         \"\"\"
-        @url http://www.amazon.com/s?field-keywords=selfish+gene
+        @url http://example.com
         {1}
         \"\"\"
         {2}
@@ -32,6 +32,7 @@ class CheckSpider(scrapy.Spider):
     def _test_contract(self, contracts='', parse_def='pass'):
         self._write_contract(contracts, parse_def)
         p, out, err = self.proc('check')
+        self.assertNotIn('F', out)
         self.assertIn('OK', err)
         self.assertEqual(p.returncode, 0)
 

From 9e99be982a2916559219693e6665ec7e9319f0e9 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 17 Jun 2020 15:52:57 -0300
Subject: [PATCH 210/243] Remove backslash

---
 scrapy/cmdline.py                        | 10 ++++++----
 scrapy/commands/crawl.py                 |  6 ++++--
 scrapy/commands/fetch.py                 |  6 ++++--
 scrapy/commands/genspider.py             |  7 ++++---
 scrapy/commands/startproject.py          |  7 ++++---
 scrapy/commands/view.py                  |  3 +--
 scrapy/core/engine.py                    |  8 +++++---
 scrapy/downloadermiddlewares/redirect.py | 13 +++++++------
 scrapy/downloadermiddlewares/retry.py    |  6 ++++--
 scrapy/extensions/memusage.py            | 12 ++++++++----
 scrapy/http/request/form.py              |  3 +--
 scrapy/http/response/text.py             |  5 ++++-
 scrapy/link.py                           | 14 ++++++++++----
 scrapy/settings/__init__.py              |  3 +--
 scrapy/utils/benchserver.py              |  3 +--
 scrapy/utils/conf.py                     | 12 +++++++-----
 scrapy/utils/curl.py                     |  3 +--
 scrapy/utils/ossignal.py                 |  3 +--
 scrapy/utils/response.py                 | 16 ++++++++++------
 scrapy/utils/spider.py                   | 10 ++++++----
 20 files changed, 89 insertions(+), 61 deletions(-)

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index b189e016b..3e88536e4 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -19,10 +19,12 @@ def _iter_command_classes(module_name):
     # scrapy.utils.spider.iter_spider_classes
     for module in walk_modules(module_name):
         for obj in vars(module).values():
-            if inspect.isclass(obj) and \
-                    issubclass(obj, ScrapyCommand) and \
-                    obj.__module__ == module.__name__ and \
-                    not obj == ScrapyCommand:
+            if (
+                inspect.isclass(obj)
+                and issubclass(obj, ScrapyCommand)
+                and obj.__module__ == module.__name__
+                and not obj == ScrapyCommand
+            ):
                 yield obj
 
 
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index e1724c1e6..f205c40b0 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -26,6 +26,8 @@ class Command(BaseRunSpiderCommand):
         else:
             self.crawler_process.start()
 
-            if self.crawler_process.bootstrap_failed or \
-                    (hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception):
+            if (
+                self.crawler_process.bootstrap_failed
+                or hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception
+            ):
                 self.exitcode = 1
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 063195f50..95f87e8c3 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -19,8 +19,10 @@ class Command(ScrapyCommand):
         return "Fetch a URL using the Scrapy downloader"
 
     def long_desc(self):
-        return "Fetch a URL using the Scrapy downloader and print its content " \
-            "to stdout. You may want to use --nolog to disable logging"
+        return (
+            "Fetch a URL using the Scrapy downloader and print its content"
+            " to stdout. You may want to use --nolog to disable logging"
+        )
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index abf3b7a5c..4c7548e9c 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -121,6 +121,7 @@ class Command(ScrapyCommand):
 
     @property
     def templates_dir(self):
-        _templates_base_dir = self.settings['TEMPLATES_DIR'] or \
-            join(scrapy.__path__[0], 'templates')
-        return join(_templates_base_dir, 'spiders')
+        return join(
+            self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
+            'spiders'
+        )
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 852281959..ae4a15b0f 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -137,6 +137,7 @@ class Command(ScrapyCommand):
 
     @property
     def templates_dir(self):
-        _templates_base_dir = self.settings['TEMPLATES_DIR'] or \
-            join(scrapy.__path__[0], 'templates')
-        return join(_templates_base_dir, 'project')
+        return join(
+            self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
+            'project'
+        )
diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py
index 41e77ba3b..908bee966 100644
--- a/scrapy/commands/view.py
+++ b/scrapy/commands/view.py
@@ -8,8 +8,7 @@ class Command(fetch.Command):
         return "Open URL in browser, as seen by Scrapy"
 
     def long_desc(self):
-        return "Fetch a URL using the Scrapy downloader and show its " \
-            "contents in a browser"
+        return "Fetch a URL using the Scrapy downloader and show its contents in a browser"
 
     def add_options(self, parser):
         super(Command, self).add_options(parser)
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index de0da4b70..86a6abb23 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -141,10 +141,12 @@ class ExecutionEngine:
 
     def _needs_backout(self, spider):
         slot = self.slot
-        return not self.running \
-            or slot.closing \
-            or self.downloader.needs_backout() \
+        return (
+            not self.running
+            or slot.closing
+            or self.downloader.needs_backout()
             or self.scraper.slot.needs_backout()
+        )
 
     def _next_request_from_scheduler(self, spider):
         slot = self.slot
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index b32afb8e4..366d60dcb 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -33,10 +33,8 @@ class BaseRedirectMiddleware:
         if ttl and redirects <= self.max_redirect_times:
             redirected.meta['redirect_times'] = redirects
             redirected.meta['redirect_ttl'] = ttl - 1
-            redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + \
-                [request.url]
-            redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + \
-                [reason]
+            redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + [request.url]
+            redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + [reason]
             redirected.dont_filter = request.dont_filter
             redirected.priority = request.priority + self.priority_adjust
             logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s",
@@ -99,8 +97,11 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
         self._maxdelay = settings.getint('METAREFRESH_MAXDELAY')
 
     def process_response(self, request, response, spider):
-        if request.meta.get('dont_redirect', False) or request.method == 'HEAD' or \
-                not isinstance(response, HtmlResponse):
+        if (
+            request.meta.get('dont_redirect', False)
+            or request.method == 'HEAD'
+            or not isinstance(response, HtmlResponse)
+        ):
             return response
 
         interval, url = get_meta_refresh(response,
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 6d11af5b2..67be8c282 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -60,8 +60,10 @@ class RetryMiddleware:
         return response
 
     def process_exception(self, request, exception, spider):
-        if isinstance(exception, self.EXCEPTIONS_TO_RETRY) \
-                and not request.meta.get('dont_retry', False):
+        if (
+            isinstance(exception, self.EXCEPTIONS_TO_RETRY)
+            and not request.meta.get('dont_retry', False)
+        ):
             return self._retry(request, exception, spider)
 
     def _retry(self, request, reason, spider):
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index a0540bf8f..ab2e43e8c 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -81,8 +81,10 @@ class MemoryUsage:
             logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
                          {'memusage': mem}, extra={'crawler': self.crawler})
             if self.notify_mails:
-                subj = "%s terminated: memory usage exceeded %dM at %s" % \
-                        (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
+                subj = (
+                    "%s terminated: memory usage exceeded %dM at %s"
+                    % (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
+                )
                 self._send_report(self.notify_mails, subj)
                 self.crawler.stats.set_value('memusage/limit_notified', 1)
 
@@ -102,8 +104,10 @@ class MemoryUsage:
             logger.warning("Memory usage reached %(memusage)dM",
                            {'memusage': mem}, extra={'crawler': self.crawler})
             if self.notify_mails:
-                subj = "%s warning: memory usage reached %dM at %s" % \
-                        (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
+                subj = (
+                    "%s warning: memory usage reached %dM at %s"
+                    % (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
+                )
                 self._send_report(self.notify_mails, subj)
                 self.crawler.stats.set_value('memusage/warning_notified', 1)
             self.warned = True
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index cd4e3373f..0e6ceef0b 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -205,8 +205,7 @@ def _get_clickable(clickdata, form):
 
     # We didn't find it, so now we build an XPath expression out of the other
     # arguments, because they can be used as such
-    xpath = u'.//*' + \
-            u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
+    xpath = u'.//*' + u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
     el = form.xpath(xpath)
     if len(el) == 1:
         return (el[0].get('name'), el[0].get('value') or '')
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index b43fe5c19..0f300c8da 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -62,8 +62,11 @@ class TextResponse(Response):
         return self._declared_encoding() or self._body_inferred_encoding()
 
     def _declared_encoding(self):
-        return self._encoding or self._headers_encoding() \
+        return (
+            self._encoding
+            or self._headers_encoding()
             or self._body_declared_encoding()
+        )
 
     def body_as_unicode(self):
         """Return body as unicode"""
diff --git a/scrapy/link.py b/scrapy/link.py
index 7cb0765cc..1ef50b113 100644
--- a/scrapy/link.py
+++ b/scrapy/link.py
@@ -21,12 +21,18 @@ class Link:
         self.nofollow = nofollow
 
     def __eq__(self, other):
-        return self.url == other.url and self.text == other.text and \
-            self.fragment == other.fragment and self.nofollow == other.nofollow
+        return (
+            self.url == other.url
+            and self.text == other.text
+            and self.fragment == other.fragment
+            and self.nofollow == other.nofollow
+        )
 
     def __hash__(self):
         return hash(self.url) ^ hash(self.text) ^ hash(self.fragment) ^ hash(self.nofollow)
 
     def __repr__(self):
-        return 'Link(url=%r, text=%r, fragment=%r, nofollow=%r)' % \
-            (self.url, self.text, self.fragment, self.nofollow)
+        return (
+            'Link(url=%r, text=%r, fragment=%r, nofollow=%r)'
+            % (self.url, self.text, self.fragment, self.nofollow)
+        )
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index b9a13c018..ff8317cd1 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -52,8 +52,7 @@ class SettingsAttribute:
             self.priority = priority
 
     def __str__(self):
-        return "<SettingsAttribute value={self.value!r} " \
-               "priority={self.priority}>".format(self=self)
+        return "<SettingsAttribute value={self.value!r} priority={self.priority}>".format(self=self)
 
     __repr__ = __str__
 
diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
index 9d8d64612..f595a1acb 100644
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@@ -28,8 +28,7 @@ class Root(Resource):
 
 
 def _getarg(request, name, default=None, type=str):
-    return type(request.args[name][0]) \
-        if name in request.args else default
+    return type(request.args[name][0]) if name in request.args else default
 
 
 if __name__ == '__main__':
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 5921f82bf..728bb5f1b 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -101,11 +101,13 @@ def get_config(use_closest=True):
 
 
 def get_sources(use_closest=True):
-    xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or \
-        os.path.expanduser('~/.config')
-    sources = ['/etc/scrapy.cfg', r'c:\scrapy\scrapy.cfg',
-               xdg_config_home + '/scrapy.cfg',
-               os.path.expanduser('~/.scrapy.cfg')]
+    xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or os.path.expanduser('~/.config')
+    sources = [
+        '/etc/scrapy.cfg',
+        r'c:\scrapy\scrapy.cfg',
+        xdg_config_home + '/scrapy.cfg',
+        os.path.expanduser('~/.scrapy.cfg'),
+    ]
     if use_closest:
         sources.append(closest_scrapy_cfg())
     return sources
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index 67b22dbc5..aa681522f 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -9,8 +9,7 @@ from w3lib.http import basic_auth_header
 
 class CurlParser(argparse.ArgumentParser):
     def error(self, message):
-        error_msg = \
-            'There was an error parsing the curl command: {}'.format(message)
+        error_msg = 'There was an error parsing the curl command: {}'.format(message)
         raise ValueError(error_msg)
 
 
diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py
index 45c9cef0c..cf867f3f8 100644
--- a/scrapy/utils/ossignal.py
+++ b/scrapy/utils/ossignal.py
@@ -18,8 +18,7 @@ def install_shutdown_handlers(function, override_sigint=True):
     from twisted.internet import reactor
     reactor._handleSignals()
     signal.signal(signal.SIGTERM, function)
-    if signal.getsignal(signal.SIGINT) == signal.default_int_handler or \
-            override_sigint:
+    if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint:
         signal.signal(signal.SIGINT, function)
     # Catch Ctrl-Break in windows
     if hasattr(signal, 'SIGBREAK'):
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index edbc0db25..c29b619ce 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -47,13 +47,17 @@ def response_httprepr(response):
     is provided only for reference, since it's not the exact stream of bytes
     that was received (that's not exposed by Twisted).
     """
-    s = b"HTTP/1.1 " + to_bytes(str(response.status)) + b" " + \
-        to_bytes(http.RESPONSES.get(response.status, b'')) + b"\r\n"
+    values = [
+        b"HTTP/1.1 ",
+        to_bytes(str(response.status)),
+        b" ",
+        to_bytes(http.RESPONSES.get(response.status, b'')),
+        b"\r\n",
+    ]
     if response.headers:
-        s += response.headers.to_string() + b"\r\n"
-    s += b"\r\n"
-    s += response.body
-    return s
+        values.extend([response.headers.to_string(), b"\r\n"])
+    values.extend([b"\r\n", response.body])
+    return b"".join(values)
 
 
 def open_in_browser(response, _openfunc=webbrowser.open):
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 7e7a50c88..f3a9a67a3 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -34,10 +34,12 @@ def iter_spider_classes(module):
     from scrapy.spiders import Spider
 
     for obj in vars(module).values():
-        if inspect.isclass(obj) and \
-           issubclass(obj, Spider) and \
-           obj.__module__ == module.__name__ and \
-           getattr(obj, 'name', None):
+        if (
+            inspect.isclass(obj)
+            and issubclass(obj, Spider)
+            and obj.__module__ == module.__name__
+            and getattr(obj, 'name', None)
+        ):
             yield obj
 
 

From 9aea1f096171d38348b1403302c6c40eeef7f0a6 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 9 Jul 2020 11:04:46 -0300
Subject: [PATCH 211/243] Remove backslash (tests)

---
 ...st_downloadermiddleware_httpcompression.py |  3 +-
 tests/test_downloadermiddleware_redirect.py   | 18 ++++-------
 tests/test_selector.py                        |  3 +-
 tests/test_settings/__init__.py               |  3 +-
 tests/test_spider.py                          | 10 ++++--
 tests/test_spidermiddleware_referer.py        | 32 +++++++++++++------
 tests/test_utils_curl.py                      |  6 ++--
 tests/test_utils_defer.py                     |  9 ++++--
 tests/test_utils_iterators.py                 | 30 +++++++++--------
 tests/test_utils_request.py                   |  8 +++--
 tests/test_utils_response.py                  |  3 +-
 tests/test_utils_sitemap.py                   |  3 +-
 tests/test_utils_url.py                       |  3 +-
 tests/test_webclient.py                       |  5 ++-
 14 files changed, 75 insertions(+), 61 deletions(-)

diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 87304d76c..a806f55ce 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -5,8 +5,7 @@ from gzip import GzipFile
 
 from scrapy.spiders import Spider
 from scrapy.http import Response, Request, HtmlResponse
-from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \
-    ACCEPTED_ENCODINGS
+from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.gz import gunzip
 from tests import tests_datadir
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index c46b1bb87..919dbed23 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -77,12 +77,9 @@ class RedirectMiddlewareTest(unittest.TestCase):
         assert isinstance(req2, Request)
         self.assertEqual(req2.url, url2)
         self.assertEqual(req2.method, 'GET')
-        assert 'Content-Type' not in req2.headers, \
-            "Content-Type header must not be present in redirected request"
-        assert 'Content-Length' not in req2.headers, \
-            "Content-Length header must not be present in redirected request"
-        assert not req2.body, \
-            "Redirected body must be empty, not '%s'" % req2.body
+        assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request"
+        assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request"
+        assert not req2.body, "Redirected body must be empty, not '%s'" % req2.body
 
         # response without Location header but with status code is 3XX should be ignored
         del rsp.headers['Location']
@@ -244,12 +241,9 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
         assert isinstance(req2, Request)
         self.assertEqual(req2.url, 'http://example.org/newpage')
         self.assertEqual(req2.method, 'GET')
-        assert 'Content-Type' not in req2.headers, \
-            "Content-Type header must not be present in redirected request"
-        assert 'Content-Length' not in req2.headers, \
-            "Content-Length header must not be present in redirected request"
-        assert not req2.body, \
-            "Redirected body must be empty, not '%s'" % req2.body
+        assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request"
+        assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request"
+        assert not req2.body, "Redirected body must be empty, not '%s'" % req2.body
 
     def test_max_redirect_times(self):
         self.mw.max_redirect_times = 1
diff --git a/tests/test_selector.py b/tests/test_selector.py
index bcf653444..00e663c11 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -88,8 +88,7 @@ class SelectorTestCase(unittest.TestCase):
         """Check that classes are using slots and are weak-referenceable"""
         x = Selector(text='')
         weakref.ref(x)
-        assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
-            x.__class__.__name__
+        assert not hasattr(x, '__dict__'), "%s does not use __slots__" % x.__class__.__name__
 
     def test_selector_bad_args(self):
         with self.assertRaisesRegex(ValueError, 'received both response and text'):
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 2da6aa4b5..6e56a28f5 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -86,8 +86,7 @@ class BaseSettingsTest(unittest.TestCase):
 
     def test_set_calls_settings_attributes_methods_on_update(self):
         attr = SettingsAttribute('value', 10)
-        with mock.patch.object(attr, '__setattr__') as mock_setattr, \
-                mock.patch.object(attr, 'set') as mock_set:
+        with mock.patch.object(attr, '__setattr__') as mock_setattr, mock.patch.object(attr, 'set') as mock_set:
 
             self.settings.attributes = {'TEST_OPTION': attr}
 
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 805d70459..bd9238810 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -11,8 +11,14 @@ from scrapy import signals
 from scrapy.settings import Settings
 from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse
 from scrapy.spiders.init import InitSpider
-from scrapy.spiders import Spider, CrawlSpider, Rule, XMLFeedSpider, \
-    CSVFeedSpider, SitemapSpider
+from scrapy.spiders import (
+    CSVFeedSpider,
+    CrawlSpider,
+    Rule,
+    SitemapSpider,
+    Spider,
+    XMLFeedSpider,
+)
 from scrapy.linkextractors import LinkExtractor
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.test import get_crawler
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 067118cf0..5141f47af 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -6,16 +6,28 @@ from scrapy.http import Response, Request
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
 from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
-from scrapy.spidermiddlewares.referer import RefererMiddleware, \
-    POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \
-    POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \
-    POLICY_SCRAPY_DEFAULT, POLICY_UNSAFE_URL, \
-    POLICY_STRICT_ORIGIN, POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, \
-    DefaultReferrerPolicy, \
-    NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \
-    OriginWhenCrossOriginPolicy, OriginPolicy, \
-    StrictOriginWhenCrossOriginPolicy, StrictOriginPolicy, \
-    SameOriginPolicy, UnsafeUrlPolicy, ReferrerPolicy
+from scrapy.spidermiddlewares.referer import (
+    DefaultReferrerPolicy,
+    NoReferrerPolicy,
+    NoReferrerWhenDowngradePolicy,
+    OriginPolicy,
+    OriginWhenCrossOriginPolicy,
+    POLICY_NO_REFERRER,
+    POLICY_NO_REFERRER_WHEN_DOWNGRADE,
+    POLICY_ORIGIN,
+    POLICY_ORIGIN_WHEN_CROSS_ORIGIN,
+    POLICY_SAME_ORIGIN,
+    POLICY_SCRAPY_DEFAULT,
+    POLICY_STRICT_ORIGIN,
+    POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN,
+    POLICY_UNSAFE_URL,
+    RefererMiddleware,
+    ReferrerPolicy,
+    SameOriginPolicy,
+    StrictOriginPolicy,
+    StrictOriginWhenCrossOriginPolicy,
+    UnsafeUrlPolicy,
+)
 
 
 class TestRefererMiddleware(TestCase):
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index 299a51efe..6b05c8771 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -29,8 +29,7 @@ class CurlToRequestKwargsTest(unittest.TestCase):
         self._test_command(curl_command, expected_result)
 
     def test_get_basic_auth(self):
-        curl_command = 'curl "https://api.test.com/" -u ' \
-                       '"some_username:some_password"'
+        curl_command = 'curl "https://api.test.com/" -u "some_username:some_password"'
         expected_result = {
             "method": "GET",
             "url": "https://api.test.com/",
@@ -212,8 +211,7 @@ class CurlToRequestKwargsTest(unittest.TestCase):
         with warnings.catch_warnings():  # avoid warning when executing tests
             warnings.simplefilter('ignore')
             curl_command = 'curl --bar --baz http://www.example.com'
-            expected_result = \
-                {"method": "GET", "url": "http://www.example.com"}
+            expected_result = {"method": "GET", "url": "http://www.example.com"}
             self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
 
         # case 2: ignore_unknown_options=False (raise exception):
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 2d4b88121..8c84331b9 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -2,8 +2,13 @@ from twisted.trial import unittest
 from twisted.internet import reactor, defer
 from twisted.python.failure import Failure
 
-from scrapy.utils.defer import mustbe_deferred, process_chain, \
-    process_chain_both, process_parallel, iter_errback
+from scrapy.utils.defer import (
+    iter_errback,
+    mustbe_deferred,
+    process_chain,
+    process_chain_both,
+    process_parallel,
+)
 
 
 class MustbeDeferredTest(unittest.TestCase):
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 8344c6701..3ebe3ac24 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -15,18 +15,20 @@ class XmliterTestCase(unittest.TestCase):
     xmliter = staticmethod(xmliter)
 
     def test_xmliter(self):
-        body = b"""<?xml version="1.0" encoding="UTF-8"?>\
+        body = b"""
+            <?xml version="1.0" encoding="UTF-8"?>
             <products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-                      xsi:noNamespaceSchemaLocation="someschmea.xsd">\
-              <product id="001">\
-                <type>Type 1</type>\
-                <name>Name 1</name>\
-              </product>\
-              <product id="002">\
-                <type>Type 2</type>\
-                <name>Name 2</name>\
-              </product>\
-            </products>"""
+                      xsi:noNamespaceSchemaLocation="someschmea.xsd">
+              <product id="001">
+                <type>Type 1</type>
+                <name>Name 1</name>
+              </product>
+              <product id="002">
+                <type>Type 2</type>
+                <name>Name 2</name>
+              </product>
+            </products>
+        """
 
         response = XmlResponse(url="http://example.com", body=body)
         attrs = []
@@ -115,7 +117,7 @@ class XmliterTestCase(unittest.TestCase):
                          [[u'one'], [u'two']])
 
     def test_xmliter_namespaces(self):
-        body = b"""\
+        body = b"""
             <?xml version="1.0" encoding="UTF-8"?>
             <rss version="2.0" xmlns:g="http://base.google.com/ns/1.0">
                 <channel>
@@ -185,7 +187,7 @@ class LxmlXmliterTestCase(XmliterTestCase):
     xmliter = staticmethod(xmliter_lxml)
 
     def test_xmliter_iterate_namespace(self):
-        body = b"""\
+        body = b"""
             <?xml version="1.0" encoding="UTF-8"?>
             <rss version="2.0" xmlns="http://base.google.com/ns/1.0">
                 <channel>
@@ -214,7 +216,7 @@ class LxmlXmliterTestCase(XmliterTestCase):
         self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item2.jpg'])
 
     def test_xmliter_namespaces_prefix(self):
-        body = b"""\
+        body = b"""
         <?xml version="1.0" encoding="UTF-8"?>
         <root>
             <h:table xmlns:h="http://www.w3.org/TR/html4/">
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 4cd4b7010..7e0049b1d 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -1,7 +1,11 @@
 import unittest
 from scrapy.http import Request
-from scrapy.utils.request import request_fingerprint, _fingerprint_cache, \
-    request_authenticate, request_httprepr
+from scrapy.utils.request import (
+    _fingerprint_cache,
+    request_authenticate,
+    request_fingerprint,
+    request_httprepr,
+)
 
 
 class UtilsRequestTest(unittest.TestCase):
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 6ebf290c0..d6f4c0bb5 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -37,8 +37,7 @@ class ResponseUtilsTest(unittest.TestCase):
             self.assertIn(b'<base href="' + to_bytes(url) + b'">', bbody)
             return True
         response = HtmlResponse(url, body=body)
-        assert open_in_browser(response, _openfunc=browser_open), \
-            "Browser not called"
+        assert open_in_browser(response, _openfunc=browser_open), "Browser not called"
 
         resp = Response(url, body=body)
         self.assertRaises(TypeError, open_in_browser, resp, debug=True)
diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py
index bfbf9abb3..23eb261b7 100644
--- a/tests/test_utils_sitemap.py
+++ b/tests/test_utils_sitemap.py
@@ -156,8 +156,7 @@ Disallow: /forum/active/
 
     def test_sitemap_blanklines(self):
         """Assert we can deal with starting blank lines before <xml> tag"""
-        s = Sitemap(b"""\
-
+        s = Sitemap(b"""
 <?xml version="1.0" encoding="UTF-8"?>
 <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 09a6d6c70..a194a0998 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -207,8 +207,7 @@ def create_guess_scheme_t(args):
     def do_expected(self):
         url = guess_scheme(args[0])
         assert url.startswith(args[1]), \
-            'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (
-                args[0], url, args[1])
+            'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (args[0], url, args[1])
     return do_expected
 
 
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 188e54602..c1c5945c2 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -356,9 +356,8 @@ class WebClientTestCase(unittest.TestCase):
         """ Test that non-standart body encoding matches
         Content-Encoding header """
         body = b'\xd0\x81\xd1\x8e\xd0\xaf'
-        return getPage(
-            self.getURL('encoding'), body=body, response_transform=lambda r: r)\
-            .addCallback(self._check_Encoding, body)
+        dfd = getPage(self.getURL('encoding'), body=body, response_transform=lambda r: r)
+        return dfd.addCallback(self._check_Encoding, body)
 
     def _check_Encoding(self, response, original_body):
         content_encoding = to_unicode(response.headers[b'Content-Encoding'])

From cbe4dc57f3f65ecb851941dcfae0bc18c6c8582a Mon Sep 17 00:00:00 2001
From: Ajay Mittur <ajay.cs18@bmsce.ac.in>
Date: Fri, 10 Jul 2020 18:22:43 +0530
Subject: [PATCH 212/243] Update pytest.ini

---
 pytest.ini | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pytest.ini b/pytest.ini
index 92c5bcb75..ca8191f42 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -39,4 +39,5 @@ flake8-ignore =
     scrapy/utils/markup.py F403
     scrapy/utils/multipart.py F403
     scrapy/utils/url.py F403 F405
-    tests/test_loader.py E741
\ No newline at end of file
+    tests/test_loader.py E741
+    

From a6a5fa91da8944943e2c9d8f34f09662be17b781 Mon Sep 17 00:00:00 2001
From: Artur Shellunts <shellunts.artur@gmail.com>
Date: Fri, 10 Jul 2020 23:10:49 +0200
Subject: [PATCH 213/243] Remove deprecated class HtmlParserLinkExtractor

Issue #4356
---
 scrapy/linkextractors/htmlparser.py | 91 -----------------------------
 1 file changed, 91 deletions(-)
 delete mode 100644 scrapy/linkextractors/htmlparser.py

diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
deleted file mode 100644
index 0425d4340..000000000
--- a/scrapy/linkextractors/htmlparser.py
+++ /dev/null
@@ -1,91 +0,0 @@
-"""
-HTMLParser-based link extractor
-"""
-import warnings
-from html.parser import HTMLParser
-from urllib.parse import urljoin
-
-from w3lib.url import safe_url_string
-from w3lib.html import strip_html5_whitespace
-
-from scrapy.link import Link
-from scrapy.utils.python import unique as unique_list
-from scrapy.exceptions import ScrapyDeprecationWarning
-
-
-class HtmlParserLinkExtractor(HTMLParser):
-
-    def __init__(self, tag="a", attr="href", process=None, unique=False,
-                 strip=True):
-        HTMLParser.__init__(self)
-
-        warnings.warn(
-            "HtmlParserLinkExtractor is deprecated and will be removed in "
-            "future releases. Please use scrapy.linkextractors.LinkExtractor",
-            ScrapyDeprecationWarning, stacklevel=2,
-        )
-
-        self.scan_tag = tag if callable(tag) else lambda t: t == tag
-        self.scan_attr = attr if callable(attr) else lambda a: a == attr
-        self.process_attr = process if callable(process) else lambda v: v
-        self.unique = unique
-        self.strip = strip
-
-    def _extract_links(self, response_text, response_url, response_encoding):
-        self.reset()
-        self.feed(response_text)
-        self.close()
-
-        links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links
-
-        ret = []
-        base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
-        for link in links:
-            if isinstance(link.url, str):
-                link.url = link.url.encode(response_encoding)
-            try:
-                link.url = urljoin(base_url, link.url)
-            except ValueError:
-                continue
-            link.url = safe_url_string(link.url, response_encoding)
-            link.text = link.text.decode(response_encoding)
-            ret.append(link)
-
-        return ret
-
-    def extract_links(self, response):
-        # wrapper needed to allow to work directly with text
-        return self._extract_links(response.body, response.url, response.encoding)
-
-    def reset(self):
-        HTMLParser.reset(self)
-
-        self.base_url = None
-        self.current_link = None
-        self.links = []
-
-    def handle_starttag(self, tag, attrs):
-        if tag == 'base':
-            self.base_url = dict(attrs).get('href')
-        if self.scan_tag(tag):
-            for attr, value in attrs:
-                if self.scan_attr(attr):
-                    if self.strip:
-                        value = strip_html5_whitespace(value)
-                    url = self.process_attr(value)
-                    link = Link(url=url)
-                    self.links.append(link)
-                    self.current_link = link
-
-    def handle_endtag(self, tag):
-        if self.scan_tag(tag):
-            self.current_link = None
-
-    def handle_data(self, data):
-        if self.current_link:
-            self.current_link.text = self.current_link.text + data
-
-    def matches(self, url):
-        """This extractor matches with any url, since
-        it doesn't contain any patterns"""
-        return True

From 3f7e8635f479f4de2ab1c3d518010730a5f6f0a6 Mon Sep 17 00:00:00 2001
From: Aditya Kumar <k.aditya00@gmail.com>
Date: Sat, 11 Jul 2020 12:18:24 +0530
Subject: [PATCH 214/243] Allow the parse command to write data to a file 
 (#4377)

---
 docs/topics/commands.rst    |  4 +++-
 scrapy/commands/__init__.py |  2 +-
 scrapy/commands/parse.py    | 24 ++++++++----------------
 tests/test_command_parse.py | 23 ++++++++++++++++++++++-
 4 files changed, 34 insertions(+), 19 deletions(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index a0dcba90d..4fce51abc 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -468,7 +468,7 @@ Supported options:
 * ``--callback`` or ``-c``: spider method to use as callback for parsing the
   response
 
-* ``--meta`` or ``-m``: additional request meta that will be passed to the callback 
+* ``--meta`` or ``-m``: additional request meta that will be passed to the callback
   request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
 
 * ``--cbkwargs``: additional keyword arguments that will be passed to the callback.
@@ -491,6 +491,8 @@ Supported options:
 
 * ``--verbose`` or ``-v``: display information for each depth level
 
+* ``--output`` or ``-o``: dump scraped items to a file
+
 .. skip: start
 
 Usage example::
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index ab850dcb3..57ce4e522 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -108,7 +108,7 @@ class ScrapyCommand:
 
 class BaseRunSpiderCommand(ScrapyCommand):
     """
-    Common class used to share functionality between the crawl and runspider commands
+    Common class used to share functionality between the crawl, parse and runspider commands
     """
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 8b7fa8b58..abc8ba9ff 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -4,18 +4,16 @@ import logging
 from itemadapter import is_item, ItemAdapter
 from w3lib.url import is_url
 
-from scrapy.commands import ScrapyCommand
+from scrapy.commands import BaseRunSpiderCommand
 from scrapy.http import Request
 from scrapy.utils import display
-from scrapy.utils.conf import arglist_to_dict
 from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
 from scrapy.exceptions import UsageError
 
 logger = logging.getLogger(__name__)
 
 
-class Command(ScrapyCommand):
-
+class Command(BaseRunSpiderCommand):
     requires_project = True
 
     spider = None
@@ -31,11 +29,9 @@ class Command(ScrapyCommand):
         return "Parse URL (using its spider) and print the results"
 
     def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
+        BaseRunSpiderCommand.add_options(self, parser)
         parser.add_option("--spider", dest="spider", default=None,
                           help="use this spider without looking for one")
-        parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
-                          help="set spider argument (may be repeated)")
         parser.add_option("--pipelines", action="store_true",
                           help="process items through pipelines")
         parser.add_option("--nolinks", dest="nolinks", action="store_true",
@@ -200,12 +196,15 @@ class Command(ScrapyCommand):
             self.add_items(depth, items)
             self.add_requests(depth, requests)
 
+            scraped_data = items if opts.output else []
             if depth < opts.depth:
                 for req in requests:
                     req.meta['_depth'] = depth + 1
                     req.meta['_callback'] = req.callback
                     req.callback = callback
-                return requests
+                scraped_data += requests
+
+            return scraped_data
 
         # update request meta if any extra meta was passed through the --meta/-m opts.
         if opts.meta:
@@ -221,18 +220,11 @@ class Command(ScrapyCommand):
         return request
 
     def process_options(self, args, opts):
-        ScrapyCommand.process_options(self, args, opts)
+        BaseRunSpiderCommand.process_options(self, args, opts)
 
-        self.process_spider_arguments(opts)
         self.process_request_meta(opts)
         self.process_request_cb_kwargs(opts)
 
-    def process_spider_arguments(self, opts):
-        try:
-            opts.spargs = arglist_to_dict(opts.spargs)
-        except ValueError:
-            raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
-
     def process_request_meta(self, opts):
         if opts.meta:
             try:
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index a09dcf072..5754a5478 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -1,5 +1,5 @@
 import os
-from os.path import join, abspath
+from os.path import join, abspath, isfile, exists
 from twisted.internet import defer
 from scrapy.utils.testsite import SiteTest
 from scrapy.utils.testproc import ProcessTest
@@ -218,3 +218,24 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         )
         self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
         self.assertIn("""Cannot find a rule that matches""", _textmode(stderr))
+
+    @defer.inlineCallbacks
+    def test_output_flag(self):
+        """Checks if a file was created successfully having
+        correct format containing correct data in it.
+        """
+        file_name = 'data.json'
+        file_path = join(self.proj_path, file_name)
+        yield self.execute([
+            '--spider', self.spider_name,
+            '-c', 'parse',
+            '-o', file_name,
+            self.url('/html')
+        ])
+
+        self.assertTrue(exists(file_path))
+        self.assertTrue(isfile(file_path))
+
+        content = '[\n{},\n{"foo": "bar"}\n]'
+        with open(file_path, 'r') as f:
+            self.assertEqual(f.read(), content)

From d54c4496ee57785f3d6f882e2d128bb64b6b262c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 13 Jul 2020 14:36:33 +0200
Subject: [PATCH 215/243] Refactor guess_scheme

---
 scrapy/utils/url.py     | 60 ++++++++++++++++++++++++++++++-----------
 tests/test_utils_url.py | 33 +++++++++++++++++++++--
 2 files changed, 75 insertions(+), 18 deletions(-)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 60e2b50eb..1e431d3bf 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -83,26 +83,54 @@ def add_http_if_no_scheme(url):
     return url
 
 
+def _is_posix_path(string):
+    return bool(
+        re.match(
+            r'''
+            ^                   # start with...
+            (
+                \.              # ...a single dot,
+                (
+                    \. | [^/\.]+  # optionally followed by
+                )?                # either a second dot or some characters
+                |
+                ~   # $HOME
+            )?      # optional match of ".", ".." or ".blabla"
+            /       # at least one "/" for a file path,
+            .       # and something after the "/"
+            ''',
+            string,
+            flags=re.VERBOSE,
+        )
+    )
+
+
+def _is_windows_path(string):
+    return bool(
+        re.match(
+            r'''
+            ^
+            (
+                [a-z]:\\
+                | \\\\
+            )
+            ''',
+            string,
+            flags=re.IGNORECASE | re.VERBOSE,
+        )
+    )
+
+
+def _is_path(string):
+    return _is_posix_path(string) or _is_windows_path(string)
+
+
 def guess_scheme(url):
     """Add an URL scheme if missing: file:// for filepath-like input or
     http:// otherwise."""
-    # POSIX path
-    if re.match(r'''^                   # start with...
-                    (
-                        \.              # ...a single dot,
-                        (
-                            \. | [^/\.]+  # optionally followed by
-                        )?                # either a second dot or some characters
-                    )?      # optional match of ".", ".." or ".blabla"
-                    /       # at least one "/" for a file path,
-                    .       # and something after the "/"
-                    ''', url, flags=re.VERBOSE):
+    if _is_path(url):
         return any_to_uri(url)
-    # Windows drive-letter path
-    elif re.match(r'''^[a-z]:\\''', url, flags=re.IGNORECASE):
-        return any_to_uri(url)
-    else:
-        return add_http_if_no_scheme(url)
+    return add_http_if_no_scheme(url)
 
 
 def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True):
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 09a6d6c70..6a5254d54 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,8 +1,14 @@
 import unittest
 
 from scrapy.spiders import Spider
-from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
-                              add_http_if_no_scheme, guess_scheme, strip_url)
+from scrapy.utils.url import (
+    add_http_if_no_scheme,
+    guess_scheme,
+    _is_path,
+    strip_url,
+    url_is_from_any_domain,
+    url_is_from_spider,
+)
 
 
 __doctests__ = ['scrapy.utils.url']
@@ -434,5 +440,28 @@ class StripUrl(unittest.TestCase):
             self.assertEqual(strip_url(i, origin_only=True), o)
 
 
+class IsPathTestCase(unittest.TestCase):
+
+    def test_path(self):
+        for input_value, output_value in (
+            # https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell
+            # Unix-like OS, Microsoft Windows / cmd.exe
+            ("/home/user/docs/Letter.txt", True),
+            ("./inthisdir", True),
+            ("../../greatgrandparent", True),
+            ("~/.rcinfo", True),
+            (r"C:\user\docs\Letter.txt", True),
+            ("/user/docs/Letter.txt", True),
+            (r"C:\Letter.txt", True),
+            (r"\\Server01\user\docs\Letter.txt", True),
+            (r"\\?\UNC\Server01\user\docs\Letter.txt", True),
+            (r"\\?\C:\user\docs\Letter.txt", True),
+            (r"C:\user\docs\somefile.ext:alternate_stream_name", True),
+
+            (r"https://example.com", False),
+        ):
+            self.assertEqual(_is_path(input_value), output_value, input_value)
+
+
 if __name__ == "__main__":
     unittest.main()

From 53c323b19d81784e6c376ce8b9602de24d8e3037 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 13 Jul 2020 15:29:30 +0200
Subject: [PATCH 216/243] =?UTF-8?q?=5Fis=5Fpath=20=E2=86=92=20=5Fis=5Ffile?=
 =?UTF-8?q?system=5Fpath?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scrapy/utils/url.py     | 4 ++--
 tests/test_utils_url.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 1e431d3bf..b23ddb459 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -121,14 +121,14 @@ def _is_windows_path(string):
     )
 
 
-def _is_path(string):
+def _is_filesystem_path(string):
     return _is_posix_path(string) or _is_windows_path(string)
 
 
 def guess_scheme(url):
     """Add an URL scheme if missing: file:// for filepath-like input or
     http:// otherwise."""
-    if _is_path(url):
+    if _is_filesystem_path(url):
         return any_to_uri(url)
     return add_http_if_no_scheme(url)
 
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 6a5254d54..3a143ba2f 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -4,7 +4,7 @@ from scrapy.spiders import Spider
 from scrapy.utils.url import (
     add_http_if_no_scheme,
     guess_scheme,
-    _is_path,
+    _is_filesystem_path,
     strip_url,
     url_is_from_any_domain,
     url_is_from_spider,
@@ -460,7 +460,7 @@ class IsPathTestCase(unittest.TestCase):
 
             (r"https://example.com", False),
         ):
-            self.assertEqual(_is_path(input_value), output_value, input_value)
+            self.assertEqual(_is_filesystem_path(input_value), output_value, input_value)
 
 
 if __name__ == "__main__":

From ed5247ca4cbef98d1acf499c888ded74e444ef48 Mon Sep 17 00:00:00 2001
From: Artur Shellunts <shellunts.artur@gmail.com>
Date: Tue, 14 Jul 2020 18:06:11 +0200
Subject: [PATCH 217/243] Remove htmlparser.py from tests/ignore.txt

---
 tests/ignores.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/ignores.txt b/tests/ignores.txt
index 45cf6fb92..f6e0d6fbe 100644
--- a/tests/ignores.txt
+++ b/tests/ignores.txt
@@ -1,6 +1,5 @@
 scrapy/linkextractors/sgml.py
 scrapy/linkextractors/regex.py
-scrapy/linkextractors/htmlparser.py
 scrapy/downloadermiddlewares/cookies.py
 scrapy/extensions/statsmailer.py
 scrapy/extensions/memusage.py

From 38496a00b7d2bcfa9d435551409e2c44007d168d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Wed, 15 Jul 2020 07:08:36 -0300
Subject: [PATCH 218/243] Use the itemlaoders library (#4516)

---
 docs/conf.py                    |  13 +
 docs/topics/loaders.rst         | 418 +-----------------
 scrapy/loader/__init__.py       | 269 +++---------
 scrapy/loader/common.py         |  19 +-
 scrapy/loader/processors.py     |  99 +----
 scrapy/utils/misc.py            |   6 +
 setup.cfg                       |   3 +
 setup.py                        |   1 +
 tests/requirements-py3.txt      |   1 -
 tests/test_loader.py            | 531 +----------------------
 tests/test_loader_deprecated.py | 720 ++++++++++++++++++++++++++++++++
 11 files changed, 854 insertions(+), 1226 deletions(-)
 create mode 100644 tests/test_loader_deprecated.py

diff --git a/docs/conf.py b/docs/conf.py
index 86734fae7..427c79481 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -284,6 +284,7 @@ intersphinx_mapping = {
     'attrs': ('https://www.attrs.org/en/stable/', None),
     'coverage': ('https://coverage.readthedocs.io/en/stable', None),
     'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
+    'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None),
     'pytest': ('https://docs.pytest.org/en/latest', None),
     'python': ('https://docs.python.org/3', None),
     'sphinx': ('https://www.sphinx-doc.org/en/master', None),
@@ -305,3 +306,15 @@ hoverxref_role_types = {
     "ref": "tooltip",
 }
 hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']
+
+
+def setup(app):
+    app.connect('autodoc-skip-member', maybe_skip_member)
+
+
+def maybe_skip_member(app, what, name, obj, skip, options):
+    if not skip:
+        # autodocs was generating a text "alias of" for the following members
+        # https://github.com/sphinx-doc/sphinx/issues/4422
+        return name in {'default_item_class', 'default_selector_class'}
+    return skip
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 9c82bb4d9..d0eeb4097 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -20,6 +20,10 @@ Item Loaders are designed to provide a flexible, efficient and easy mechanism
 for extending and overriding different field parsing rules, either by spider,
 or by source format (HTML, XML, etc) without becoming a nightmare to maintain.
 
+.. note:: Item Loaders are an extension of the itemloaders_ library that make it
+    easier to work with Scrapy by adding support for
+    :ref:`responses <topics-request-response>`.
+
 Using Item Loaders to populate items
 ====================================
 
@@ -173,8 +177,8 @@ The other thing you need to keep in mind is that the values returned by input
 processors are collected internally (in lists) and then passed to output
 processors to populate the fields.
 
-Last, but not least, Scrapy comes with some :ref:`commonly used processors
-<topics-loaders-available-processors>` built-in for convenience.
+Last, but not least, itemloaders_ comes with some :ref:`commonly used
+processors <itemloaders:built-in-processors>` built-in for convenience.
 
 
 Declaring Item Loaders
@@ -182,8 +186,8 @@ Declaring Item Loaders
 
 Item Loaders are declared using a class definition syntax. Here is an example::
 
+    from itemloaders.processors import TakeFirst, MapCompose, Join
     from scrapy.loader import ItemLoader
-    from scrapy.loader.processors import TakeFirst, MapCompose, Join
 
     class ProductLoader(ItemLoader):
 
@@ -214,7 +218,7 @@ output processors to use: in the :ref:`Item Field <topics-items-fields>`
 metadata. Here is an example::
 
     import scrapy
-    from scrapy.loader.processors import Join, MapCompose, TakeFirst
+    from itemloaders.processors import Join, MapCompose, TakeFirst
     from w3lib.html import remove_tags
 
     def filter_price(value):
@@ -295,250 +299,9 @@ There are several ways to modify Item Loader context values:
 ItemLoader objects
 ==================
 
-.. class:: ItemLoader([item, selector, response], **kwargs)
-
-    Return a new Item Loader for populating the given :ref:`item object
-    <topics-items>`. If no item object is given, one is instantiated
-    automatically using the class in :attr:`default_item_class`.
-
-    When instantiated with a ``selector`` or a ``response`` parameters
-    the :class:`ItemLoader` class provides convenient mechanisms for extracting
-    data from web pages using :ref:`selectors <topics-selectors>`.
-
-    :param item: The item instance to populate using subsequent calls to
-        :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
-        or :meth:`~ItemLoader.add_value`.
-    :type item: :ref:`item object <topics-items>`
-
-    :param selector: The selector to extract data from, when using the
-        :meth:`add_xpath` (resp. :meth:`add_css`) or :meth:`replace_xpath`
-        (resp. :meth:`replace_css`) method.
-    :type selector: :class:`~scrapy.selector.Selector` object
-
-    :param response: The response used to construct the selector using the
-        :attr:`default_selector_class`, unless the selector argument is given,
-        in which case this argument is ignored.
-    :type response: :class:`~scrapy.http.Response` object
-
-    The item, selector, response and the remaining keyword arguments are
-    assigned to the Loader context (accessible through the :attr:`context` attribute).
-
-    :class:`ItemLoader` instances have the following methods:
-
-    .. method:: get_value(value, *processors, **kwargs)
-
-        Process the given ``value`` by the given ``processors`` and keyword
-        arguments.
-
-        Available keyword arguments:
-
-        :param re: a regular expression to use for extracting data from the
-            given value using :meth:`~scrapy.utils.misc.extract_regex` method,
-            applied before processors
-        :type re: str or compiled regex
-
-        Examples:
-
-        >>> from scrapy.loader.processors import TakeFirst
-        >>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
-        'FOO`
-
-    .. method:: add_value(field_name, value, *processors, **kwargs)
-
-        Process and then add the given ``value`` for the given field.
-
-        The value is first passed through :meth:`get_value` by giving the
-        ``processors`` and ``kwargs``, and then passed through the
-        :ref:`field input processor <topics-loaders-processors>` and its result
-        appended to the data collected for that field. If the field already
-        contains collected data, the new data is added.
-
-        The given ``field_name`` can be ``None``, in which case values for
-        multiple fields may be added. And the processed value should be a dict
-        with field_name mapped to values.
-
-        Examples::
-
-            loader.add_value('name', u'Color TV')
-            loader.add_value('colours', [u'white', u'blue'])
-            loader.add_value('length', u'100')
-            loader.add_value('name', u'name: foo', TakeFirst(), re='name: (.+)')
-            loader.add_value(None, {'name': u'foo', 'sex': u'male'})
-
-    .. method:: replace_value(field_name, value, *processors, **kwargs)
-
-        Similar to :meth:`add_value` but replaces the collected data with the
-        new value instead of adding it.
-    .. method:: get_xpath(xpath, *processors, **kwargs)
-
-        Similar to :meth:`ItemLoader.get_value` but receives an XPath instead of a
-        value, which is used to extract a list of unicode strings from the
-        selector associated with this :class:`ItemLoader`.
-
-        :param xpath: the XPath to extract data from
-        :type xpath: str
-
-        :param re: a regular expression to use for extracting data from the
-            selected XPath region
-        :type re: str or compiled regex
-
-        Examples::
-
-            # HTML snippet: <p class="product-name">Color TV</p>
-            loader.get_xpath('//p[@class="product-name"]')
-            # HTML snippet: <p id="price">the price is $1200</p>
-            loader.get_xpath('//p[@id="price"]', TakeFirst(), re='the price is (.*)')
-
-    .. method:: add_xpath(field_name, xpath, *processors, **kwargs)
-
-        Similar to :meth:`ItemLoader.add_value` but receives an XPath instead of a
-        value, which is used to extract a list of unicode strings from the
-        selector associated with this :class:`ItemLoader`.
-
-        See :meth:`get_xpath` for ``kwargs``.
-
-        :param xpath: the XPath to extract data from
-        :type xpath: str
-
-        Examples::
-
-            # HTML snippet: <p class="product-name">Color TV</p>
-            loader.add_xpath('name', '//p[@class="product-name"]')
-            # HTML snippet: <p id="price">the price is $1200</p>
-            loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)')
-
-    .. method:: replace_xpath(field_name, xpath, *processors, **kwargs)
-
-        Similar to :meth:`add_xpath` but replaces collected data instead of
-        adding it.
-
-    .. method:: get_css(css, *processors, **kwargs)
-
-        Similar to :meth:`ItemLoader.get_value` but receives a CSS selector
-        instead of a value, which is used to extract a list of unicode strings
-        from the selector associated with this :class:`ItemLoader`.
-
-        :param css: the CSS selector to extract data from
-        :type css: str
-
-        :param re: a regular expression to use for extracting data from the
-            selected CSS region
-        :type re: str or compiled regex
-
-        Examples::
-
-            # HTML snippet: <p class="product-name">Color TV</p>
-            loader.get_css('p.product-name')
-            # HTML snippet: <p id="price">the price is $1200</p>
-            loader.get_css('p#price', TakeFirst(), re='the price is (.*)')
-
-    .. method:: add_css(field_name, css, *processors, **kwargs)
-
-        Similar to :meth:`ItemLoader.add_value` but receives a CSS selector
-        instead of a value, which is used to extract a list of unicode strings
-        from the selector associated with this :class:`ItemLoader`.
-
-        See :meth:`get_css` for ``kwargs``.
-
-        :param css: the CSS selector to extract data from
-        :type css: str
-
-        Examples::
-
-            # HTML snippet: <p class="product-name">Color TV</p>
-            loader.add_css('name', 'p.product-name')
-            # HTML snippet: <p id="price">the price is $1200</p>
-            loader.add_css('price', 'p#price', re='the price is (.*)')
-
-    .. method:: replace_css(field_name, css, *processors, **kwargs)
-
-        Similar to :meth:`add_css` but replaces collected data instead of
-        adding it.
-
-    .. method:: load_item()
-
-        Populate the item with the data collected so far, and return it. The
-        data collected is first passed through the :ref:`output processors
-        <topics-loaders-processors>` to get the final value to assign to each
-        item field.
-
-    .. method:: nested_xpath(xpath)
-
-        Create a nested loader with an xpath selector.
-        The supplied selector is applied relative to selector associated
-        with this :class:`ItemLoader`. The nested loader shares the :ref:`item
-        object <topics-items>` with the parent :class:`ItemLoader` so calls to
-        :meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
-        behave as expected.
-
-    .. method:: nested_css(css)
-
-        Create a nested loader with a css selector.
-        The supplied selector is applied relative to selector associated
-        with this :class:`ItemLoader`. The nested loader shares the :ref:`item
-        object <topics-items>` with the parent :class:`ItemLoader` so calls to
-        :meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
-        behave as expected.
-
-    .. method:: get_collected_values(field_name)
-
-        Return the collected values for the given field.
-
-    .. method:: get_output_value(field_name)
-
-        Return the collected values parsed using the output processor, for the
-        given field. This method doesn't populate or modify the item at all.
-
-    .. method:: get_input_processor(field_name)
-
-        Return the input processor for the given field.
-
-    .. method:: get_output_processor(field_name)
-
-        Return the output processor for the given field.
-
-    :class:`ItemLoader` instances have the following attributes:
-
-    .. attribute:: item
-
-        The :ref:`item object <topics-items>` being parsed by this Item Loader.
-        This is mostly used as a property so when attempting to override this
-        value, you may want to check out :attr:`default_item_class` first.
-
-    .. attribute:: context
-
-        The currently active :ref:`Context <topics-loaders-context>` of this
-        Item Loader.
-
-    .. attribute:: default_item_class
-
-        An :ref:`item object <topics-items>` class or factory, used to
-        instantiate items when not given in the ``__init__`` method.
-
-    .. attribute:: default_input_processor
-
-        The default input processor to use for those fields which don't specify
-        one.
-
-    .. attribute:: default_output_processor
-
-        The default output processor to use for those fields which don't specify
-        one.
-
-    .. attribute:: default_selector_class
-
-        The class used to construct the :attr:`selector` of this
-        :class:`ItemLoader`, if only a response is given in the ``__init__`` method.
-        If a selector is given in the ``__init__`` method this attribute is ignored.
-        This attribute is sometimes overridden in subclasses.
-
-    .. attribute:: selector
-
-        The :class:`~scrapy.selector.Selector` object to extract data from.
-        It's either the selector given in the ``__init__`` method or one created from
-        the response given in the ``__init__`` method using the
-        :attr:`default_selector_class`. This attribute is meant to be
-        read-only.
+.. autoclass:: scrapy.loader.ItemLoader
+    :members:
+    :inherited-members:
 
 .. _topics-loaders-nested:
 
@@ -609,7 +372,7 @@ those dashes in the final product names.
 Here's how you can remove those dashes by reusing and extending the default
 Product Item Loader (``ProductLoader``)::
 
-    from scrapy.loader.processors import MapCompose
+    from itemloaders.processors import MapCompose
     from myproject.ItemLoaders import ProductLoader
 
     def strip_dashes(x):
@@ -622,7 +385,7 @@ Another case where extending Item Loaders can be very helpful is when you have
 multiple source formats, for example XML and HTML. In the XML version you may
 want to remove ``CDATA`` occurrences. Here's an example of how to do it::
 
-    from scrapy.loader.processors import MapCompose
+    from itemloaders.processors import MapCompose
     from myproject.ItemLoaders import ProductLoader
     from myproject.utils.xml import remove_cdata
 
@@ -642,156 +405,5 @@ projects. Scrapy only provides the mechanism; it doesn't impose any specific
 organization of your Loaders collection - that's up to you and your project's
 needs.
 
-.. _topics-loaders-available-processors:
-
-Available built-in processors
-=============================
-
-.. module:: scrapy.loader.processors
-   :synopsis: A collection of processors to use with Item Loaders
-
-Even though you can use any callable function as input and output processors,
-Scrapy provides some commonly used processors, which are described below. Some
-of them, like the :class:`MapCompose` (which is typically used as input
-processor) compose the output of several functions executed in order, to
-produce the final parsed value.
-
-Here is a list of all built-in processors:
-
-.. class:: Identity
-
-    The simplest processor, which doesn't do anything. It returns the original
-    values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it
-    accept Loader contexts.
-
-    Example:
-
-    >>> from scrapy.loader.processors import Identity
-    >>> proc = Identity()
-    >>> proc(['one', 'two', 'three'])
-    ['one', 'two', 'three']
-
-.. class:: TakeFirst
-
-    Returns the first non-null/non-empty value from the values received,
-    so it's typically used as an output processor to single-valued fields.
-    It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts.
-
-    Example:
-
-    >>> from scrapy.loader.processors import TakeFirst
-    >>> proc = TakeFirst()
-    >>> proc(['', 'one', 'two', 'three'])
-    'one'
-
-.. class:: Join(separator=u' ')
-
-    Returns the values joined with the separator given in the ``__init__`` method, which
-    defaults to ``u' '``. It doesn't accept Loader contexts.
-
-    When using the default separator, this processor is equivalent to the
-    function: ``u' '.join``
-
-    Examples:
-
-    >>> from scrapy.loader.processors import Join
-    >>> proc = Join()
-    >>> proc(['one', 'two', 'three'])
-    'one two three'
-    >>> proc = Join('<br>')
-    >>> proc(['one', 'two', 'three'])
-    'one<br>two<br>three'
-
-.. class:: Compose(*functions, **default_loader_context)
-
-    A processor which is constructed from the composition of the given
-    functions. This means that each input value of this processor is passed to
-    the first function, and the result of that function is passed to the second
-    function, and so on, until the last function returns the output value of
-    this processor.
-
-    By default, stop process on ``None`` value. This behaviour can be changed by
-    passing keyword argument ``stop_on_none=False``.
-
-    Example:
-
-    >>> from scrapy.loader.processors import Compose
-    >>> proc = Compose(lambda v: v[0], str.upper)
-    >>> proc(['hello', 'world'])
-    'HELLO'
-
-    Each function can optionally receive a ``loader_context`` parameter. For
-    those which do, this processor will pass the currently active :ref:`Loader
-    context <topics-loaders-context>` through that parameter.
-
-    The keyword arguments passed in the ``__init__`` method are used as the default
-    Loader context values passed to each function call. However, the final
-    Loader context values passed to functions are overridden with the currently
-    active Loader context accessible through the :meth:`ItemLoader.context`
-    attribute.
-
-.. class:: MapCompose(*functions, **default_loader_context)
-
-    A processor which is constructed from the composition of the given
-    functions, similar to the :class:`Compose` processor. The difference with
-    this processor is the way internal results are passed among functions,
-    which is as follows:
-
-    The input value of this processor is *iterated* and the first function is
-    applied to each element. The results of these function calls (one for each element)
-    are concatenated to construct a new iterable, which is then used to apply the
-    second function, and so on, until the last function is applied to each
-    value of the list of values collected so far. The output values of the last
-    function are concatenated together to produce the output of this processor.
-
-    Each particular function can return a value or a list of values, which is
-    flattened with the list of values returned by the same function applied to
-    the other input values. The functions can also return ``None`` in which
-    case the output of that function is ignored for further processing over the
-    chain.
-
-    This processor provides a convenient way to compose functions that only
-    work with single values (instead of iterables). For this reason the
-    :class:`MapCompose` processor is typically used as input processor, since
-    data is often extracted using the
-    :meth:`~scrapy.selector.Selector.extract` method of :ref:`selectors
-    <topics-selectors>`, which returns a list of unicode strings.
-
-    The example below should clarify how it works:
-
-    >>> def filter_world(x):
-    ...     return None if x == 'world' else x
-    ...
-    >>> from scrapy.loader.processors import MapCompose
-    >>> proc = MapCompose(filter_world, str.upper)
-    >>> proc(['hello', 'world', 'this', 'is', 'scrapy'])
-    ['HELLO, 'THIS', 'IS', 'SCRAPY']
-
-    As with the Compose processor, functions can receive Loader contexts, and
-    ``__init__`` method keyword arguments are used as default context values. See
-    :class:`Compose` processor for more info.
-
-.. class:: SelectJmes(json_path)
-
-    Queries the value using the json path provided to the ``__init__`` method and returns the output.
-    Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
-    This processor takes only one input at a time.
-
-    Example:
-
-    >>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose
-    >>> proc = SelectJmes("foo") #for direct use on lists and dictionaries
-    >>> proc({'foo': 'bar'})
-    'bar'
-    >>> proc({'foo': {'bar': 'baz'}})
-    {'bar': 'baz'}
-
-    Working with Json:
-
-    >>> import json
-    >>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
-    >>> proc_single_json_str('{"foo": "bar"}')
-    'bar'
-    >>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
-    >>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
-    ['bar']
+.. _itemloaders: https://itemloaders.readthedocs.io/en/latest/
+.. _processors: https://itemloaders.readthedocs.io/en/latest/built-in-processors.html
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 18f57945f..014951a8e 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -3,217 +3,86 @@ Item Loader
 
 See documentation in docs/topics/loaders.rst
 """
-from collections import defaultdict
-from contextlib import suppress
-
-from itemadapter import ItemAdapter
+import itemloaders
 
 from scrapy.item import Item
-from scrapy.loader.common import wrap_loader_context
-from scrapy.loader.processors import Identity
 from scrapy.selector import Selector
-from scrapy.utils.misc import arg_to_iter, extract_regex
-from scrapy.utils.python import flatten
 
 
-def unbound_method(method):
+class ItemLoader(itemloaders.ItemLoader):
     """
-    Allow to use single-argument functions as input or output processors
-    (no need to define an unused first 'self' argument)
+    A user-friendly abstraction to populate an :ref:`item <topics-items>` with data
+    by applying :ref:`field processors <topics-loaders-processors>` to scraped data.
+    When instantiated with a ``selector`` or a ``response`` it supports
+    data extraction from web pages using :ref:`selectors <topics-selectors>`.
+
+    :param item: The item instance to populate using subsequent calls to
+        :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
+        or :meth:`~ItemLoader.add_value`.
+    :type item: scrapy.item.Item
+
+    :param selector: The selector to extract data from, when using the
+        :meth:`add_xpath`, :meth:`add_css`, :meth:`replace_xpath`, or
+        :meth:`replace_css` method.
+    :type selector: :class:`~scrapy.selector.Selector` object
+
+    :param response: The response used to construct the selector using the
+        :attr:`default_selector_class`, unless the selector argument is given,
+        in which case this argument is ignored.
+    :type response: :class:`~scrapy.http.Response` object
+
+    If no item is given, one is instantiated automatically using the class in
+    :attr:`default_item_class`.
+
+    The item, selector, response and remaining keyword arguments are
+    assigned to the Loader context (accessible through the :attr:`context` attribute).
+
+    .. attribute:: item
+
+        The item object being parsed by this Item Loader.
+        This is mostly used as a property so, when attempting to override this
+        value, you may want to check out :attr:`default_item_class` first.
+
+    .. attribute:: context
+
+        The currently active :ref:`Context <loaders-context>` of this Item Loader.
+
+    .. attribute:: default_item_class
+
+        An :ref:`item <topics-items>` class (or factory), used to instantiate
+        items when not given in the ``__init__`` method.
+
+    .. attribute:: default_input_processor
+
+        The default input processor to use for those fields which don't specify
+        one.
+
+    .. attribute:: default_output_processor
+
+        The default output processor to use for those fields which don't specify
+        one.
+
+    .. attribute:: default_selector_class
+
+        The class used to construct the :attr:`selector` of this
+        :class:`ItemLoader`, if only a response is given in the ``__init__`` method.
+        If a selector is given in the ``__init__`` method this attribute is ignored.
+        This attribute is sometimes overridden in subclasses.
+
+    .. attribute:: selector
+
+        The :class:`~scrapy.selector.Selector` object to extract data from.
+        It's either the selector given in the ``__init__`` method or one created from
+        the response given in the ``__init__`` method using the
+        :attr:`default_selector_class`. This attribute is meant to be
+        read-only.
     """
-    with suppress(AttributeError):
-        if '.' not in method.__qualname__:
-            return method.__func__
-    return method
-
-
-class ItemLoader:
 
     default_item_class = Item
-    default_input_processor = Identity()
-    default_output_processor = Identity()
     default_selector_class = Selector
 
     def __init__(self, item=None, selector=None, response=None, parent=None, **context):
         if selector is None and response is not None:
             selector = self.default_selector_class(response)
-        self.selector = selector
-        context.update(selector=selector, response=response)
-        if item is None:
-            item = self.default_item_class()
-        self.context = context
-        self.parent = parent
-        self._local_item = context['item'] = item
-        self._local_values = defaultdict(list)
-        # values from initial item
-        for field_name, value in ItemAdapter(item).items():
-            self._values[field_name] += arg_to_iter(value)
-
-    @property
-    def _values(self):
-        if self.parent is not None:
-            return self.parent._values
-        else:
-            return self._local_values
-
-    @property
-    def item(self):
-        if self.parent is not None:
-            return self.parent.item
-        else:
-            return self._local_item
-
-    def nested_xpath(self, xpath, **context):
-        selector = self.selector.xpath(xpath)
-        context.update(selector=selector)
-        subloader = self.__class__(
-            item=self.item, parent=self, **context
-        )
-        return subloader
-
-    def nested_css(self, css, **context):
-        selector = self.selector.css(css)
-        context.update(selector=selector)
-        subloader = self.__class__(
-            item=self.item, parent=self, **context
-        )
-        return subloader
-
-    def add_value(self, field_name, value, *processors, **kw):
-        value = self.get_value(value, *processors, **kw)
-        if value is None:
-            return
-        if not field_name:
-            for k, v in value.items():
-                self._add_value(k, v)
-        else:
-            self._add_value(field_name, value)
-
-    def replace_value(self, field_name, value, *processors, **kw):
-        value = self.get_value(value, *processors, **kw)
-        if value is None:
-            return
-        if not field_name:
-            for k, v in value.items():
-                self._replace_value(k, v)
-        else:
-            self._replace_value(field_name, value)
-
-    def _add_value(self, field_name, value):
-        value = arg_to_iter(value)
-        processed_value = self._process_input_value(field_name, value)
-        if processed_value:
-            self._values[field_name] += arg_to_iter(processed_value)
-
-    def _replace_value(self, field_name, value):
-        self._values.pop(field_name, None)
-        self._add_value(field_name, value)
-
-    def get_value(self, value, *processors, **kw):
-        regex = kw.get('re', None)
-        if regex:
-            value = arg_to_iter(value)
-            value = flatten(extract_regex(regex, x) for x in value)
-
-        for proc in processors:
-            if value is None:
-                break
-            _proc = proc
-            proc = wrap_loader_context(proc, self.context)
-            try:
-                value = proc(value)
-            except Exception as e:
-                raise ValueError("Error with processor %s value=%r error='%s: %s'" %
-                                 (_proc.__class__.__name__, value,
-                                  type(e).__name__, str(e)))
-        return value
-
-    def load_item(self):
-        adapter = ItemAdapter(self.item)
-        for field_name in tuple(self._values):
-            value = self.get_output_value(field_name)
-            if value is not None:
-                adapter[field_name] = value
-        return adapter.item
-
-    def get_output_value(self, field_name):
-        proc = self.get_output_processor(field_name)
-        proc = wrap_loader_context(proc, self.context)
-        try:
-            return proc(self._values[field_name])
-        except Exception as e:
-            raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" %
-                             (field_name, self._values[field_name], type(e).__name__, str(e)))
-
-    def get_collected_values(self, field_name):
-        return self._values[field_name]
-
-    def get_input_processor(self, field_name):
-        proc = getattr(self, '%s_in' % field_name, None)
-        if not proc:
-            proc = self._get_item_field_attr(field_name, 'input_processor',
-                                             self.default_input_processor)
-        return unbound_method(proc)
-
-    def get_output_processor(self, field_name):
-        proc = getattr(self, '%s_out' % field_name, None)
-        if not proc:
-            proc = self._get_item_field_attr(field_name, 'output_processor',
-                                             self.default_output_processor)
-        return unbound_method(proc)
-
-    def _process_input_value(self, field_name, value):
-        proc = self.get_input_processor(field_name)
-        _proc = proc
-        proc = wrap_loader_context(proc, self.context)
-        try:
-            return proc(value)
-        except Exception as e:
-            raise ValueError(
-                "Error with input processor %s: field=%r value=%r "
-                "error='%s: %s'" % (_proc.__class__.__name__, field_name,
-                                    value, type(e).__name__, str(e)))
-
-    def _get_item_field_attr(self, field_name, key, default=None):
-        field_meta = ItemAdapter(self.item).get_field_meta(field_name)
-        return field_meta.get(key, default)
-
-    def _check_selector_method(self):
-        if self.selector is None:
-            raise RuntimeError("To use XPath or CSS selectors, "
-                               "%s must be instantiated with a selector "
-                               "or a response" % self.__class__.__name__)
-
-    def add_xpath(self, field_name, xpath, *processors, **kw):
-        values = self._get_xpathvalues(xpath, **kw)
-        self.add_value(field_name, values, *processors, **kw)
-
-    def replace_xpath(self, field_name, xpath, *processors, **kw):
-        values = self._get_xpathvalues(xpath, **kw)
-        self.replace_value(field_name, values, *processors, **kw)
-
-    def get_xpath(self, xpath, *processors, **kw):
-        values = self._get_xpathvalues(xpath, **kw)
-        return self.get_value(values, *processors, **kw)
-
-    def _get_xpathvalues(self, xpaths, **kw):
-        self._check_selector_method()
-        xpaths = arg_to_iter(xpaths)
-        return flatten(self.selector.xpath(xpath).getall() for xpath in xpaths)
-
-    def add_css(self, field_name, css, *processors, **kw):
-        values = self._get_cssvalues(css, **kw)
-        self.add_value(field_name, values, *processors, **kw)
-
-    def replace_css(self, field_name, css, *processors, **kw):
-        values = self._get_cssvalues(css, **kw)
-        self.replace_value(field_name, values, *processors, **kw)
-
-    def get_css(self, css, *processors, **kw):
-        values = self._get_cssvalues(css, **kw)
-        return self.get_value(values, *processors, **kw)
-
-    def _get_cssvalues(self, csss, **kw):
-        self._check_selector_method()
-        csss = arg_to_iter(csss)
-        return flatten(self.selector.css(css).getall() for css in csss)
+        context.update(response=response)
+        super().__init__(item=item, selector=selector, parent=parent, **context)
diff --git a/scrapy/loader/common.py b/scrapy/loader/common.py
index 42f8de636..3b8a6ee94 100644
--- a/scrapy/loader/common.py
+++ b/scrapy/loader/common.py
@@ -1,14 +1,21 @@
 """Common functions used in Item Loaders code"""
 
-from functools import partial
-from scrapy.utils.python import get_func_args
+import warnings
+
+from itemloaders import common
+
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
 
 
 def wrap_loader_context(function, context):
     """Wrap functions that receive loader_context to contain the context
     "pre-loaded" and expose a interface that receives only one argument
     """
-    if 'loader_context' in get_func_args(function):
-        return partial(function, loader_context=context)
-    else:
-        return function
+    warnings.warn(
+        "scrapy.loader.common.wrap_loader_context has moved to a new library."
+        "Please update your reference to itemloaders.common.wrap_loader_context",
+        ScrapyDeprecationWarning,
+        stacklevel=2
+    )
+
+    return common.wrap_loader_context(function, context)
diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py
index a7be65609..51fbd19eb 100644
--- a/scrapy/loader/processors.py
+++ b/scrapy/loader/processors.py
@@ -3,102 +3,19 @@ This module provides some commonly used processors for Item Loaders.
 
 See documentation in docs/topics/loaders.rst
 """
-from collections import ChainMap
+from itemloaders import processors
 
-from scrapy.utils.misc import arg_to_iter
-from scrapy.loader.common import wrap_loader_context
+from scrapy.utils.deprecate import create_deprecated_class
 
 
-class MapCompose:
+MapCompose = create_deprecated_class('MapCompose', processors.MapCompose)
 
-    def __init__(self, *functions, **default_loader_context):
-        self.functions = functions
-        self.default_loader_context = default_loader_context
+Compose = create_deprecated_class('Compose', processors.Compose)
 
-    def __call__(self, value, loader_context=None):
-        values = arg_to_iter(value)
-        if loader_context:
-            context = ChainMap(loader_context, self.default_loader_context)
-        else:
-            context = self.default_loader_context
-        wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions]
-        for func in wrapped_funcs:
-            next_values = []
-            for v in values:
-                try:
-                    next_values += arg_to_iter(func(v))
-                except Exception as e:
-                    raise ValueError("Error in MapCompose with "
-                                     "%s value=%r error='%s: %s'" %
-                                     (str(func), value, type(e).__name__,
-                                      str(e)))
-            values = next_values
-        return values
+TakeFirst = create_deprecated_class('TakeFirst', processors.TakeFirst)
 
+Identity = create_deprecated_class('Identity', processors.Identity)
 
-class Compose:
+SelectJmes = create_deprecated_class('SelectJmes', processors.SelectJmes)
 
-    def __init__(self, *functions, **default_loader_context):
-        self.functions = functions
-        self.stop_on_none = default_loader_context.get('stop_on_none', True)
-        self.default_loader_context = default_loader_context
-
-    def __call__(self, value, loader_context=None):
-        if loader_context:
-            context = ChainMap(loader_context, self.default_loader_context)
-        else:
-            context = self.default_loader_context
-        wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions]
-        for func in wrapped_funcs:
-            if value is None and self.stop_on_none:
-                break
-            try:
-                value = func(value)
-            except Exception as e:
-                raise ValueError("Error in Compose with "
-                                 "%s value=%r error='%s: %s'" %
-                                 (str(func), value, type(e).__name__, str(e)))
-        return value
-
-
-class TakeFirst:
-
-    def __call__(self, values):
-        for value in values:
-            if value is not None and value != '':
-                return value
-
-
-class Identity:
-
-    def __call__(self, values):
-        return values
-
-
-class SelectJmes:
-    """
-        Query the input string for the jmespath (given at instantiation),
-        and return the answer
-        Requires : jmespath(https://github.com/jmespath/jmespath)
-        Note: SelectJmes accepts only one input element at a time.
-    """
-    def __init__(self, json_path):
-        self.json_path = json_path
-        import jmespath
-        self.compiled_path = jmespath.compile(self.json_path)
-
-    def __call__(self, value):
-        """Query value for the jmespath query and return answer
-        :param value: a data structure (dict, list) to extract from
-        :return: Element extracted according to jmespath query
-        """
-        return self.compiled_path.search(value)
-
-
-class Join:
-
-    def __init__(self, separator=u' '):
-        self.separator = separator
-
-    def __call__(self, values):
-        return self.separator.join(values)
+Join = create_deprecated_class('Join', processors.Join)
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 8e5fde246..d6966be8e 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -15,6 +15,7 @@ from w3lib.html import replace_entities
 from scrapy.utils.datatypes import LocalWeakReferencedCache
 from scrapy.utils.python import flatten, to_unicode
 from scrapy.item import _BaseItem
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
 
 
 _ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes
@@ -86,6 +87,11 @@ def extract_regex(regex, text, encoding='utf-8'):
     * if the regex contains multiple numbered groups, all those will be returned (flattened)
     * if the regex doesn't contain any group the entire regex matching is returned
     """
+    warnings.warn(
+        "scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.",
+        ScrapyDeprecationWarning,
+        stacklevel=2
+    )
 
     if isinstance(regex, str):
         regex = re.compile(regex, re.UNICODE)
diff --git a/setup.cfg b/setup.cfg
index a9138c1c0..46a3d13fc 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -118,6 +118,9 @@ ignore_errors = True
 [mypy-tests.test_loader]
 ignore_errors = True
 
+[mypy-tests.test_loader_deprecated]
+ignore_errors = True
+
 [mypy-tests.test_pipeline_crawl]
 ignore_errors = True
 
diff --git a/setup.py b/setup.py
index 5a99fd1bf..f8d9b491b 100644
--- a/setup.py
+++ b/setup.py
@@ -71,6 +71,7 @@ setup(
         'Twisted>=17.9.0',
         'cryptography>=2.0',
         'cssselect>=0.9.1',
+        'itemloaders>=1.0.1',
         'lxml>=3.5.0',
         'parsel>=1.5.0',
         'PyDispatcher>=2.0.5',
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index dacb86e56..0551b1e95 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,7 +1,6 @@
 # Tests requirements
 attrs
 dataclasses; python_version == '3.6'
-jmespath
 mitmproxy; python_version >= '3.6'
 mitmproxy<4.0.0; python_version < '3.6'
 # https://github.com/pytest-dev/pytest-twisted/issues/93
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 8a9c6fca9..581183625 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -1,14 +1,12 @@
-from functools import partial
 import unittest
 
 import attr
 from itemadapter import ItemAdapter
+from itemloaders.processors import Compose, Identity, MapCompose, TakeFirst
 
 from scrapy.http import HtmlResponse
 from scrapy.item import Item, Field
 from scrapy.loader import ItemLoader
-from scrapy.loader.processors import (Compose, Identity, Join,
-                                      MapCompose, SelectJmes, TakeFirst)
 from scrapy.selector import Selector
 
 
@@ -69,6 +67,10 @@ def processor_with_args(value, other=None, loader_context=None):
 
 class BasicItemLoaderTest(unittest.TestCase):
 
+    def test_add_value_on_unknown_field(self):
+        il = TestItemLoader()
+        self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo'])
+
     def test_load_item_using_default_loader(self):
         i = TestItem()
         i['summary'] = u'lala'
@@ -85,391 +87,6 @@ class BasicItemLoaderTest(unittest.TestCase):
         item = il.load_item()
         self.assertEqual(item['name'], [u'Marta'])
 
-    def test_load_item_ignore_none_field_values(self):
-        def validate_sku(value):
-            # Let's assume a SKU is only digits.
-            if value.isdigit():
-                return value
-
-        class MyLoader(ItemLoader):
-            name_out = Compose(lambda vs: vs[0])  # take first which allows empty values
-            price_out = Compose(TakeFirst(), float)
-            sku_out = Compose(TakeFirst(), validate_sku)
-
-        valid_fragment = u'SKU: 1234'
-        invalid_fragment = u'SKU: not available'
-        sku_re = 'SKU: (.+)'
-
-        il = MyLoader(item={})
-        # Should not return "sku: None".
-        il.add_value('sku', [invalid_fragment], re=sku_re)
-        # Should not ignore empty values.
-        il.add_value('name', u'')
-        il.add_value('price', [u'0'])
-        self.assertEqual(il.load_item(), {
-            'name': u'',
-            'price': 0.0,
-        })
-
-        il.replace_value('sku', [valid_fragment], re=sku_re)
-        self.assertEqual(il.load_item()['sku'], u'1234')
-
-    def test_self_referencing_loader(self):
-        class MyLoader(ItemLoader):
-            url_out = TakeFirst()
-
-            def img_url_out(self, values):
-                return (self.get_output_value('url') or '') + values[0]
-
-        il = MyLoader(item={})
-        il.add_value('url', 'http://example.com/')
-        il.add_value('img_url', '1234.png')
-        self.assertEqual(il.load_item(), {
-            'url': 'http://example.com/',
-            'img_url': 'http://example.com/1234.png',
-        })
-
-        il = MyLoader(item={})
-        il.add_value('img_url', '1234.png')
-        self.assertEqual(il.load_item(), {
-            'img_url': '1234.png',
-        })
-
-    def test_add_value(self):
-        il = TestItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'Marta'])
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
-        il.add_value('name', u'pepe')
-        self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe'])
-        self.assertEqual(il.get_output_value('name'), [u'Marta', u'Pepe'])
-
-        # test add object value
-        il.add_value('summary', {'key': 1})
-        self.assertEqual(il.get_collected_values('summary'), [{'key': 1}])
-
-        il.add_value(None, u'Jim', lambda x: {'name': x})
-        self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe', u'Jim'])
-
-    def test_add_zero(self):
-        il = NameItemLoader()
-        il.add_value('name', 0)
-        self.assertEqual(il.get_collected_values('name'), [0])
-
-    def test_replace_value(self):
-        il = TestItemLoader()
-        il.replace_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'Marta'])
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
-        il.replace_value('name', u'pepe')
-        self.assertEqual(il.get_collected_values('name'), [u'Pepe'])
-        self.assertEqual(il.get_output_value('name'), [u'Pepe'])
-
-        il.replace_value(None, u'Jim', lambda x: {'name': x})
-        self.assertEqual(il.get_collected_values('name'), [u'Jim'])
-
-    def test_get_value(self):
-        il = NameItemLoader()
-        self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper))
-        self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$'))
-        self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$'))
-
-        il.add_value('name', [u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')
-        self.assertEqual([u'foo'], il.get_collected_values('name'))
-        il.replace_value('name', u'name:bar', re=u'name:(.*)$')
-        self.assertEqual([u'bar'], il.get_collected_values('name'))
-
-    def test_iter_on_input_processor_input(self):
-        class NameFirstItemLoader(NameItemLoader):
-            name_in = TakeFirst()
-
-        il = NameFirstItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
-        il = NameFirstItemLoader()
-        il.add_value('name', [u'marta', u'jose'])
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
-
-        il = NameFirstItemLoader()
-        il.replace_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
-        il = NameFirstItemLoader()
-        il.replace_value('name', [u'marta', u'jose'])
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
-
-        il = NameFirstItemLoader()
-        il.add_value('name', u'marta')
-        il.add_value('name', [u'jose', u'pedro'])
-        self.assertEqual(il.get_collected_values('name'), [u'marta', u'jose'])
-
-    def test_map_compose_filter(self):
-        def filter_world(x):
-            return None if x == 'world' else x
-
-        proc = MapCompose(filter_world, str.upper)
-        self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']),
-                         ['HELLO', 'THIS', 'IS', 'SCRAPY'])
-
-    def test_map_compose_filter_multil(self):
-        class TestItemLoader(NameItemLoader):
-            name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1])
-
-        il = TestItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'Mart'])
-        item = il.load_item()
-        self.assertEqual(item['name'], [u'Mart'])
-
-    def test_default_input_processor(self):
-        il = DefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'mart'])
-
-    def test_inherited_default_input_processor(self):
-        class InheritDefaultedItemLoader(DefaultedItemLoader):
-            pass
-
-        il = InheritDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'mart'])
-
-    def test_input_processor_inheritance(self):
-        class ChildItemLoader(TestItemLoader):
-            url_in = MapCompose(lambda v: v.lower())
-
-        il = ChildItemLoader()
-        il.add_value('url', u'HTTP://scrapy.ORG')
-        self.assertEqual(il.get_output_value('url'), [u'http://scrapy.org'])
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
-
-        class ChildChildItemLoader(ChildItemLoader):
-            url_in = MapCompose(lambda v: v.upper())
-            summary_in = MapCompose(lambda v: v)
-
-        il = ChildChildItemLoader()
-        il.add_value('url', u'http://scrapy.org')
-        self.assertEqual(il.get_output_value('url'), [u'HTTP://SCRAPY.ORG'])
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
-
-    def test_empty_map_compose(self):
-        class IdentityDefaultedItemLoader(DefaultedItemLoader):
-            name_in = MapCompose()
-
-        il = IdentityDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'marta'])
-
-    def test_identity_input_processor(self):
-        class IdentityDefaultedItemLoader(DefaultedItemLoader):
-            name_in = Identity()
-
-        il = IdentityDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'marta'])
-
-    def test_extend_custom_input_processors(self):
-        class ChildItemLoader(TestItemLoader):
-            name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
-
-        il = ChildItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'mARTA'])
-
-    def test_extend_default_input_processors(self):
-        class ChildDefaultedItemLoader(DefaultedItemLoader):
-            name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase)
-
-        il = ChildDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'MART'])
-
-    def test_output_processor_using_function(self):
-        il = TestItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
-
-        class TakeFirstItemLoader(TestItemLoader):
-            name_out = u" ".join
-
-        il = TakeFirstItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), u'Mar Ta')
-
-    def test_output_processor_error(self):
-        class TestItemLoader(ItemLoader):
-            default_item_class = TestItem
-            name_out = MapCompose(float)
-
-        il = TestItemLoader()
-        il.add_value('name', [u'$10'])
-        try:
-            float(u'$10')
-        except Exception as e:
-            expected_exc_str = str(e)
-
-        exc = None
-        try:
-            il.load_item()
-        except Exception as e:
-            exc = e
-        assert isinstance(exc, ValueError)
-        s = str(exc)
-        assert 'name' in s, s
-        assert '$10' in s, s
-        assert 'ValueError' in s, s
-        assert expected_exc_str in s, s
-
-    def test_output_processor_using_classes(self):
-        il = TestItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
-
-        class TakeFirstItemLoader(TestItemLoader):
-            name_out = Join()
-
-        il = TakeFirstItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), u'Mar Ta')
-
-        class TakeFirstItemLoader(TestItemLoader):
-            name_out = Join("<br>")
-
-        il = TakeFirstItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), u'Mar<br>Ta')
-
-    def test_default_output_processor(self):
-        il = TestItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
-
-        class LalaItemLoader(TestItemLoader):
-            default_output_processor = Identity()
-
-        il = LalaItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
-
-    def test_loader_context_on_declaration(self):
-        class ChildItemLoader(TestItemLoader):
-            url_in = MapCompose(processor_with_args, key=u'val')
-
-        il = ChildItemLoader()
-        il.add_value('url', u'text')
-        self.assertEqual(il.get_output_value('url'), ['val'])
-        il.replace_value('url', u'text2')
-        self.assertEqual(il.get_output_value('url'), ['val'])
-
-    def test_loader_context_on_instantiation(self):
-        class ChildItemLoader(TestItemLoader):
-            url_in = MapCompose(processor_with_args)
-
-        il = ChildItemLoader(key=u'val')
-        il.add_value('url', u'text')
-        self.assertEqual(il.get_output_value('url'), ['val'])
-        il.replace_value('url', u'text2')
-        self.assertEqual(il.get_output_value('url'), ['val'])
-
-    def test_loader_context_on_assign(self):
-        class ChildItemLoader(TestItemLoader):
-            url_in = MapCompose(processor_with_args)
-
-        il = ChildItemLoader()
-        il.context['key'] = u'val'
-        il.add_value('url', u'text')
-        self.assertEqual(il.get_output_value('url'), ['val'])
-        il.replace_value('url', u'text2')
-        self.assertEqual(il.get_output_value('url'), ['val'])
-
-    def test_item_passed_to_input_processor_functions(self):
-        def processor(value, loader_context):
-            return loader_context['item']['name']
-
-        class ChildItemLoader(TestItemLoader):
-            url_in = MapCompose(processor)
-
-        it = TestItem(name='marta')
-        il = ChildItemLoader(item=it)
-        il.add_value('url', u'text')
-        self.assertEqual(il.get_output_value('url'), ['marta'])
-        il.replace_value('url', u'text2')
-        self.assertEqual(il.get_output_value('url'), ['marta'])
-
-    def test_add_value_on_unknown_field(self):
-        il = TestItemLoader()
-        self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo'])
-
-    def test_compose_processor(self):
-        class TestItemLoader(NameItemLoader):
-            name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1])
-
-        il = TestItemLoader()
-        il.add_value('name', [u'marta', u'other'])
-        self.assertEqual(il.get_output_value('name'), u'Mart')
-        item = il.load_item()
-        self.assertEqual(item['name'], u'Mart')
-
-    def test_partial_processor(self):
-        def join(values, sep=None, loader_context=None, ignored=None):
-            if sep is not None:
-                return sep.join(values)
-            elif loader_context and 'sep' in loader_context:
-                return loader_context['sep'].join(values)
-            else:
-                return ''.join(values)
-
-        class TestItemLoader(NameItemLoader):
-            name_out = Compose(partial(join, sep='+'))
-            url_out = Compose(partial(join, loader_context={'sep': '.'}))
-            summary_out = Compose(partial(join, ignored='foo'))
-
-        il = TestItemLoader()
-        il.add_value('name', [u'rabbit', u'hole'])
-        il.add_value('url', [u'rabbit', u'hole'])
-        il.add_value('summary', [u'rabbit', u'hole'])
-        item = il.load_item()
-        self.assertEqual(item['name'], u'rabbit+hole')
-        self.assertEqual(item['url'], u'rabbit.hole')
-        self.assertEqual(item['summary'], u'rabbithole')
-
-    def test_error_input_processor(self):
-        class TestItem(Item):
-            name = Field()
-
-        class TestItemLoader(ItemLoader):
-            default_item_class = TestItem
-            name_in = MapCompose(float)
-
-        il = TestItemLoader()
-        self.assertRaises(ValueError, il.add_value, 'name',
-                          [u'marta', u'other'])
-
-    def test_error_output_processor(self):
-        class TestItem(Item):
-            name = Field()
-
-        class TestItemLoader(ItemLoader):
-            default_item_class = TestItem
-            name_out = Compose(Join(), float)
-
-        il = TestItemLoader()
-        il.add_value('name', u'marta')
-        with self.assertRaises(ValueError):
-            il.load_item()
-
-    def test_error_processor_as_argument(self):
-        class TestItem(Item):
-            name = Field()
-
-        class TestItemLoader(ItemLoader):
-            default_item_class = TestItem
-
-        il = TestItemLoader()
-        self.assertRaises(ValueError, il.add_value, 'name',
-                          [u'marta', u'other'], Compose(float))
-
 
 class InitializationTestMixin:
 
@@ -587,41 +204,6 @@ class BaseNoInputReprocessingLoader(ItemLoader):
     title_out = TakeFirst()
 
 
-class NoInputReprocessingDictLoader(BaseNoInputReprocessingLoader):
-    default_item_class = dict
-
-
-class NoInputReprocessingFromDictTest(unittest.TestCase):
-    """
-    Loaders initialized from loaded items must not reprocess fields (dict instances)
-    """
-    def test_avoid_reprocessing_with_initial_values_single(self):
-        il = NoInputReprocessingDictLoader(item=dict(title='foo'))
-        il_loaded = il.load_item()
-        self.assertEqual(il_loaded, dict(title='foo'))
-        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo'))
-
-    def test_avoid_reprocessing_with_initial_values_list(self):
-        il = NoInputReprocessingDictLoader(item=dict(title=['foo', 'bar']))
-        il_loaded = il.load_item()
-        self.assertEqual(il_loaded, dict(title='foo'))
-        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo'))
-
-    def test_avoid_reprocessing_without_initial_values_single(self):
-        il = NoInputReprocessingDictLoader()
-        il.add_value('title', 'foo')
-        il_loaded = il.load_item()
-        self.assertEqual(il_loaded, dict(title='FOO'))
-        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO'))
-
-    def test_avoid_reprocessing_without_initial_values_list(self):
-        il = NoInputReprocessingDictLoader()
-        il.add_value('title', ['foo', 'bar'])
-        il_loaded = il.load_item()
-        self.assertEqual(il_loaded, dict(title='FOO'))
-        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO'))
-
-
 class NoInputReprocessingItem(Item):
     title = Field()
 
@@ -661,25 +243,6 @@ class NoInputReprocessingFromItemTest(unittest.TestCase):
         self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'FOO'})
 
 
-class TestOutputProcessorDict(unittest.TestCase):
-    def test_output_processor(self):
-
-        class TempDict(dict):
-            def __init__(self, *args, **kwargs):
-                super(TempDict, self).__init__(self, *args, **kwargs)
-                self.setdefault('temp', 0.3)
-
-        class TempLoader(ItemLoader):
-            default_item_class = TempDict
-            default_input_processor = Identity()
-            default_output_processor = Compose(TakeFirst())
-
-        loader = TempLoader()
-        item = loader.load_item()
-        self.assertIsInstance(item, TempDict)
-        self.assertEqual(dict(item), {'temp': 0.3})
-
-
 class TestOutputProcessorItem(unittest.TestCase):
     def test_output_processor(self):
 
@@ -701,49 +264,6 @@ class TestOutputProcessorItem(unittest.TestCase):
         self.assertEqual(dict(item), {'temp': 0.3})
 
 
-class ProcessorsTest(unittest.TestCase):
-
-    def test_take_first(self):
-        proc = TakeFirst()
-        self.assertEqual(proc([None, '', 'hello', 'world']), 'hello')
-        self.assertEqual(proc([None, '', 0, 'hello', 'world']), 0)
-
-    def test_identity(self):
-        proc = Identity()
-        self.assertEqual(proc([None, '', 'hello', 'world']),
-                         [None, '', 'hello', 'world'])
-
-    def test_join(self):
-        proc = Join()
-        self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
-        self.assertEqual(proc(['', 'hello', 'world']), u' hello world')
-        self.assertEqual(proc(['hello', 'world']), u'hello world')
-        self.assertIsInstance(proc(['hello', 'world']), str)
-
-    def test_compose(self):
-        proc = Compose(lambda v: v[0], str.upper)
-        self.assertEqual(proc(['hello', 'world']), 'HELLO')
-        proc = Compose(str.upper)
-        self.assertEqual(proc(None), None)
-        proc = Compose(str.upper, stop_on_none=False)
-        self.assertRaises(ValueError, proc, None)
-        proc = Compose(str.upper, lambda x: x + 1)
-        self.assertRaises(ValueError, proc, 'hello')
-
-    def test_mapcompose(self):
-        def filter_world(x):
-            return None if x == 'world' else x
-        proc = MapCompose(filter_world, str.upper)
-        self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
-                         [u'HELLO', u'THIS', u'IS', u'SCRAPY'])
-        proc = MapCompose(filter_world, str.upper)
-        self.assertEqual(proc(None), [])
-        proc = MapCompose(filter_world, str.upper)
-        self.assertRaises(ValueError, proc, [1])
-        proc = MapCompose(filter_world, lambda x: x + 1)
-        self.assertRaises(ValueError, proc, 'hello')
-
-
 class SelectortemLoaderTest(unittest.TestCase):
     response = HtmlResponse(url="", encoding='utf-8', body=b"""
     <html>
@@ -921,6 +441,7 @@ class SubselectorLoaderTest(unittest.TestCase):
 
     def test_nested_xpath(self):
         l = NestedItemLoader(response=self.response)
+
         nl = l.nested_xpath("//header")
         nl.add_xpath('name', 'div/text()')
         nl.add_css('name_div', '#id')
@@ -998,31 +519,6 @@ class SubselectorLoaderTest(unittest.TestCase):
         self.assertEqual(item['image'], [u'/images/logo.png'])
 
 
-class SelectJmesTestCase(unittest.TestCase):
-    test_list_equals = {
-        'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
-        'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None),
-        'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}),
-        'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
-        'dict': (
-            'foo.bar[*].name',
-            {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}},
-            ['one', 'two']
-        ),
-        'list': ('[1]', [1, 2], 2)
-    }
-
-    def test_output(self):
-        for l in self.test_list_equals:
-            expr, test_list, expected = self.test_list_equals[l]
-            test = SelectJmes(expr)(test_list)
-            self.assertEqual(
-                test,
-                expected,
-                msg='test "{}" got {} expected {}'.format(l, test, expected)
-            )
-
-
 # Functions as processors
 
 def function_processor_strip(iterable):
@@ -1044,12 +540,6 @@ class FunctionProcessorItemLoader(ItemLoader):
     default_item_class = FunctionProcessorItem
 
 
-class FunctionProcessorDictLoader(ItemLoader):
-    default_item_class = dict
-    foo_in = function_processor_strip
-    foo_out = function_processor_upper
-
-
 class FunctionProcessorTestCase(unittest.TestCase):
 
     def test_processor_defined_in_item(self):
@@ -1061,15 +551,6 @@ class FunctionProcessorTestCase(unittest.TestCase):
             {'foo': ['BAR', 'ASDF', 'QWERTY']}
         )
 
-    def test_processor_defined_in_item_loader(self):
-        lo = FunctionProcessorDictLoader()
-        lo.add_value('foo', '  bar  ')
-        lo.add_value('foo', ['  asdf  ', '  qwerty  '])
-        self.assertEqual(
-            dict(lo.load_item()),
-            {'foo': ['BAR', 'ASDF', 'QWERTY']}
-        )
-
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py
new file mode 100644
index 000000000..d0a59e8cd
--- /dev/null
+++ b/tests/test_loader_deprecated.py
@@ -0,0 +1,720 @@
+"""
+These tests are kept as references from the ones that were ported to a itemloaders library.
+Once we remove the references from scrapy, we can remove these tests.
+"""
+
+import unittest
+import warnings
+from functools import partial
+
+from itemloaders.processors import (Compose, Identity, Join,
+                                    MapCompose, SelectJmes, TakeFirst)
+
+from scrapy.item import Item, Field
+from scrapy.loader import ItemLoader
+from scrapy.loader.common import wrap_loader_context
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
+from scrapy.utils.misc import extract_regex
+
+
+# test items
+class NameItem(Item):
+    name = Field()
+
+
+class TestItem(NameItem):
+    url = Field()
+    summary = Field()
+
+
+# test item loaders
+class NameItemLoader(ItemLoader):
+    default_item_class = TestItem
+
+
+class TestItemLoader(NameItemLoader):
+    name_in = MapCompose(lambda v: v.title())
+
+
+class DefaultedItemLoader(NameItemLoader):
+    default_input_processor = MapCompose(lambda v: v[:-1])
+
+
+# test processors
+def processor_with_args(value, other=None, loader_context=None):
+    if 'key' in loader_context:
+        return loader_context['key']
+    return value
+
+
+class BasicItemLoaderTest(unittest.TestCase):
+
+    def test_load_item_using_default_loader(self):
+        i = TestItem()
+        i['summary'] = u'lala'
+        il = ItemLoader(item=i)
+        il.add_value('name', u'marta')
+        item = il.load_item()
+        assert item is i
+        self.assertEqual(item['summary'], [u'lala'])
+        self.assertEqual(item['name'], [u'marta'])
+
+    def test_load_item_using_custom_loader(self):
+        il = TestItemLoader()
+        il.add_value('name', u'marta')
+        item = il.load_item()
+        self.assertEqual(item['name'], [u'Marta'])
+
+    def test_load_item_ignore_none_field_values(self):
+        def validate_sku(value):
+            # Let's assume a SKU is only digits.
+            if value.isdigit():
+                return value
+
+        class MyLoader(ItemLoader):
+            name_out = Compose(lambda vs: vs[0])  # take first which allows empty values
+            price_out = Compose(TakeFirst(), float)
+            sku_out = Compose(TakeFirst(), validate_sku)
+
+        valid_fragment = u'SKU: 1234'
+        invalid_fragment = u'SKU: not available'
+        sku_re = 'SKU: (.+)'
+
+        il = MyLoader(item={})
+        # Should not return "sku: None".
+        il.add_value('sku', [invalid_fragment], re=sku_re)
+        # Should not ignore empty values.
+        il.add_value('name', u'')
+        il.add_value('price', [u'0'])
+        self.assertEqual(il.load_item(), {
+            'name': u'',
+            'price': 0.0,
+        })
+
+        il.replace_value('sku', [valid_fragment], re=sku_re)
+        self.assertEqual(il.load_item()['sku'], u'1234')
+
+    def test_self_referencing_loader(self):
+        class MyLoader(ItemLoader):
+            url_out = TakeFirst()
+
+            def img_url_out(self, values):
+                return (self.get_output_value('url') or '') + values[0]
+
+        il = MyLoader(item={})
+        il.add_value('url', 'http://example.com/')
+        il.add_value('img_url', '1234.png')
+        self.assertEqual(il.load_item(), {
+            'url': 'http://example.com/',
+            'img_url': 'http://example.com/1234.png',
+        })
+
+        il = MyLoader(item={})
+        il.add_value('img_url', '1234.png')
+        self.assertEqual(il.load_item(), {
+            'img_url': '1234.png',
+        })
+
+    def test_add_value(self):
+        il = TestItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_collected_values('name'), [u'Marta'])
+        self.assertEqual(il.get_output_value('name'), [u'Marta'])
+        il.add_value('name', u'pepe')
+        self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe'])
+        self.assertEqual(il.get_output_value('name'), [u'Marta', u'Pepe'])
+
+        # test add object value
+        il.add_value('summary', {'key': 1})
+        self.assertEqual(il.get_collected_values('summary'), [{'key': 1}])
+
+        il.add_value(None, u'Jim', lambda x: {'name': x})
+        self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe', u'Jim'])
+
+    def test_add_zero(self):
+        il = NameItemLoader()
+        il.add_value('name', 0)
+        self.assertEqual(il.get_collected_values('name'), [0])
+
+    def test_replace_value(self):
+        il = TestItemLoader()
+        il.replace_value('name', u'marta')
+        self.assertEqual(il.get_collected_values('name'), [u'Marta'])
+        self.assertEqual(il.get_output_value('name'), [u'Marta'])
+        il.replace_value('name', u'pepe')
+        self.assertEqual(il.get_collected_values('name'), [u'Pepe'])
+        self.assertEqual(il.get_output_value('name'), [u'Pepe'])
+
+        il.replace_value(None, u'Jim', lambda x: {'name': x})
+        self.assertEqual(il.get_collected_values('name'), [u'Jim'])
+
+    def test_get_value(self):
+        il = NameItemLoader()
+        self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper))
+        self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$'))
+        self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$'))
+
+        il.add_value('name', [u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')
+        self.assertEqual([u'foo'], il.get_collected_values('name'))
+        il.replace_value('name', u'name:bar', re=u'name:(.*)$')
+        self.assertEqual([u'bar'], il.get_collected_values('name'))
+
+    def test_iter_on_input_processor_input(self):
+        class NameFirstItemLoader(NameItemLoader):
+            name_in = TakeFirst()
+
+        il = NameFirstItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+        il = NameFirstItemLoader()
+        il.add_value('name', [u'marta', u'jose'])
+        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+
+        il = NameFirstItemLoader()
+        il.replace_value('name', u'marta')
+        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+        il = NameFirstItemLoader()
+        il.replace_value('name', [u'marta', u'jose'])
+        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+
+        il = NameFirstItemLoader()
+        il.add_value('name', u'marta')
+        il.add_value('name', [u'jose', u'pedro'])
+        self.assertEqual(il.get_collected_values('name'), [u'marta', u'jose'])
+
+    def test_map_compose_filter(self):
+        def filter_world(x):
+            return None if x == 'world' else x
+
+        proc = MapCompose(filter_world, str.upper)
+        self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']),
+                         ['HELLO', 'THIS', 'IS', 'SCRAPY'])
+
+    def test_map_compose_filter_multil(self):
+        class TestItemLoader(NameItemLoader):
+            name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1])
+
+        il = TestItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'Mart'])
+        item = il.load_item()
+        self.assertEqual(item['name'], [u'Mart'])
+
+    def test_default_input_processor(self):
+        il = DefaultedItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'mart'])
+
+    def test_inherited_default_input_processor(self):
+        class InheritDefaultedItemLoader(DefaultedItemLoader):
+            pass
+
+        il = InheritDefaultedItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'mart'])
+
+    def test_input_processor_inheritance(self):
+        class ChildItemLoader(TestItemLoader):
+            url_in = MapCompose(lambda v: v.lower())
+
+        il = ChildItemLoader()
+        il.add_value('url', u'HTTP://scrapy.ORG')
+        self.assertEqual(il.get_output_value('url'), [u'http://scrapy.org'])
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'Marta'])
+
+        class ChildChildItemLoader(ChildItemLoader):
+            url_in = MapCompose(lambda v: v.upper())
+            summary_in = MapCompose(lambda v: v)
+
+        il = ChildChildItemLoader()
+        il.add_value('url', u'http://scrapy.org')
+        self.assertEqual(il.get_output_value('url'), [u'HTTP://SCRAPY.ORG'])
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'Marta'])
+
+    def test_empty_map_compose(self):
+        class IdentityDefaultedItemLoader(DefaultedItemLoader):
+            name_in = MapCompose()
+
+        il = IdentityDefaultedItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'marta'])
+
+    def test_identity_input_processor(self):
+        class IdentityDefaultedItemLoader(DefaultedItemLoader):
+            name_in = Identity()
+
+        il = IdentityDefaultedItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'marta'])
+
+    def test_extend_custom_input_processors(self):
+        class ChildItemLoader(TestItemLoader):
+            name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
+
+        il = ChildItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'mARTA'])
+
+    def test_extend_default_input_processors(self):
+        class ChildDefaultedItemLoader(DefaultedItemLoader):
+            name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase)
+
+        il = ChildDefaultedItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'MART'])
+
+    def test_output_processor_using_function(self):
+        il = TestItemLoader()
+        il.add_value('name', [u'mar', u'ta'])
+        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+
+        class TakeFirstItemLoader(TestItemLoader):
+            name_out = u" ".join
+
+        il = TakeFirstItemLoader()
+        il.add_value('name', [u'mar', u'ta'])
+        self.assertEqual(il.get_output_value('name'), u'Mar Ta')
+
+    def test_output_processor_error(self):
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+            name_out = MapCompose(float)
+
+        il = TestItemLoader()
+        il.add_value('name', [u'$10'])
+        try:
+            float(u'$10')
+        except Exception as e:
+            expected_exc_str = str(e)
+
+        exc = None
+        try:
+            il.load_item()
+        except Exception as e:
+            exc = e
+        assert isinstance(exc, ValueError)
+        s = str(exc)
+        assert 'name' in s, s
+        assert '$10' in s, s
+        assert 'ValueError' in s, s
+        assert expected_exc_str in s, s
+
+    def test_output_processor_using_classes(self):
+        il = TestItemLoader()
+        il.add_value('name', [u'mar', u'ta'])
+        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+
+        class TakeFirstItemLoader(TestItemLoader):
+            name_out = Join()
+
+        il = TakeFirstItemLoader()
+        il.add_value('name', [u'mar', u'ta'])
+        self.assertEqual(il.get_output_value('name'), u'Mar Ta')
+
+        class TakeFirstItemLoader(TestItemLoader):
+            name_out = Join("<br>")
+
+        il = TakeFirstItemLoader()
+        il.add_value('name', [u'mar', u'ta'])
+        self.assertEqual(il.get_output_value('name'), u'Mar<br>Ta')
+
+    def test_default_output_processor(self):
+        il = TestItemLoader()
+        il.add_value('name', [u'mar', u'ta'])
+        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+
+        class LalaItemLoader(TestItemLoader):
+            default_output_processor = Identity()
+
+        il = LalaItemLoader()
+        il.add_value('name', [u'mar', u'ta'])
+        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+
+    def test_loader_context_on_declaration(self):
+        class ChildItemLoader(TestItemLoader):
+            url_in = MapCompose(processor_with_args, key=u'val')
+
+        il = ChildItemLoader()
+        il.add_value('url', u'text')
+        self.assertEqual(il.get_output_value('url'), ['val'])
+        il.replace_value('url', u'text2')
+        self.assertEqual(il.get_output_value('url'), ['val'])
+
+    def test_loader_context_on_instantiation(self):
+        class ChildItemLoader(TestItemLoader):
+            url_in = MapCompose(processor_with_args)
+
+        il = ChildItemLoader(key=u'val')
+        il.add_value('url', u'text')
+        self.assertEqual(il.get_output_value('url'), ['val'])
+        il.replace_value('url', u'text2')
+        self.assertEqual(il.get_output_value('url'), ['val'])
+
+    def test_loader_context_on_assign(self):
+        class ChildItemLoader(TestItemLoader):
+            url_in = MapCompose(processor_with_args)
+
+        il = ChildItemLoader()
+        il.context['key'] = u'val'
+        il.add_value('url', u'text')
+        self.assertEqual(il.get_output_value('url'), ['val'])
+        il.replace_value('url', u'text2')
+        self.assertEqual(il.get_output_value('url'), ['val'])
+
+    def test_item_passed_to_input_processor_functions(self):
+        def processor(value, loader_context):
+            return loader_context['item']['name']
+
+        class ChildItemLoader(TestItemLoader):
+            url_in = MapCompose(processor)
+
+        it = TestItem(name='marta')
+        il = ChildItemLoader(item=it)
+        il.add_value('url', u'text')
+        self.assertEqual(il.get_output_value('url'), ['marta'])
+        il.replace_value('url', u'text2')
+        self.assertEqual(il.get_output_value('url'), ['marta'])
+
+    def test_compose_processor(self):
+        class TestItemLoader(NameItemLoader):
+            name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1])
+
+        il = TestItemLoader()
+        il.add_value('name', [u'marta', u'other'])
+        self.assertEqual(il.get_output_value('name'), u'Mart')
+        item = il.load_item()
+        self.assertEqual(item['name'], u'Mart')
+
+    def test_partial_processor(self):
+        def join(values, sep=None, loader_context=None, ignored=None):
+            if sep is not None:
+                return sep.join(values)
+            elif loader_context and 'sep' in loader_context:
+                return loader_context['sep'].join(values)
+            else:
+                return ''.join(values)
+
+        class TestItemLoader(NameItemLoader):
+            name_out = Compose(partial(join, sep='+'))
+            url_out = Compose(partial(join, loader_context={'sep': '.'}))
+            summary_out = Compose(partial(join, ignored='foo'))
+
+        il = TestItemLoader()
+        il.add_value('name', [u'rabbit', u'hole'])
+        il.add_value('url', [u'rabbit', u'hole'])
+        il.add_value('summary', [u'rabbit', u'hole'])
+        item = il.load_item()
+        self.assertEqual(item['name'], u'rabbit+hole')
+        self.assertEqual(item['url'], u'rabbit.hole')
+        self.assertEqual(item['summary'], u'rabbithole')
+
+    def test_error_input_processor(self):
+        class TestItem(Item):
+            name = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+            name_in = MapCompose(float)
+
+        il = TestItemLoader()
+        self.assertRaises(ValueError, il.add_value, 'name',
+                          [u'marta', u'other'])
+
+    def test_error_output_processor(self):
+        class TestItem(Item):
+            name = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+            name_out = Compose(Join(), float)
+
+        il = TestItemLoader()
+        il.add_value('name', u'marta')
+        with self.assertRaises(ValueError):
+            il.load_item()
+
+    def test_error_processor_as_argument(self):
+        class TestItem(Item):
+            name = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+
+        il = TestItemLoader()
+        self.assertRaises(ValueError, il.add_value, 'name',
+                          [u'marta', u'other'], Compose(float))
+
+
+class InitializationFromDictTest(unittest.TestCase):
+
+    item_class = dict
+
+    def test_keep_single_value(self):
+        """Loaded item should contain values from the initial item"""
+        input_item = self.item_class(name='foo')
+        il = ItemLoader(item=input_item)
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo']})
+
+    def test_keep_list(self):
+        """Loaded item should contain values from the initial item"""
+        input_item = self.item_class(name=['foo', 'bar'])
+        il = ItemLoader(item=input_item)
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']})
+
+    def test_add_value_singlevalue_singlevalue(self):
+        """Values added after initialization should be appended"""
+        input_item = self.item_class(name='foo')
+        il = ItemLoader(item=input_item)
+        il.add_value('name', 'bar')
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']})
+
+    def test_add_value_singlevalue_list(self):
+        """Values added after initialization should be appended"""
+        input_item = self.item_class(name='foo')
+        il = ItemLoader(item=input_item)
+        il.add_value('name', ['item', 'loader'])
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo', 'item', 'loader']})
+
+    def test_add_value_list_singlevalue(self):
+        """Values added after initialization should be appended"""
+        input_item = self.item_class(name=['foo', 'bar'])
+        il = ItemLoader(item=input_item)
+        il.add_value('name', 'qwerty')
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'qwerty']})
+
+    def test_add_value_list_list(self):
+        """Values added after initialization should be appended"""
+        input_item = self.item_class(name=['foo', 'bar'])
+        il = ItemLoader(item=input_item)
+        il.add_value('name', ['item', 'loader'])
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'item', 'loader']})
+
+    def test_get_output_value_singlevalue(self):
+        """Getting output value must not remove value from item"""
+        input_item = self.item_class(name='foo')
+        il = ItemLoader(item=input_item)
+        self.assertEqual(il.get_output_value('name'), ['foo'])
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(loaded_item, dict({'name': ['foo']}))
+
+    def test_get_output_value_list(self):
+        """Getting output value must not remove value from item"""
+        input_item = self.item_class(name=['foo', 'bar'])
+        il = ItemLoader(item=input_item)
+        self.assertEqual(il.get_output_value('name'), ['foo', 'bar'])
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(loaded_item, dict({'name': ['foo', 'bar']}))
+
+    def test_values_single(self):
+        """Values from initial item must be added to loader._values"""
+        input_item = self.item_class(name='foo')
+        il = ItemLoader(item=input_item)
+        self.assertEqual(il._values.get('name'), ['foo'])
+
+    def test_values_list(self):
+        """Values from initial item must be added to loader._values"""
+        input_item = self.item_class(name=['foo', 'bar'])
+        il = ItemLoader(item=input_item)
+        self.assertEqual(il._values.get('name'), ['foo', 'bar'])
+
+
+class BaseNoInputReprocessingLoader(ItemLoader):
+    title_in = MapCompose(str.upper)
+    title_out = TakeFirst()
+
+
+class NoInputReprocessingDictLoader(BaseNoInputReprocessingLoader):
+    default_item_class = dict
+
+
+class NoInputReprocessingFromDictTest(unittest.TestCase):
+    """
+    Loaders initialized from loaded items must not reprocess fields (dict instances)
+    """
+    def test_avoid_reprocessing_with_initial_values_single(self):
+        il = NoInputReprocessingDictLoader(item=dict(title='foo'))
+        il_loaded = il.load_item()
+        self.assertEqual(il_loaded, dict(title='foo'))
+        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo'))
+
+    def test_avoid_reprocessing_with_initial_values_list(self):
+        il = NoInputReprocessingDictLoader(item=dict(title=['foo', 'bar']))
+        il_loaded = il.load_item()
+        self.assertEqual(il_loaded, dict(title='foo'))
+        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo'))
+
+    def test_avoid_reprocessing_without_initial_values_single(self):
+        il = NoInputReprocessingDictLoader()
+        il.add_value('title', 'foo')
+        il_loaded = il.load_item()
+        self.assertEqual(il_loaded, dict(title='FOO'))
+        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO'))
+
+    def test_avoid_reprocessing_without_initial_values_list(self):
+        il = NoInputReprocessingDictLoader()
+        il.add_value('title', ['foo', 'bar'])
+        il_loaded = il.load_item()
+        self.assertEqual(il_loaded, dict(title='FOO'))
+        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO'))
+
+
+class TestOutputProcessorDict(unittest.TestCase):
+    def test_output_processor(self):
+
+        class TempDict(dict):
+            def __init__(self, *args, **kwargs):
+                super(TempDict, self).__init__(self, *args, **kwargs)
+                self.setdefault('temp', 0.3)
+
+        class TempLoader(ItemLoader):
+            default_item_class = TempDict
+            default_input_processor = Identity()
+            default_output_processor = Compose(TakeFirst())
+
+        loader = TempLoader()
+        item = loader.load_item()
+        self.assertIsInstance(item, TempDict)
+        self.assertEqual(dict(item), {'temp': 0.3})
+
+
+class ProcessorsTest(unittest.TestCase):
+
+    def test_take_first(self):
+        proc = TakeFirst()
+        self.assertEqual(proc([None, '', 'hello', 'world']), 'hello')
+        self.assertEqual(proc([None, '', 0, 'hello', 'world']), 0)
+
+    def test_identity(self):
+        proc = Identity()
+        self.assertEqual(proc([None, '', 'hello', 'world']),
+                         [None, '', 'hello', 'world'])
+
+    def test_join(self):
+        proc = Join()
+        self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
+        self.assertEqual(proc(['', 'hello', 'world']), u' hello world')
+        self.assertEqual(proc(['hello', 'world']), u'hello world')
+        self.assertIsInstance(proc(['hello', 'world']), str)
+
+    def test_compose(self):
+        proc = Compose(lambda v: v[0], str.upper)
+        self.assertEqual(proc(['hello', 'world']), 'HELLO')
+        proc = Compose(str.upper)
+        self.assertEqual(proc(None), None)
+        proc = Compose(str.upper, stop_on_none=False)
+        self.assertRaises(ValueError, proc, None)
+        proc = Compose(str.upper, lambda x: x + 1)
+        self.assertRaises(ValueError, proc, 'hello')
+
+    def test_mapcompose(self):
+        def filter_world(x):
+            return None if x == 'world' else x
+        proc = MapCompose(filter_world, str.upper)
+        self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
+                         [u'HELLO', u'THIS', u'IS', u'SCRAPY'])
+        proc = MapCompose(filter_world, str.upper)
+        self.assertEqual(proc(None), [])
+        proc = MapCompose(filter_world, str.upper)
+        self.assertRaises(ValueError, proc, [1])
+        proc = MapCompose(filter_world, lambda x: x + 1)
+        self.assertRaises(ValueError, proc, 'hello')
+
+
+class SelectJmesTestCase(unittest.TestCase):
+    test_list_equals = {
+        'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
+        'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None),
+        'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}),
+        'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
+        'dict': (
+            'foo.bar[*].name',
+            {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}},
+            ['one', 'two']
+        ),
+        'list': ('[1]', [1, 2], 2)
+    }
+
+    def test_output(self):
+        for tl in self.test_list_equals:
+            expr, test_list, expected = self.test_list_equals[tl]
+            test = SelectJmes(expr)(test_list)
+            self.assertEqual(
+                test,
+                expected,
+                msg='test "{}" got {} expected {}'.format(tl, test, expected)
+            )
+
+
+# Functions as processors
+
+def function_processor_strip(iterable):
+    return [x.strip() for x in iterable]
+
+
+def function_processor_upper(iterable):
+    return [x.upper() for x in iterable]
+
+
+class FunctionProcessorItem(Item):
+    foo = Field(
+        input_processor=function_processor_strip,
+        output_processor=function_processor_upper,
+    )
+
+
+class FunctionProcessorDictLoader(ItemLoader):
+    default_item_class = dict
+    foo_in = function_processor_strip
+    foo_out = function_processor_upper
+
+
+class FunctionProcessorTestCase(unittest.TestCase):
+
+    def test_processor_defined_in_item_loader(self):
+        lo = FunctionProcessorDictLoader()
+        lo.add_value('foo', '  bar  ')
+        lo.add_value('foo', ['  asdf  ', '  qwerty  '])
+        self.assertEqual(
+            dict(lo.load_item()),
+            {'foo': ['BAR', 'ASDF', 'QWERTY']}
+        )
+
+
+class DeprecatedUtilityFunctionsTestCase(unittest.TestCase):
+
+    def test_deprecated_wrap_loader_context(self):
+        def function(*args):
+            return None
+
+        with warnings.catch_warnings(record=True) as w:
+            wrap_loader_context(function, context=dict())
+
+            assert len(w) == 1
+            assert issubclass(w[0].category, ScrapyDeprecationWarning)
+
+    def test_deprecated_extract_regex(self):
+        with warnings.catch_warnings(record=True) as w:
+            extract_regex(r'\w+', 'this is a test')
+
+            assert len(w) == 1
+            assert issubclass(w[0].category, ScrapyDeprecationWarning)
+
+
+if __name__ == "__main__":
+    unittest.main()

From 8bdcdb0a76e3780681dcfbbd4a0eee62e2bb05b1 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 16 Jul 2020 09:13:54 +0300
Subject: [PATCH 219/243] Add quotes to example in docs

---
 docs/topics/feed-exports.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 0bb5f1733..7e91b365d 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -461,7 +461,7 @@ For instance, if your settings include::
 
 And your :command:`crawl` command line is::
 
-    scrapy crawl spidername -o dirname/%(batch_id)d-filename%(batch_time)s.json
+    scrapy crawl spidername -o 'dirname/%(batch_id)d-filename%(batch_time)s.json'
 
 The command line above can generate a directory tree like::
 

From 0e0d1ad64323033017c4a01893b5337a980cb5ec Mon Sep 17 00:00:00 2001
From: Marc <noviluni@gmail.com>
Date: Thu, 16 Jul 2020 14:19:46 +0200
Subject: [PATCH 220/243] remove python 2 reminiscence in cookies

---
 scrapy/http/cookies.py     | 3 ---
 tests/test_http_cookies.py | 3 ---
 2 files changed, 6 deletions(-)

diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 3e810992c..0c97e6999 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -186,9 +186,6 @@ class WrappedResponse:
     def info(self):
         return self
 
-    # python3 cookiejars calls get_all
     def get_all(self, name, default=None):
         return [to_unicode(v, errors='replace')
                 for v in self.response.headers.getlist(name)]
-    # python2 cookiejars calls getheaders
-    getheaders = get_all
diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py
index 45ddb42ba..540e27907 100644
--- a/tests/test_http_cookies.py
+++ b/tests/test_http_cookies.py
@@ -64,9 +64,6 @@ class WrappedResponseTest(TestCase):
     def test_info(self):
         self.assertIs(self.wrapped.info(), self.wrapped)
 
-    def test_getheaders(self):
-        self.assertEqual(self.wrapped.getheaders('content-type'), ['text/html'])
-
     def test_get_all(self):
         # get_all result must be native string
         self.assertEqual(self.wrapped.get_all('content-type'), ['text/html'])

From b97a39fda0b72d8a4dcd631599e5d9bf530a5bee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com>
Date: Thu, 16 Jul 2020 17:38:22 +0200
Subject: [PATCH 221/243] deprecate retry_on_eintr (#4683)

---
 scrapy/utils/python.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index afa8a8135..9204977cf 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -285,6 +285,7 @@ class WeakKeyCache:
         return self._weakdict[key]
 
 
+@deprecated
 def retry_on_eintr(function, *args, **kw):
     """Run a function and retry it while getting EINTR errors"""
     while True:

From 41263f61c6de8048023ba4c80e062f56b21e5a19 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 16 Jul 2020 18:41:45 +0300
Subject: [PATCH 222/243] Change single quotes to double in example in docs

---
 docs/topics/feed-exports.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 7e91b365d..fdc6e7cba 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -461,7 +461,7 @@ For instance, if your settings include::
 
 And your :command:`crawl` command line is::
 
-    scrapy crawl spidername -o 'dirname/%(batch_id)d-filename%(batch_time)s.json'
+    scrapy crawl spidername -o "dirname/%(batch_id)d-filename%(batch_time)s.json"
 
 The command line above can generate a directory tree like::
 

From d29bec60d795b13ecb6e5978cb9e4d8fbd298b08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 16 Jul 2020 23:19:24 +0200
Subject: [PATCH 223/243] Upgrade PyPy for CI, and test both 3.5 (oldest) and
 3.6 (newest) (#4504)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Upgrade PyPy for CI, and test both 3.5 (oldest) and 3.6 (newest)

* Log a detailed error message to discover why MockServer is not working

* Go for all lines!

* Disable tests based on mitmproxy while running on PyPy

* Fix test_get_func_args for PyPy 3.6+

* Make testPayloadDefaultCiphers work regardless of OpenSSL default ciphers

* Crossing fingers…

* Rename: testPayloadDefaultCiphers → testPayloadDisabledCipher

* Test the PyPy version currently documented as the minimum required version

* Fix the PYPY_VERSION tag

* Update the documentation about supported PyPy versions

* Also test the latest 3.5 Python version with PyPy

* Fix the PYPY_VERSION value for the latest 3.5 version

* Use pinned dependencies for asyncio and PyPy tests against oldest supported Python versions

* Fix PyPy installation for the pypy3-pinned Tox environment

* Try installing Cython

* Maybe PyPy requires lxml 3.6.0?

* install.rst: minor clarification

* lxml 4.0.0 is required on PyPy

* Require setuptools 18.5+

* Revert "Require setuptools 18.5+"

This reverts commit 017ec33ac2d237523cdd53be9be8169dd540759e.

* Maintain lxml as a dependency if setuptools < 18.5 is used
---
 .travis.yml                 | 16 +++++++++-----
 docs/faq.rst                | 14 ------------
 docs/intro/install.rst      | 12 +++++++---
 setup.py                    | 44 +++++++++++++++++++++++--------------
 tests/test_proxy_connect.py |  2 ++
 tests/test_utils_python.py  |  6 ++++-
 tests/test_webclient.py     |  6 +++--
 tox.ini                     | 31 +++++++++++++++++---------
 8 files changed, 80 insertions(+), 51 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index b403ac54c..db720b918 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,19 +18,25 @@ matrix:
     - env: TOXENV=typing
       python: 3.8
 
-    - env: TOXENV=pypy3
     - env: TOXENV=pinned
       python: 3.5.2
-    - env: TOXENV=asyncio
+    - env: TOXENV=asyncio-pinned
       python: 3.5.2  # We use additional code to support 3.5.3 and earlier
+    - env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0
+
     - env: TOXENV=py
       python: 3.5
     - env: TOXENV=asyncio
       python: 3.5  # We use specific code to support >= 3.5.4, < 3.6
+    - env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0
+
     - env: TOXENV=py
       python: 3.6
+    - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1
+
     - env: TOXENV=py
       python: 3.7
+
     - env: TOXENV=py PYPI_RELEASE_JOB=true
       python: 3.8
       dist: bionic
@@ -42,9 +48,9 @@ matrix:
       dist: bionic
 install:
   - |
-      if [ "$TOXENV" = "pypy3" ]; then
-        export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
-        wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
+      if [[ ! -z "$PYPY_VERSION" ]]; then
+        export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
+        wget "https://bitbucket.org/pypy/pypy/downloads/${PYPY_VERSION}.tar.bz2"
         tar -jxf ${PYPY_VERSION}.tar.bz2
         virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
         source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
diff --git a/docs/faq.rst b/docs/faq.rst
index d5ea3cb87..ea2c8216f 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -64,20 +64,6 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
 
 .. _BeautifulSoup's official documentation: https://www.crummy.com/software/BeautifulSoup/bs4/doc/#specifying-the-parser-to-use
 
-.. _faq-python-versions:
-
-What Python versions does Scrapy support?
------------------------------------------
-
-Scrapy is supported under Python 3.5.2+
-under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
-Python 3 support was added in Scrapy 1.1.
-PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
-Python 2 support was dropped in Scrapy 2.0.
-
-.. note::
-    For Python 3 support on Windows, it is recommended to use
-    Anaconda/Miniconda as :ref:`outlined in the installation guide <intro-install-windows>`.
 
 Did Scrapy "steal" X from Django?
 ---------------------------------
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index fb64d443c..6d65ae2ee 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -4,12 +4,18 @@
 Installation guide
 ==================
 
+.. _faq-python-versions:
+
+Supported Python versions
+=========================
+
+Scrapy requires Python 3.5.2+, either the CPython implementation (default) or
+the PyPy 5.9+ implementation (see :ref:`python:implementations`).
+
+
 Installing Scrapy
 =================
 
-Scrapy runs on Python 3.5.2 or above under CPython (default Python
-implementation) and PyPy (starting with PyPy 5.9).
-
 If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
 the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows
 and macOS.
diff --git a/setup.py b/setup.py
index f8d9b491b..58090f7a2 100644
--- a/setup.py
+++ b/setup.py
@@ -18,12 +18,39 @@ def has_environment_marker_platform_impl_support():
     return parse_version(setuptools_version) >= parse_version('18.5')
 
 
+install_requires = [
+    'Twisted>=17.9.0',
+    'cryptography>=2.0',
+    'cssselect>=0.9.1',
+    'itemloaders>=1.0.1',
+    'lxml>=3.5.0',
+    'parsel>=1.5.0',
+    'PyDispatcher>=2.0.5',
+    'pyOpenSSL>=16.2.0',
+    'queuelib>=1.4.2',
+    'service_identity>=16.0.0',
+    'w3lib>=1.17.0',
+    'zope.interface>=4.1.3',
+    'protego>=0.1.15',
+    'itemadapter>=0.1.0',
+]
 extras_require = {}
 
 if has_environment_marker_platform_impl_support():
+    extras_require[':platform_python_implementation == "CPython"'] = [
+        'lxml>=3.5.0',
+    ]
     extras_require[':platform_python_implementation == "PyPy"'] = [
+        # Earlier lxml versions are affected by
+        # https://bitbucket.org/pypy/pypy/issues/2498/cython-on-pypy-3-dict-object-has-no,
+        # which was fixed in Cython 0.26, released on 2017-06-19, and used to
+        # generate the C headers of lxml release tarballs published since then, the
+        # first of which was:
+        'lxml>=4.0.0',
         'PyPyDispatcher>=2.1.0',
     ]
+else:
+    install_requires.append('lxml>=3.5.0')
 
 
 setup(
@@ -67,21 +94,6 @@ setup(
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
     python_requires='>=3.5.2',
-    install_requires=[
-        'Twisted>=17.9.0',
-        'cryptography>=2.0',
-        'cssselect>=0.9.1',
-        'itemloaders>=1.0.1',
-        'lxml>=3.5.0',
-        'parsel>=1.5.0',
-        'PyDispatcher>=2.0.5',
-        'pyOpenSSL>=16.2.0',
-        'queuelib>=1.4.2',
-        'service_identity>=16.0.0',
-        'w3lib>=1.17.0',
-        'zope.interface>=4.1.3',
-        'protego>=0.1.15',
-        'itemadapter>=0.1.0',
-    ],
+    install_requires=install_requires,
     extras_require=extras_require,
 )
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index fc5658ae7..a56e3c39a 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -60,6 +60,8 @@ def _wrong_credentials(proxy_url):
 
 @skipIf(sys.version_info < (3, 5, 4),
         "requires mitmproxy < 3.0.0, which these tests do not support")
+@skipIf("pypy" in sys.executable,
+        "mitmproxy does not support PyPy")
 @skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7),
         "mitmproxy does not support Windows when running Python < 3.7")
 class ProxyConnectTestCase(TestCase):
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 65e6ba876..ebce3c079 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -4,6 +4,7 @@ import operator
 import platform
 import unittest
 from itertools import count
+from sys import version_info
 from warnings import catch_warnings
 
 from scrapy.utils.python import (
@@ -214,9 +215,12 @@ class UtilsPythonTestCase(unittest.TestCase):
         else:
             self.assertEqual(
                 get_func_args(str.split, stripself=True), ['sep', 'maxsplit'])
-            self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])
             self.assertEqual(
                 get_func_args(operator.itemgetter(2), stripself=True), ['obj'])
+            if version_info < (3, 6):
+                self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])
+            else:
+                self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable'])
 
     def test_without_none_values(self):
         self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index c1c5945c2..ee64d455c 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -413,7 +413,9 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
             self.getURL("payload"), body=s, contextFactory=client_context_factory
         ).addCallback(self.assertEqual, to_bytes(s))
 
-    def testPayloadDefaultCiphers(self):
+    def testPayloadDisabledCipher(self):
         s = "0123456789" * 10
-        d = getPage(self.getURL("payload"), body=s, contextFactory=ScrapyClientContextFactory())
+        settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'})
+        client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None)
+        d = getPage(self.getURL("payload"), body=s, contextFactory=client_context_factory)
         return self.assertFailure(d, OpenSSL.SSL.Error)
diff --git a/tox.ini b/tox.ini
index 5d79739bb..4557c63e3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -58,11 +58,6 @@ deps =
 commands =
     pylint conftest.py docs extras scrapy setup.py tests
 
-[testenv:pypy3]
-basepython = pypy3
-commands =
-    py.test {posargs:--durations=10 docs scrapy tests}
-
 [pinned]
 deps =
     -ctests/constraints.txt
@@ -85,7 +80,6 @@ deps =
     Pillow==3.4.2
 
 [testenv:pinned]
-basepython = python3
 deps =
     {[pinned]deps}
     lxml==3.5.0
@@ -104,6 +98,27 @@ deps =
     reppy
     robotexclusionrulesparser
 
+[testenv:asyncio]
+commands =
+    {[testenv]commands} --reactor=asyncio
+
+[testenv:asyncio-pinned]
+commands = {[testenv:asyncio]commands}
+deps = {[testenv:pinned]deps}
+
+[testenv:pypy3]
+basepython = pypy3
+commands =
+    py.test {posargs:--durations=10 docs scrapy tests}
+
+[testenv:pypy3-pinned]
+basepython = {[testenv:pypy3]basepython}
+commands = {[testenv:pypy3]commands}
+deps =
+    {[pinned]deps}
+    lxml==4.0.0
+    PyPyDispatcher==2.1.0
+
 [docs]
 changedir = docs
 deps =
@@ -135,7 +150,3 @@ deps = {[docs]deps}
 setenv = {[docs]setenv}
 commands =
     sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
-
-[testenv:asyncio]
-commands =
-    {[testenv]commands} --reactor=asyncio

From 62a4ede5e995f83abd5a90f7dd6ac242f2f3870d Mon Sep 17 00:00:00 2001
From: Artur Shellunts <shellunts.artur@gmail.com>
Date: Fri, 17 Jul 2020 12:40:04 +0200
Subject: [PATCH 224/243] Remove deprecated classes BaseSgmlLinkExtractor,
 RegexLinkExtractor and SgmlLinkExtractor (#4356)

---
 scrapy/linkextractors/regex.py                |  41 -----
 scrapy/linkextractors/sgml.py                 | 151 ------------------
 tests/ignores.txt                             |   2 -
 .../link_extractor/linkextractor.html         |   2 +-
 .../link_extractor/linkextractor_latin1.html  |   2 +-
 5 files changed, 2 insertions(+), 196 deletions(-)
 delete mode 100644 scrapy/linkextractors/regex.py
 delete mode 100644 scrapy/linkextractors/sgml.py

diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py
deleted file mode 100644
index 3f2557248..000000000
--- a/scrapy/linkextractors/regex.py
+++ /dev/null
@@ -1,41 +0,0 @@
-import re
-from urllib.parse import urljoin
-
-from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url
-
-from scrapy.link import Link
-from scrapy.linkextractors.sgml import SgmlLinkExtractor
-
-
-linkre = re.compile(
-        "<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",
-        re.DOTALL | re.IGNORECASE)
-
-
-def clean_link(link_text):
-    """Remove leading and trailing whitespace and punctuation"""
-    return link_text.strip("\t\r\n '\"\x0c")
-
-
-class RegexLinkExtractor(SgmlLinkExtractor):
-    """High performant link extractor"""
-
-    def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
-        def clean_text(text):
-            return replace_escape_chars(remove_tags(text.decode(response_encoding))).strip()
-
-        def clean_url(url):
-            clean_url = ''
-            try:
-                clean_url = urljoin(base_url, replace_entities(clean_link(url.decode(response_encoding))))
-            except ValueError:
-                pass
-            return clean_url
-
-        if base_url is None:
-            base_url = get_base_url(response_text, response_url, response_encoding)
-
-        links_text = linkre.findall(response_text)
-        return [Link(clean_url(url).encode(response_encoding),
-                     clean_text(text))
-                for url, _, text in links_text]
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
deleted file mode 100644
index 2ba6bca45..000000000
--- a/scrapy/linkextractors/sgml.py
+++ /dev/null
@@ -1,151 +0,0 @@
-"""
-SGMLParser-based Link extractors
-"""
-import warnings
-from urllib.parse import urljoin
-from sgmllib import SGMLParser
-
-from w3lib.url import safe_url_string, canonicalize_url
-from w3lib.html import strip_html5_whitespace
-
-from scrapy.link import Link
-from scrapy.linkextractors import FilteringLinkExtractor
-from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
-from scrapy.utils.python import unique as unique_list, to_unicode
-from scrapy.utils.response import get_base_url
-from scrapy.exceptions import ScrapyDeprecationWarning
-
-
-class BaseSgmlLinkExtractor(SGMLParser):
-
-    def __init__(self, tag="a", attr="href", unique=False, process_value=None,
-                 strip=True, canonicalized=False):
-        warnings.warn(
-            "BaseSgmlLinkExtractor is deprecated and will be removed in future releases. "
-            "Please use scrapy.linkextractors.LinkExtractor",
-            ScrapyDeprecationWarning, stacklevel=2,
-        )
-        SGMLParser.__init__(self)
-        self.scan_tag = tag if callable(tag) else lambda t: t == tag
-        self.scan_attr = attr if callable(attr) else lambda a: a == attr
-        self.process_value = (lambda v: v) if process_value is None else process_value
-        self.current_link = None
-        self.unique = unique
-        self.strip = strip
-        if canonicalized:
-            self.link_key = lambda link: link.url
-        else:
-            self.link_key = lambda link: canonicalize_url(link.url,
-                                                          keep_fragments=True)
-
-    def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
-        """ Do the real extraction work """
-        self.reset()
-        self.feed(response_text)
-        self.close()
-
-        ret = []
-        if base_url is None:
-            base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
-        for link in self.links:
-            if isinstance(link.url, str):
-                link.url = link.url.encode(response_encoding)
-            try:
-                link.url = urljoin(base_url, link.url)
-            except ValueError:
-                continue
-            link.url = safe_url_string(link.url, response_encoding)
-            link.text = to_unicode(link.text, response_encoding, errors='replace').strip()
-            ret.append(link)
-
-        return ret
-
-    def _process_links(self, links):
-        """ Normalize and filter extracted links
-
-        The subclass should override it if necessary
-        """
-        return unique_list(links, key=self.link_key) if self.unique else links
-
-    def extract_links(self, response):
-        # wrapper needed to allow to work directly with text
-        links = self._extract_links(response.body, response.url, response.encoding)
-        links = self._process_links(links)
-        return links
-
-    def reset(self):
-        SGMLParser.reset(self)
-        self.links = []
-        self.base_url = None
-        self.current_link = None
-
-    def unknown_starttag(self, tag, attrs):
-        if tag == 'base':
-            self.base_url = dict(attrs).get('href')
-        if self.scan_tag(tag):
-            for attr, value in attrs:
-                if self.scan_attr(attr):
-                    if self.strip and value is not None:
-                        value = strip_html5_whitespace(value)
-                    url = self.process_value(value)
-                    if url is not None:
-                        link = Link(url=url, nofollow=rel_has_nofollow(dict(attrs).get('rel')))
-                        self.links.append(link)
-                        self.current_link = link
-
-    def unknown_endtag(self, tag):
-        if self.scan_tag(tag):
-            self.current_link = None
-
-    def handle_data(self, data):
-        if self.current_link:
-            self.current_link.text = self.current_link.text + data
-
-    def matches(self, url):
-        """This extractor matches with any url, since
-        it doesn't contain any patterns"""
-        return True
-
-
-class SgmlLinkExtractor(FilteringLinkExtractor):
-
-    def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
-                 tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True,
-                 process_value=None, deny_extensions=None, restrict_css=(),
-                 strip=True, restrict_text=()):
-        warnings.warn(
-            "SgmlLinkExtractor is deprecated and will be removed in future releases. "
-            "Please use scrapy.linkextractors.LinkExtractor",
-            ScrapyDeprecationWarning, stacklevel=2,
-        )
-
-        tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
-        tag_func = lambda x: x in tags
-        attr_func = lambda x: x in attrs
-
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
-            lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,
-                                       unique=unique, process_value=process_value, strip=strip,
-                                       canonicalized=canonicalize)
-
-        super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
-                                                allow_domains=allow_domains, deny_domains=deny_domains,
-                                                restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
-                                                canonicalize=canonicalize, deny_extensions=deny_extensions,
-                                                restrict_text=restrict_text)
-
-    def extract_links(self, response):
-        base_url = None
-        if self.restrict_xpaths:
-            base_url = get_base_url(response)
-            body = u''.join(f
-                            for x in self.restrict_xpaths
-                            for f in response.xpath(x).getall()
-                            ).encode(response.encoding, errors='xmlcharrefreplace')
-        else:
-            body = response.body
-
-        links = self._extract_links(body, response.url, response.encoding, base_url)
-        links = self._process_links(links)
-        return links
diff --git a/tests/ignores.txt b/tests/ignores.txt
index f6e0d6fbe..222288841 100644
--- a/tests/ignores.txt
+++ b/tests/ignores.txt
@@ -1,5 +1,3 @@
-scrapy/linkextractors/sgml.py
-scrapy/linkextractors/regex.py
 scrapy/downloadermiddlewares/cookies.py
 scrapy/extensions/statsmailer.py
 scrapy/extensions/memusage.py
diff --git a/tests/sample_data/link_extractor/linkextractor.html b/tests/sample_data/link_extractor/linkextractor.html
index 7d5db368a..2307ea865 100644
--- a/tests/sample_data/link_extractor/linkextractor.html
+++ b/tests/sample_data/link_extractor/linkextractor.html
@@ -1,7 +1,7 @@
 <html>
 <head>
 <base href='http://example.com' />
-<title>Sample page with links for testing RegexLinkExtractor</title>
+<title>Sample page with links for testing LinkExtractor</title>
 </head>
 <body>
 <div id='wrapper'>
diff --git a/tests/sample_data/link_extractor/linkextractor_latin1.html b/tests/sample_data/link_extractor/linkextractor_latin1.html
index fc31d7e5d..e7eee18de 100644
--- a/tests/sample_data/link_extractor/linkextractor_latin1.html
+++ b/tests/sample_data/link_extractor/linkextractor_latin1.html
@@ -2,7 +2,7 @@
   <head>
     <meta http-equiv="Content-Type" content="text/html; charset=latin-1">
     <base href='http://example.com' />
-    <title>Sample page with links for testing RegexLinkExtractor</title>
+    <title>Sample page with links for testing LinkExtractor</title>
   </head>
 <body>
   <div id='wrapper'>

From 86f7ac2f2b5d58e0b2588fa2aa4c777a8decf299 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Fri, 17 Jul 2020 17:48:25 +0300
Subject: [PATCH 225/243] Try to fix error at Windows

---
 tests/test_feedexport.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 129b7fc0b..cc124624d 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1204,6 +1204,8 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
             for path, feed in FEEDS.items():
                 dir_name = os.path.dirname(path)
+                if not os.path.exists(str(dir_name)):
+                    continue
                 for file in sorted(os.listdir(dir_name)):
                     with open(os.path.join(dir_name, file), 'rb') as f:
                         data = f.read()

From 3e0492741d93b05c464457b3b128a2b0d24c994b Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Sun, 19 Jul 2020 00:10:29 +0300
Subject: [PATCH 226/243] Another try to fix test errors on Windows

---
 tests/test_feedexport.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index cc124624d..c49b2e92f 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1190,9 +1190,14 @@ class BatchDeliveriesTest(FeedExportTestBase):
     def run_and_export(self, spider_cls, settings):
         """ Run spider with specified settings; return exported data. """
 
+        def build_url(path):
+            if path[0] != '/':
+                path = '/' + path
+            return urljoin('file:', path)
+
         FEEDS = settings.get('FEEDS') or {}
         settings['FEEDS'] = {
-            urljoin('file:', file_path): feed
+            build_url(file_path): feed
             for file_path, feed in FEEDS.items()
         }
         content = defaultdict(list)
@@ -1204,8 +1209,6 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
             for path, feed in FEEDS.items():
                 dir_name = os.path.dirname(path)
-                if not os.path.exists(str(dir_name)):
-                    continue
                 for file in sorted(os.listdir(dir_name)):
                     with open(os.path.join(dir_name, file), 'rb') as f:
                         data = f.read()

From de297a3a167a6be3ac1ed94a891effefc12a6d00 Mon Sep 17 00:00:00 2001
From: Akshay Sharma <42249933+AKSHAYSHARMAJS@users.noreply.github.com>
Date: Mon, 20 Jul 2020 17:53:38 +0530
Subject: [PATCH 227/243] enable ANSI color (instead of ANSI color codes) in
 the Windows terminal #4393 (#4403)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* changed ie. -> i.e.(spelling error) on lines 667, 763 (issue scrapy#4332)

* updated all text files for issue #4332 (ie. -> i.e.)

* Apply ie. → i.e. in source comments

* ie → e.g.

* modified scrapy/utils/display.py to stop ANSI color sequences in the Windows terminal (issue #4393)

* modified scrapy/utils/display.py to stop ANSI color sequences in the Windows terminal (issue #4393)

* enabled virtual terminal processing (pr #4403)

* check for specific windows 10 version (pr #4403)

* fixing flake-8 test (pr #4403)

* added error handling for terminal info (pr #4403)

* corrected stderr (pr #4403)

* changed orientation, removed unwanted spaces (pr #4403)

* no need for style variable (pr #4403)

* fixing trailing whitespaces

* commenting windows check

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* small fixes

* Shifting _color_support_info() function

* enabled virtual terminal processing (pr #4403)

* check for specific windows 10 version (pr #4403)

* fixing flake-8 test (pr #4403)

* added error handling for terminal info (pr #4403)

* corrected stderr (pr #4403)

* changed orientation, removed unwanted spaces (pr #4403)

* no need for style variable (pr #4403)

* fixing trailing whitespaces

* commenting windows check

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* small fixes

* Shifting _color_support_info() function

* error handling

* error handlingy

* raise ValueError

* added in-built function for version comparison

* recommit changes

* changed check -> parse

* version comparison -> parse_version

* added scrapy/utils/display.py in pytest.ini

* Trigger

* Add simple test for scrapy.utils.display._colorize

* Flake8: E501 for tests/test_utils_display.py

* assertEquals -> assertEqual

* Normal formatter for all platforms

* separate test for windows

* all curses under try block

* added global TestStr

* more test added

* small fix

* covering exceptions

* windows test failing

* Refactor output color handling

* Fix pprint test

* fix flake8

Co-authored-by: Adrián Chaves <adrian@chaves.io>
Co-authored-by: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
---
 scrapy/utils/display.py     | 28 +++++++++++--
 tests/test_utils_display.py | 78 +++++++++++++++++++++++++++++++++++++
 2 files changed, 103 insertions(+), 3 deletions(-)
 create mode 100644 tests/test_utils_display.py

diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py
index 9735220ef..f4d17224b 100644
--- a/scrapy/utils/display.py
+++ b/scrapy/utils/display.py
@@ -2,20 +2,42 @@
 pprint and pformat wrappers with colorization support
 """
 
+import ctypes
+import platform
 import sys
+from distutils.version import LooseVersion as parse_version
 from pprint import pformat as pformat_
 
 
+def _enable_windows_terminal_processing():
+    # https://stackoverflow.com/a/36760881
+    kernel32 = ctypes.windll.kernel32
+    return bool(kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7))
+
+
+def _tty_supports_color():
+    if sys.platform != "win32":
+        return True
+
+    if parse_version(platform.version()) < parse_version("10.0.14393"):
+        return True
+
+    # Windows >= 10.0.14393 interprets ANSI escape sequences providing terminal
+    # processing is enabled.
+    return _enable_windows_terminal_processing()
+
+
 def _colorize(text, colorize=True):
-    if not colorize or not sys.stdout.isatty():
+    if not colorize or not sys.stdout.isatty() or not _tty_supports_color():
         return text
     try:
         from pygments import highlight
+    except ImportError:
+        return text
+    else:
         from pygments.formatters import TerminalFormatter
         from pygments.lexers import PythonLexer
         return highlight(text, PythonLexer(), TerminalFormatter())
-    except ImportError:
-        return text
 
 
 def pformat(obj, *args, **kwargs):
diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py
new file mode 100644
index 000000000..9ec8311d9
--- /dev/null
+++ b/tests/test_utils_display.py
@@ -0,0 +1,78 @@
+from io import StringIO
+
+from unittest import mock, TestCase
+
+from scrapy.utils.display import pformat, pprint
+
+
+class TestDisplay(TestCase):
+    object = {'a': 1}
+    colorized_string = (
+        "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'"
+        "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}\n"
+    )
+    plain_string = "{'a': 1}"
+
+    @mock.patch('sys.platform', 'linux')
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat(self, isatty):
+        isatty.return_value = True
+        self.assertEqual(pformat(self.object), self.colorized_string)
+
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat_dont_colorize(self, isatty):
+        isatty.return_value = True
+        self.assertEqual(pformat(self.object, colorize=False), self.plain_string)
+
+    def test_pformat_not_tty(self):
+        self.assertEqual(pformat(self.object), self.plain_string)
+
+    @mock.patch('sys.platform', 'win32')
+    @mock.patch('platform.version')
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat_old_windows(self, isatty, version):
+        isatty.return_value = True
+        version.return_value = '10.0.14392'
+        self.assertEqual(pformat(self.object), self.colorized_string)
+
+    @mock.patch('sys.platform', 'win32')
+    @mock.patch('scrapy.utils.display._enable_windows_terminal_processing')
+    @mock.patch('platform.version')
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat_windows_no_terminal_processing(self, isatty, version, terminal_processing):
+        isatty.return_value = True
+        version.return_value = '10.0.14393'
+        terminal_processing.return_value = False
+        self.assertEqual(pformat(self.object), self.plain_string)
+
+    @mock.patch('sys.platform', 'win32')
+    @mock.patch('scrapy.utils.display._enable_windows_terminal_processing')
+    @mock.patch('platform.version')
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat_windows(self, isatty, version, terminal_processing):
+        isatty.return_value = True
+        version.return_value = '10.0.14393'
+        terminal_processing.return_value = True
+        self.assertEqual(pformat(self.object), self.colorized_string)
+
+    @mock.patch('sys.platform', 'linux')
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat_no_pygments(self, isatty):
+        isatty.return_value = True
+
+        import builtins
+        real_import = builtins.__import__
+
+        def mock_import(name, globals, locals, fromlist, level):
+            if 'pygments' in name:
+                raise ImportError
+            return real_import(name, globals, locals, fromlist, level)
+
+        builtins.__import__ = mock_import
+        self.assertEqual(pformat(self.object), self.plain_string)
+        builtins.__import__ = real_import
+
+    def test_pprint(self):
+        with mock.patch('sys.stdout', new=StringIO()) as mock_out:
+            pprint(self.object)
+            self.assertEqual(mock_out.getvalue(), "{'a': 1}\n")

From 8fae3d5bb768ebb3f9674cc38172226f404fb297 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 22 Jul 2020 16:08:35 -0300
Subject: [PATCH 228/243] Remove monkeypatches module from mypy section in
 setup.cfg

---
 setup.cfg | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/setup.cfg b/setup.cfg
index 46a3d13fc..f8e7c0c91 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -13,9 +13,6 @@ follow_imports = skip
 [mypy-scrapy]
 ignore_errors = True
 
-[mypy-scrapy._monkeypatches]
-ignore_errors = True
-
 [mypy-scrapy.commands]
 ignore_errors = True
 

From a6c1d79b7cc3bc2c408eab356bbbf99a0536f110 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Tue, 28 Jul 2020 11:53:05 +0300
Subject: [PATCH 229/243] pep8 tiny changes

---
 docs/topics/feed-exports.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 2c9774b55..dd4eb3c61 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -453,6 +453,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 
 FEED_EXPORT_BATCH_ITEM_COUNT
 -----------------------------
+
 Default: ``0``
 
 If assigned an integer number higher than ``0``, Scrapy generates multiple output files
@@ -474,7 +475,7 @@ generated:
 
 For instance, if your settings include::
 
-    FEED_EXPORT_BATCH_ITEM_COUNT=100
+    FEED_EXPORT_BATCH_ITEM_COUNT = 100
 
 And your :command:`crawl` command line is::
 

From 52658539370c442e63102a3208781335953cdf53 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Tue, 28 Jul 2020 06:15:14 -0300
Subject: [PATCH 230/243] Use ItemAdapter.field_names when writing header in
 CsvItemExporter (#4668)

---
 scrapy/exporters.py     |   8 +-
 tests/test_exporters.py | 203 ++++++++++++++++++++++++++++------------
 2 files changed, 146 insertions(+), 65 deletions(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 712572673..0aba1c904 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -243,12 +243,8 @@ class CsvItemExporter(BaseItemExporter):
     def _write_headers_and_set_fields_to_export(self, item):
         if self.include_headers_line:
             if not self.fields_to_export:
-                if isinstance(item, dict):
-                    # for dicts try using fields of the first item
-                    self.fields_to_export = list(item.keys())
-                else:
-                    # use fields declared in Item
-                    self.fields_to_export = list(item.fields.keys())
+                # use declared field names, or keys if the item is a dict
+                self.fields_to_export = ItemAdapter(item).field_names()
             row = list(self._build_row(self.fields_to_export))
             self.csv_writer.writerow(row)
 
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index b27380309..25da54a65 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -8,6 +8,7 @@ from io import BytesIO
 from datetime import datetime
 
 import lxml.etree
+from itemadapter import ItemAdapter
 
 from scrapy.item import Item, Field
 from scrapy.utils.python import to_unicode
@@ -23,10 +24,37 @@ class TestItem(Item):
     age = Field()
 
 
+def custom_serializer(value):
+    return str(int(value) + 2)
+
+
+class CustomFieldItem(Item):
+    name = Field()
+    age = Field(serializer=custom_serializer)
+
+
+try:
+    from dataclasses import make_dataclass, field
+except ImportError:
+    TestDataClass = None
+    CustomFieldDataclass = None
+else:
+    TestDataClass = make_dataclass("TestDataClass", [("name", str), ("age", int)])
+    CustomFieldDataclass = make_dataclass(
+        "CustomFieldDataclass",
+        [("name", str), ("age", int, field(metadata={"serializer": custom_serializer}))]
+    )
+
+
 class BaseItemExporterTest(unittest.TestCase):
 
+    item_class = TestItem
+    custom_field_item_class = CustomFieldItem
+
     def setUp(self):
-        self.i = TestItem(name=u'John\xa3', age=u'22')
+        if self.item_class is None:
+            raise unittest.SkipTest("item class is None")
+        self.i = self.item_class(name=u'John\xa3', age=u'22')
         self.output = BytesIO()
         self.ie = self._get_exporter()
 
@@ -39,7 +67,7 @@ class BaseItemExporterTest(unittest.TestCase):
     def _assert_expected_item(self, exported_dict):
         for k, v in exported_dict.items():
             exported_dict[k] = to_unicode(v)
-        self.assertEqual(self.i, exported_dict)
+        self.assertEqual(self.i, self.item_class(**exported_dict))
 
     def _get_nonstring_types_item(self):
         return {
@@ -63,13 +91,14 @@ class BaseItemExporterTest(unittest.TestCase):
         self.assertItemExportWorks(self.i)
 
     def test_export_dict_item(self):
-        self.assertItemExportWorks(dict(self.i))
+        self.assertItemExportWorks(ItemAdapter(self.i).asdict())
 
     def test_serialize_field(self):
-        res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name'])
+        a = ItemAdapter(self.i)
+        res = self.ie.serialize_field(a.get_field_meta('name'), 'name', a['name'])
         self.assertEqual(res, u'John\xa3')
 
-        res = self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age'])
+        res = self.ie.serialize_field(a.get_field_meta('age'), 'age', a['age'])
         self.assertEqual(res, u'22')
 
     def test_fields_to_export(self):
@@ -82,18 +111,16 @@ class BaseItemExporterTest(unittest.TestCase):
         self.assertEqual(name, u'John\xa3')
 
     def test_field_custom_serializer(self):
-        def custom_serializer(value):
-            return str(int(value) + 2)
-
-        class CustomFieldItem(Item):
-            name = Field()
-            age = Field(serializer=custom_serializer)
-
-        i = CustomFieldItem(name=u'John\xa3', age=u'22')
-
+        i = self.custom_field_item_class(name=u'John\xa3', age=u'22')
+        a = ItemAdapter(i)
         ie = self._get_exporter()
-        self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), u'John\xa3')
-        self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
+        self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), u'John\xa3')
+        self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24')
+
+
+class BaseItemExporterDataclassTest(BaseItemExporterTest):
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
 
 
 class PythonItemExporterTest(BaseItemExporterTest):
@@ -105,9 +132,9 @@ class PythonItemExporterTest(BaseItemExporterTest):
             PythonItemExporter(invalid_option='something')
 
     def test_nested_item(self):
-        i1 = TestItem(name=u'Joseph', age='22')
+        i1 = self.item_class(name=u'Joseph', age='22')
         i2 = dict(name=u'Maria', age=i1)
-        i3 = TestItem(name=u'Jesus', age=i2)
+        i3 = self.item_class(name=u'Jesus', age=i2)
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(type(exported), dict)
@@ -119,9 +146,9 @@ class PythonItemExporterTest(BaseItemExporterTest):
         self.assertEqual(type(exported['age']['age']), dict)
 
     def test_export_list(self):
-        i1 = TestItem(name=u'Joseph', age='22')
-        i2 = TestItem(name=u'Maria', age=[i1])
-        i3 = TestItem(name=u'Jesus', age=[i2])
+        i1 = self.item_class(name=u'Joseph', age='22')
+        i2 = self.item_class(name=u'Maria', age=[i1])
+        i3 = self.item_class(name=u'Jesus', age=[i2])
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(
@@ -132,9 +159,9 @@ class PythonItemExporterTest(BaseItemExporterTest):
         self.assertEqual(type(exported['age'][0]['age'][0]), dict)
 
     def test_export_item_dict_list(self):
-        i1 = TestItem(name=u'Joseph', age='22')
+        i1 = self.item_class(name=u'Joseph', age='22')
         i2 = dict(name=u'Maria', age=[i1])
-        i3 = TestItem(name=u'Jesus', age=[i2])
+        i3 = self.item_class(name=u'Jesus', age=[i2])
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(
@@ -146,7 +173,7 @@ class PythonItemExporterTest(BaseItemExporterTest):
 
     def test_export_binary(self):
         exporter = PythonItemExporter(binary=True)
-        value = TestItem(name=u'John\xa3', age=u'22')
+        value = self.item_class(name=u'John\xa3', age=u'22')
         expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
         self.assertEqual(expected, exporter.export_item(value))
 
@@ -157,6 +184,11 @@ class PythonItemExporterTest(BaseItemExporterTest):
         self.assertEqual(exported, item)
 
 
+class PythonItemExporterDataclassTest(PythonItemExporterTest):
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
 class PprintItemExporterTest(BaseItemExporterTest):
 
     def _get_exporter(self, **kwargs):
@@ -166,6 +198,11 @@ class PprintItemExporterTest(BaseItemExporterTest):
         self._assert_expected_item(eval(self.output.getvalue()))
 
 
+class PprintItemExporterDataclassTest(PprintItemExporterTest):
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
 class PickleItemExporterTest(BaseItemExporterTest):
 
     def _get_exporter(self, **kwargs):
@@ -175,8 +212,8 @@ class PickleItemExporterTest(BaseItemExporterTest):
         self._assert_expected_item(pickle.loads(self.output.getvalue()))
 
     def test_export_multiple_items(self):
-        i1 = TestItem(name='hello', age='world')
-        i2 = TestItem(name='bye', age='world')
+        i1 = self.item_class(name='hello', age='world')
+        i2 = self.item_class(name='bye', age='world')
         f = BytesIO()
         ie = PickleItemExporter(f)
         ie.start_exporting()
@@ -184,8 +221,8 @@ class PickleItemExporterTest(BaseItemExporterTest):
         ie.export_item(i2)
         ie.finish_exporting()
         f.seek(0)
-        self.assertEqual(pickle.load(f), i1)
-        self.assertEqual(pickle.load(f), i2)
+        self.assertEqual(self.item_class(**pickle.load(f)), i1)
+        self.assertEqual(self.item_class(**pickle.load(f)), i2)
 
     def test_nonstring_types_item(self):
         item = self._get_nonstring_types_item()
@@ -197,6 +234,11 @@ class PickleItemExporterTest(BaseItemExporterTest):
         self.assertEqual(pickle.loads(fp.getvalue()), item)
 
 
+class PickleItemExporterDataclassTest(PickleItemExporterTest):
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
 class MarshalItemExporterTest(BaseItemExporterTest):
 
     def _get_exporter(self, **kwargs):
@@ -219,6 +261,11 @@ class MarshalItemExporterTest(BaseItemExporterTest):
         self.assertEqual(marshal.load(fp), item)
 
 
+class MarshalItemExporterDataclassTest(MarshalItemExporterTest):
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
 class CsvItemExporterTest(BaseItemExporterTest):
     def _get_exporter(self, **kwargs):
         return CsvItemExporter(self.output, **kwargs)
@@ -245,18 +292,18 @@ class CsvItemExporterTest(BaseItemExporterTest):
     def test_header_export_all(self):
         self.assertExportResult(
             item=self.i,
-            fields_to_export=self.i.fields.keys(),
+            fields_to_export=ItemAdapter(self.i).field_names(),
             expected=b'age,name\r\n22,John\xc2\xa3\r\n',
         )
 
     def test_header_export_all_dict(self):
         self.assertExportResult(
-            item=dict(self.i),
+            item=ItemAdapter(self.i).asdict(),
             expected=b'age,name\r\n22,John\xc2\xa3\r\n',
         )
 
     def test_header_export_single_field(self):
-        for item in [self.i, dict(self.i)]:
+        for item in [self.i, ItemAdapter(self.i).asdict()]:
             self.assertExportResult(
                 item=item,
                 fields_to_export=['age'],
@@ -264,7 +311,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
             )
 
     def test_header_export_two_items(self):
-        for item in [self.i, dict(self.i)]:
+        for item in [self.i, ItemAdapter(self.i).asdict()]:
             output = BytesIO()
             ie = CsvItemExporter(output)
             ie.start_exporting()
@@ -275,7 +322,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
                                 b'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n')
 
     def test_header_no_header_line(self):
-        for item in [self.i, dict(self.i)]:
+        for item in [self.i, ItemAdapter(self.i).asdict()]:
             self.assertExportResult(
                 item=item,
                 include_headers_line=False,
@@ -309,6 +356,11 @@ class CsvItemExporterTest(BaseItemExporterTest):
         )
 
 
+class CsvItemExporterDataclassTest(CsvItemExporterTest):
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
 class XmlItemExporterTest(BaseItemExporterTest):
 
     def _get_exporter(self, **kwargs):
@@ -318,8 +370,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
         def xmltuple(elem):
             children = list(elem.iterchildren())
             if children:
-                return [(child.tag, sorted(xmltuple(child)))
-                        for child in children]
+                return [(child.tag, sorted(xmltuple(child))) for child in children]
             else:
                 return [(elem.tag, [(elem.text, ())])]
 
@@ -345,17 +396,21 @@ class XmlItemExporterTest(BaseItemExporterTest):
 
     def test_multivalued_fields(self):
         self.assertExportResult(
-            TestItem(name=[u'John\xa3', u'Doe']),
-            (
-                b'<?xml version="1.0" encoding="utf-8"?>\n'
-                b'<items><item><name><value>John\xc2\xa3</value><value>Doe</value></name></item></items>'
-            )
+            self.item_class(name=[u'John\xa3', u'Doe'], age=[1, 2, 3]),
+            b"""<?xml version="1.0" encoding="utf-8"?>\n
+            <items>
+                <item>
+                    <name><value>John\xc2\xa3</value><value>Doe</value></name>
+                    <age><value>1</value><value>2</value><value>3</value></age>
+                </item>
+            </items>
+            """
         )
 
     def test_nested_item(self):
-        i1 = TestItem(name=u'foo\xa3hoo', age='22')
+        i1 = dict(name=u'foo\xa3hoo', age='22')
         i2 = dict(name=u'bar', age=i1)
-        i3 = TestItem(name=u'buz', age=i2)
+        i3 = self.item_class(name=u'buz', age=i2)
 
         self.assertExportResult(
             i3,
@@ -376,9 +431,9 @@ class XmlItemExporterTest(BaseItemExporterTest):
         )
 
     def test_nested_list_item(self):
-        i1 = TestItem(name=u'foo')
+        i1 = dict(name=u'foo')
         i2 = dict(name=u'bar', v2={"egg": ["spam"]})
-        i3 = TestItem(name=u'buz', age=[i1, i2])
+        i3 = self.item_class(name=u'buz', age=[i1, i2])
 
         self.assertExportResult(
             i3,
@@ -412,6 +467,12 @@ class XmlItemExporterTest(BaseItemExporterTest):
         )
 
 
+class XmlItemExporterDataclassTest(XmlItemExporterTest):
+
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
 class JsonLinesItemExporterTest(BaseItemExporterTest):
 
     _expected_nested = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}}
@@ -421,12 +482,12 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
 
     def _check_output(self):
         exported = json.loads(to_unicode(self.output.getvalue().strip()))
-        self.assertEqual(exported, dict(self.i))
+        self.assertEqual(exported, ItemAdapter(self.i).asdict())
 
     def test_nested_item(self):
-        i1 = TestItem(name=u'Joseph', age='22')
+        i1 = self.item_class(name=u'Joseph', age='22')
         i2 = dict(name=u'Maria', age=i1)
-        i3 = TestItem(name=u'Jesus', age=i2)
+        i3 = self.item_class(name=u'Jesus', age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
@@ -449,6 +510,12 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
         self.assertEqual(exported, item)
 
 
+class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest):
+
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
 class JsonItemExporterTest(JsonLinesItemExporterTest):
 
     _expected_nested = [JsonLinesItemExporterTest._expected_nested]
@@ -458,7 +525,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
 
     def _check_output(self):
         exported = json.loads(to_unicode(self.output.getvalue().strip()))
-        self.assertEqual(exported, [dict(self.i)])
+        self.assertEqual(exported, [ItemAdapter(self.i).asdict()])
 
     def assertTwoItemsExported(self, item):
         self.ie.start_exporting()
@@ -466,28 +533,28 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.ie.export_item(item)
         self.ie.finish_exporting()
         exported = json.loads(to_unicode(self.output.getvalue()))
-        self.assertEqual(exported, [dict(item), dict(item)])
+        self.assertEqual(exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()])
 
     def test_two_items(self):
         self.assertTwoItemsExported(self.i)
 
     def test_two_dict_items(self):
-        self.assertTwoItemsExported(dict(self.i))
+        self.assertTwoItemsExported(ItemAdapter(self.i).asdict())
 
     def test_nested_item(self):
-        i1 = TestItem(name=u'Joseph\xa3', age='22')
-        i2 = TestItem(name=u'Maria', age=i1)
-        i3 = TestItem(name=u'Jesus', age=i2)
+        i1 = self.item_class(name=u'Joseph\xa3', age='22')
+        i2 = self.item_class(name=u'Maria', age=i1)
+        i3 = self.item_class(name=u'Jesus', age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
         exported = json.loads(to_unicode(self.output.getvalue()))
-        expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': dict(i1)}}
+        expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}}
         self.assertEqual(exported, [expected])
 
     def test_nested_dict_item(self):
         i1 = dict(name=u'Joseph\xa3', age='22')
-        i2 = TestItem(name=u'Maria', age=i1)
+        i2 = self.item_class(name=u'Maria', age=i1)
         i3 = dict(name=u'Jesus', age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
@@ -506,7 +573,19 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.assertEqual(exported, [item])
 
 
-class CustomItemExporterTest(unittest.TestCase):
+class JsonItemExporterDataclassTest(JsonItemExporterTest):
+
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
+class CustomExporterItemTest(unittest.TestCase):
+
+    item_class = TestItem
+
+    def setUp(self):
+        if self.item_class is None:
+            raise unittest.SkipTest("item class is None")
 
     def test_exporter_custom_serializer(self):
         class CustomItemExporter(BaseItemExporter):
@@ -516,16 +595,22 @@ class CustomItemExporterTest(unittest.TestCase):
                 else:
                     return super(CustomItemExporter, self).serialize_field(field, name, value)
 
-        i = TestItem(name=u'John', age='22')
+        i = self.item_class(name=u'John', age='22')
+        a = ItemAdapter(i)
         ie = CustomItemExporter()
 
-        self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John')
-        self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '23')
+        self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John')
+        self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '23')
 
         i2 = {'name': u'John', 'age': '22'}
         self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John')
         self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23')
 
 
+class CustomExporterDataclassTest(CustomExporterItemTest):
+
+    item_class = TestDataClass
+
+
 if __name__ == '__main__':
     unittest.main()

From e7a58fe1573176415a9ca054428c53c1ca29931a Mon Sep 17 00:00:00 2001
From: Kshitij Sharma <ksharma@atlassian.com>
Date: Wed, 29 Jul 2020 10:16:18 +0530
Subject: [PATCH 231/243] Code cleanup scrapy.utils.python.WeakKeyCache #4684

---
 scrapy/utils/python.py     | 12 ------------
 tests/test_utils_python.py | 18 +-----------------
 2 files changed, 1 insertion(+), 29 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 9204977cf..7a393925e 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -273,18 +273,6 @@ def equal_attributes(obj1, obj2, attributes):
     return True
 
 
-class WeakKeyCache:
-
-    def __init__(self, default_factory):
-        self.default_factory = default_factory
-        self._weakdict = weakref.WeakKeyDictionary()
-
-    def __getitem__(self, key):
-        if key not in self._weakdict:
-            self._weakdict[key] = self.default_factory(key)
-        return self._weakdict[key]
-
-
 @deprecated
 def retry_on_eintr(function, *args, **kw):
     """Run a function and retry it while getting EINTR errors"""
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index ebce3c079..b23ae2e52 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -9,7 +9,7 @@ from warnings import catch_warnings
 
 from scrapy.utils.python import (
     memoizemethod_noargs, binary_is_text, equal_attributes,
-    WeakKeyCache, get_func_args, to_bytes, to_unicode,
+    get_func_args, to_bytes, to_unicode,
     without_none_values, MutableChain)
 
 
@@ -155,22 +155,6 @@ class UtilsPythonTestCase(unittest.TestCase):
         a.meta['z'] = 2
         self.assertFalse(equal_attributes(a, b, [compare_z, 'x']))
 
-    def test_weakkeycache(self):
-        class _Weakme:
-            pass
-
-        _values = count()
-        wk = WeakKeyCache(lambda k: next(_values))
-        k = _Weakme()
-        v = wk[k]
-        self.assertEqual(v, wk[k])
-        self.assertNotEqual(v, wk[_Weakme()])
-        self.assertEqual(v, wk[k])
-        del k
-        for _ in range(100):
-            if wk._weakdict:
-                gc.collect()
-        self.assertFalse(len(wk._weakdict))
 
     def test_get_func_args(self):
         def f1(a, b, c):

From 403bc7020a5e1ba2b59eced2cc5f4453c7650666 Mon Sep 17 00:00:00 2001
From: Kshitij Sharma <ksharma@atlassian.com>
Date: Wed, 29 Jul 2020 18:05:33 +0530
Subject: [PATCH 232/243] Code cleanup scrapy.utils.python.WeakKeyCache #4684
 and fixing ci alerts

---
 tests/test_utils_python.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index b23ae2e52..5a53d89e4 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -1,9 +1,7 @@
 import functools
-import gc
 import operator
 import platform
 import unittest
-from itertools import count
 from sys import version_info
 from warnings import catch_warnings
 
@@ -12,7 +10,6 @@ from scrapy.utils.python import (
     get_func_args, to_bytes, to_unicode,
     without_none_values, MutableChain)
 
-
 __doctests__ = ['scrapy.utils.python']
 
 
@@ -155,7 +152,6 @@ class UtilsPythonTestCase(unittest.TestCase):
         a.meta['z'] = 2
         self.assertFalse(equal_attributes(a, b, [compare_z, 'x']))
 
-
     def test_get_func_args(self):
         def f1(a, b, c):
             pass

From 49337bd2ae094d97d364948569f59b8211c8dbbe Mon Sep 17 00:00:00 2001
From: Kshitij Sharma <ksharma@atlassian.com>
Date: Thu, 30 Jul 2020 12:25:21 +0530
Subject: [PATCH 234/243] Code cleanup scrapy.utils.python.WeakKeyCache #4684
 and fixing ci alerts

---
 scrapy/utils/python.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 7a393925e..c8f921ff3 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -127,6 +127,7 @@ def re_rsearch(pattern, text, chunk_size=1024):
     In case the pattern wasn't found, None is returned, otherwise it returns a tuple containing
     the start position of the match, and the ending (regarding the entire text).
     """
+
     def _chunk_iter():
         offset = len(text)
         while True:
@@ -158,6 +159,7 @@ def memoizemethod_noargs(method):
         if self not in cache:
             cache[self] = method(self, *args, **kwargs)
         return cache[self]
+
     return new_method
 
 
@@ -273,6 +275,19 @@ def equal_attributes(obj1, obj2, attributes):
     return True
 
 
+@deprecated
+class WeakKeyCache:
+
+    def __init__(self, default_factory):
+        self.default_factory = default_factory
+        self._weakdict = weakref.WeakKeyDictionary()
+
+    def __getitem__(self, key):
+        if key not in self._weakdict:
+            self._weakdict[key] = self.default_factory(key)
+        return self._weakdict[key]
+
+
 @deprecated
 def retry_on_eintr(function, *args, **kw):
     """Run a function and retry it while getting EINTR errors"""

From 890b2138a605af2bfbf340a0d48d9d83c4cda53b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 30 Jul 2020 13:39:30 +0200
Subject: [PATCH 235/243] Remove the u prefix from strings

---
 docs/_ext/scrapydocs.py                      |   2 +-
 docs/topics/loaders.rst                      |   6 +-
 docs/topics/selectors.rst                    |   4 +-
 docs/utils/linkfix.py                        |   2 +-
 scrapy/http/request/form.py                  |   6 +-
 scrapy/linkextractors/lxmlhtml.py            |   2 +-
 scrapy/logformatter.py                       |   2 +-
 tests/test_cmdline/__init__.py               |   2 +-
 tests/test_downloader_handlers.py            |   6 +-
 tests/test_downloadermiddleware_cookies.py   |  16 +-
 tests/test_downloadermiddleware_httpproxy.py |   6 +-
 tests/test_downloadermiddleware_redirect.py  |   6 +-
 tests/test_downloadermiddleware_robotstxt.py |   4 +-
 tests/test_exporters.py                      |  84 +++----
 tests/test_feedexport.py                     |   8 +-
 tests/test_http_headers.py                   |   6 +-
 tests/test_http_request.py                   |  90 ++++----
 tests/test_http_response.py                  |  72 +++---
 tests/test_item.py                           |  42 ++--
 tests/test_linkextractors.py                 | 180 +++++++--------
 tests/test_loader.py                         | 124 +++++-----
 tests/test_loader_deprecated.py              | 226 +++++++++----------
 tests/test_logformatter.py                   |  12 +-
 tests/test_mail.py                           |   8 +-
 tests/test_responsetypes.py                  |  10 +-
 tests/test_robotstxt_interface.py            |   8 +-
 tests/test_selector.py                       |  32 +--
 tests/test_spider.py                         |  12 +-
 tests/test_utils_iterators.py                |  90 ++++----
 tests/test_utils_python.py                   |  16 +-
 tests/test_utils_reqser.py                   |   2 +-
 tests/test_utils_template.py                 |   4 +-
 32 files changed, 545 insertions(+), 545 deletions(-)

diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py
index 192123473..640660943 100644
--- a/docs/_ext/scrapydocs.py
+++ b/docs/_ext/scrapydocs.py
@@ -17,7 +17,7 @@ class SettingsListDirective(Directive):
 def is_setting_index(node):
     if node.tagname == 'index':
         # index entries for setting directives look like:
-        # [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')]
+        # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
         entry_type, info, refid = node['entries'][0][:3]
         return entry_type == 'pair' and info.endswith('; setting')
     return False
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index d0eeb4097..29d9c5805 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -237,10 +237,10 @@ metadata. Here is an example::
 
 >>> from scrapy.loader import ItemLoader
 >>> il = ItemLoader(item=Product())
->>> il.add_value('name', [u'Welcome to my', u'<strong>website</strong>'])
->>> il.add_value('price', [u'&euro;', u'<span>1000</span>'])
+>>> il.add_value('name', ['Welcome to my', '<strong>website</strong>'])
+>>> il.add_value('price', ['&euro;', '<span>1000</span>'])
 >>> il.load_item()
-{'name': u'Welcome to my website', 'price': u'1000'}
+{'name': 'Welcome to my website', 'price': '1000'}
 
 The precedence order, for both input and output processors, is as follows:
 
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index bb46ea80f..5014df6ac 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -734,7 +734,7 @@ The ``test()`` function, for example, can prove quite useful when XPath's
 Example selecting links in list item with a "class" attribute ending with a digit:
 
 >>> from scrapy import Selector
->>> doc = u"""
+>>> doc = """
 ... <div>
 ...     <ul>
 ...         <li class="item-0"><a href="link1.html">first item</a></li>
@@ -765,7 +765,7 @@ extracting text elements for example.
 Example extracting microdata (sample content taken from https://schema.org/Product)
 with groups of itemscopes and corresponding itemprops::
 
-    >>> doc = u"""
+    >>> doc = """
     ... <div itemscope itemtype="http://schema.org/Product">
     ...   <span itemprop="name">Kenmore White 17" Microwave</span>
     ...   <img src="kenmore-microwave-17in.jpg" alt='Kenmore 17" Microwave' />
diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py
index 9acfc3b23..95a3f17d5 100755
--- a/docs/utils/linkfix.py
+++ b/docs/utils/linkfix.py
@@ -23,7 +23,7 @@ def main():
     _contents = None
 
     # A regex that matches standard linkcheck output lines
-    line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
+    line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
 
     # Read lines from the linkcheck output file
     try:
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 0e6ceef0b..a260798ac 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -133,7 +133,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
                         '  not(re:test(., "^(?:checkbox|radio)$", "i")))]]',
                         namespaces={
                             "re": "http://exslt.org/regular-expressions"})
-    values = [(k, u'' if v is None else v)
+    values = [(k, '' if v is None else v)
               for k, v in (_value(e) for e in inputs)
               if k and k not in formdata_keys]
 
@@ -168,7 +168,7 @@ def _select_value(ele, n, v):
         # This is a workround to bug in lxml fixed 2.3.1
         # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139
         selected_options = ele.xpath('.//option[@selected]')
-        v = [(o.get('value') or o.text or u'').strip() for o in selected_options]
+        v = [(o.get('value') or o.text or '').strip() for o in selected_options]
     return n, v
 
 
@@ -205,7 +205,7 @@ def _get_clickable(clickdata, form):
 
     # We didn't find it, so now we build an XPath expression out of the other
     # arguments, because they can be used as such
-    xpath = u'.//*' + u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
+    xpath = './/*' + ''.join('[@%s="%s"]' % c for c in clickdata.items())
     el = form.xpath(xpath)
     if len(el) == 1:
         return (el[0].get('name'), el[0].get('value') or '')
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 1615d44d7..8b9f961ee 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -76,7 +76,7 @@ class LxmlParserLinkExtractor:
             url = safe_url_string(url, encoding=response_encoding)
             # to fix relative links after process_value
             url = urljoin(response_url, url)
-            link = Link(url, _collect_string_content(el) or u'',
+            link = Link(url, _collect_string_content(el) or '',
                         nofollow=rel_has_nofollow(el.get('rel')))
             links.append(link)
         return self._deduplicate_if_needed(links)
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 219145f13..0f9e6f1cb 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -44,7 +44,7 @@ class LogFormatter:
                 def dropped(self, item, exception, response, spider):
                     return {
                         'level': logging.INFO, # lowering the level from logging.WARNING
-                        'msg': u"Dropped: %(exception)s" + os.linesep + "%(item)s",
+                        'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s",
                         'args': {
                             'exception': exception,
                             'item': item,
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index da99a6be8..591075a98 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -59,7 +59,7 @@ class CmdlineTest(unittest.TestCase):
                                     'EXTENSIONS=' + json.dumps(EXTENSIONS))
         # XXX: There's gotta be a smarter way to do this...
         self.assertNotIn("...", settingsstr)
-        for char in ("'", "<", ">", 'u"'):
+        for char in ("'", "<", ">"):
             settingsstr = settingsstr.replace(char, '"')
         settingsdict = json.loads(settingsstr)
         self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys())
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 51deb20f4..57d4cdd6b 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1110,7 +1110,7 @@ class DataURITestCase(unittest.TestCase):
 
     def test_default_mediatype(self):
         def _test(response):
-            self.assertEqual(response.text, u'\u038e\u03a3\u038e')
+            self.assertEqual(response.text, '\u038e\u03a3\u038e')
             self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
             self.assertEqual(response.encoding, "iso-8859-7")
 
@@ -1119,7 +1119,7 @@ class DataURITestCase(unittest.TestCase):
 
     def test_text_charset(self):
         def _test(response):
-            self.assertEqual(response.text, u'\u038e\u03a3\u038e')
+            self.assertEqual(response.text, '\u038e\u03a3\u038e')
             self.assertEqual(response.body, b'\xbe\xd3\xbe')
             self.assertEqual(response.encoding, "iso-8859-7")
 
@@ -1128,7 +1128,7 @@ class DataURITestCase(unittest.TestCase):
 
     def test_mediatype_parameters(self):
         def _test(response):
-            self.assertEqual(response.text, u'\u038e\u03a3\u038e')
+            self.assertEqual(response.text, '\u038e\u03a3\u038e')
             self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
             self.assertEqual(response.encoding, "utf-8")
 
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 9ccc2110b..010577415 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -277,33 +277,33 @@ class CookiesMiddlewareTest(TestCase):
 
     def test_request_cookies_encoding(self):
         # 1) UTF8-encoded bytes
-        req1 = Request('http://example.org', cookies={'a': u'á'.encode('utf8')})
+        req1 = Request('http://example.org', cookies={'a': 'á'.encode('utf8')})
         assert self.mw.process_request(req1, self.spider) is None
         self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
 
         # 2) Non UTF8-encoded bytes
-        req2 = Request('http://example.org', cookies={'a': u'á'.encode('latin1')})
+        req2 = Request('http://example.org', cookies={'a': 'á'.encode('latin1')})
         assert self.mw.process_request(req2, self.spider) is None
         self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
 
-        # 3) Unicode string
-        req3 = Request('http://example.org', cookies={'a': u'á'})
+        # 3) String
+        req3 = Request('http://example.org', cookies={'a': 'á'})
         assert self.mw.process_request(req3, self.spider) is None
         self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
 
     def test_request_headers_cookie_encoding(self):
         # 1) UTF8-encoded bytes
-        req1 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('utf8')})
+        req1 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('utf8')})
         assert self.mw.process_request(req1, self.spider) is None
         self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
 
         # 2) Non UTF8-encoded bytes
-        req2 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('latin1')})
+        req2 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('latin1')})
         assert self.mw.process_request(req2, self.spider) is None
         self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
 
-        # 3) Unicode string
-        req3 = Request('http://example.org', headers={'Cookie': u'a=á'})
+        # 3) String
+        req3 = Request('http://example.org', headers={'Cookie': 'a=á'})
         assert self.mw.process_request(req3, self.spider) is None
         self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
 
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 9841d7a76..351631eb8 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -88,7 +88,7 @@ class TestHttpProxyMiddleware(TestCase):
 
     def test_proxy_auth_encoding(self):
         # utf-8 encoding
-        os.environ['http_proxy'] = u'https://m\u00E1n:pass@proxy:3128'
+        os.environ['http_proxy'] = 'https://m\u00E1n:pass@proxy:3128'
         mw = HttpProxyMiddleware(auth_encoding='utf-8')
         req = Request('http://scrapytest.org')
         assert mw.process_request(req, spider) is None
@@ -96,7 +96,7 @@ class TestHttpProxyMiddleware(TestCase):
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz')
 
         # proxy from request.meta
-        req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})
+        req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'})
         assert mw.process_request(req, spider) is None
         self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==')
@@ -109,7 +109,7 @@ class TestHttpProxyMiddleware(TestCase):
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=')
 
         # proxy from request.meta, latin-1 encoding
-        req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})
+        req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'})
         assert mw.process_request(req, spider) is None
         self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz')
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 919dbed23..131332131 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -184,7 +184,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
 
     def test_latin1_location(self):
         req = Request('http://scrapytest.org/first')
-        latin1_location = u'/ação'.encode('latin1')  # HTTP historically supports latin1
+        latin1_location = '/ação'.encode('latin1')  # HTTP historically supports latin1
         resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302)
         req_result = self.mw.process_response(req, resp, self.spider)
         perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o'
@@ -192,7 +192,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
 
     def test_utf8_location(self):
         req = Request('http://scrapytest.org/first')
-        utf8_location = u'/ação'.encode('utf-8')  # header using UTF-8 encoding
+        utf8_location = '/ação'.encode('utf-8')  # header using UTF-8 encoding
         resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302)
         req_result = self.mw.process_response(req, resp, self.spider)
         perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o'
@@ -207,7 +207,7 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
         self.mw = MetaRefreshMiddleware.from_crawler(crawler)
 
     def _body(self, interval=5, url='http://example.org/newpage'):
-        html = u"""<html><head><meta http-equiv="refresh" content="{0};url={1}"/></head></html>"""
+        html = """<html><head><meta http-equiv="refresh" content="{0};url={1}"/></head></html>"""
         return html.format(interval, url).encode('utf-8')
 
     def test_priority_adjust(self):
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index b9452a0e7..f9936baba 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -30,7 +30,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
     def _get_successful_crawler(self):
         crawler = self.crawler
         crawler.settings.set('ROBOTSTXT_OBEY', True)
-        ROBOTS = u"""
+        ROBOTS = """
 User-Agent: *
 Disallow: /admin/
 Disallow: /static/
@@ -56,7 +56,7 @@ Disallow: /some/randome/page.html
             self.assertIgnored(Request('http://site.local/admin/main'), middleware),
             self.assertIgnored(Request('http://site.local/static/'), middleware),
             self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware),
-            self.assertIgnored(Request(u'http://site.local/wiki/Käyttäjä:'), middleware)
+            self.assertIgnored(Request('http://site.local/wiki/Käyttäjä:'), middleware)
         ], fireOnOneErrback=True)
 
     def test_robotstxt_ready_parser(self):
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 25da54a65..660c99ce1 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -54,7 +54,7 @@ class BaseItemExporterTest(unittest.TestCase):
     def setUp(self):
         if self.item_class is None:
             raise unittest.SkipTest("item class is None")
-        self.i = self.item_class(name=u'John\xa3', age=u'22')
+        self.i = self.item_class(name='John\xa3', age='22')
         self.output = BytesIO()
         self.ie = self._get_exporter()
 
@@ -96,25 +96,25 @@ class BaseItemExporterTest(unittest.TestCase):
     def test_serialize_field(self):
         a = ItemAdapter(self.i)
         res = self.ie.serialize_field(a.get_field_meta('name'), 'name', a['name'])
-        self.assertEqual(res, u'John\xa3')
+        self.assertEqual(res, 'John\xa3')
 
         res = self.ie.serialize_field(a.get_field_meta('age'), 'age', a['age'])
-        self.assertEqual(res, u'22')
+        self.assertEqual(res, '22')
 
     def test_fields_to_export(self):
         ie = self._get_exporter(fields_to_export=['name'])
-        self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', u'John\xa3')])
+        self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xa3')])
 
         ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1')
         _, name = list(ie._get_serialized_fields(self.i))[0]
         assert isinstance(name, str)
-        self.assertEqual(name, u'John\xa3')
+        self.assertEqual(name, 'John\xa3')
 
     def test_field_custom_serializer(self):
-        i = self.custom_field_item_class(name=u'John\xa3', age=u'22')
+        i = self.custom_field_item_class(name='John\xa3', age='22')
         a = ItemAdapter(i)
         ie = self._get_exporter()
-        self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), u'John\xa3')
+        self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John\xa3')
         self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24')
 
 
@@ -132,48 +132,48 @@ class PythonItemExporterTest(BaseItemExporterTest):
             PythonItemExporter(invalid_option='something')
 
     def test_nested_item(self):
-        i1 = self.item_class(name=u'Joseph', age='22')
-        i2 = dict(name=u'Maria', age=i1)
-        i3 = self.item_class(name=u'Jesus', age=i2)
+        i1 = self.item_class(name='Joseph', age='22')
+        i2 = dict(name='Maria', age=i1)
+        i3 = self.item_class(name='Jesus', age=i2)
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(type(exported), dict)
         self.assertEqual(
             exported,
-            {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'}
+            {'age': {'age': {'age': '22', 'name': 'Joseph'}, 'name': 'Maria'}, 'name': 'Jesus'}
         )
         self.assertEqual(type(exported['age']), dict)
         self.assertEqual(type(exported['age']['age']), dict)
 
     def test_export_list(self):
-        i1 = self.item_class(name=u'Joseph', age='22')
-        i2 = self.item_class(name=u'Maria', age=[i1])
-        i3 = self.item_class(name=u'Jesus', age=[i2])
+        i1 = self.item_class(name='Joseph', age='22')
+        i2 = self.item_class(name='Maria', age=[i1])
+        i3 = self.item_class(name='Jesus', age=[i2])
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(
             exported,
-            {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}
+            {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'}
         )
         self.assertEqual(type(exported['age'][0]), dict)
         self.assertEqual(type(exported['age'][0]['age'][0]), dict)
 
     def test_export_item_dict_list(self):
-        i1 = self.item_class(name=u'Joseph', age='22')
-        i2 = dict(name=u'Maria', age=[i1])
-        i3 = self.item_class(name=u'Jesus', age=[i2])
+        i1 = self.item_class(name='Joseph', age='22')
+        i2 = dict(name='Maria', age=[i1])
+        i3 = self.item_class(name='Jesus', age=[i2])
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(
             exported,
-            {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}
+            {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'}
         )
         self.assertEqual(type(exported['age'][0]), dict)
         self.assertEqual(type(exported['age'][0]['age'][0]), dict)
 
     def test_export_binary(self):
         exporter = PythonItemExporter(binary=True)
-        value = self.item_class(name=u'John\xa3', age=u'22')
+        value = self.item_class(name='John\xa3', age='22')
         expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
         self.assertEqual(expected, exporter.export_item(value))
 
@@ -279,7 +279,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
         return self.assertEqual(split_csv(first), split_csv(second), msg=msg)
 
     def _check_output(self):
-        self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n')
+        self.assertCsvEqual(to_unicode(self.output.getvalue()), 'age,name\r\n22,John\xa3\r\n')
 
     def assertExportResult(self, item, expected, **kwargs):
         fp = BytesIO()
@@ -396,7 +396,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
 
     def test_multivalued_fields(self):
         self.assertExportResult(
-            self.item_class(name=[u'John\xa3', u'Doe'], age=[1, 2, 3]),
+            self.item_class(name=['John\xa3', 'Doe'], age=[1, 2, 3]),
             b"""<?xml version="1.0" encoding="utf-8"?>\n
             <items>
                 <item>
@@ -408,9 +408,9 @@ class XmlItemExporterTest(BaseItemExporterTest):
         )
 
     def test_nested_item(self):
-        i1 = dict(name=u'foo\xa3hoo', age='22')
-        i2 = dict(name=u'bar', age=i1)
-        i3 = self.item_class(name=u'buz', age=i2)
+        i1 = dict(name='foo\xa3hoo', age='22')
+        i2 = dict(name='bar', age=i1)
+        i3 = self.item_class(name='buz', age=i2)
 
         self.assertExportResult(
             i3,
@@ -431,9 +431,9 @@ class XmlItemExporterTest(BaseItemExporterTest):
         )
 
     def test_nested_list_item(self):
-        i1 = dict(name=u'foo')
-        i2 = dict(name=u'bar', v2={"egg": ["spam"]})
-        i3 = self.item_class(name=u'buz', age=[i1, i2])
+        i1 = dict(name='foo')
+        i2 = dict(name='bar', v2={"egg": ["spam"]})
+        i3 = self.item_class(name='buz', age=[i1, i2])
 
         self.assertExportResult(
             i3,
@@ -475,7 +475,7 @@ class XmlItemExporterDataclassTest(XmlItemExporterTest):
 
 class JsonLinesItemExporterTest(BaseItemExporterTest):
 
-    _expected_nested = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}}
+    _expected_nested = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}}
 
     def _get_exporter(self, **kwargs):
         return JsonLinesItemExporter(self.output, **kwargs)
@@ -485,9 +485,9 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
         self.assertEqual(exported, ItemAdapter(self.i).asdict())
 
     def test_nested_item(self):
-        i1 = self.item_class(name=u'Joseph', age='22')
-        i2 = dict(name=u'Maria', age=i1)
-        i3 = self.item_class(name=u'Jesus', age=i2)
+        i1 = self.item_class(name='Joseph', age='22')
+        i2 = dict(name='Maria', age=i1)
+        i3 = self.item_class(name='Jesus', age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
@@ -542,25 +542,25 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.assertTwoItemsExported(ItemAdapter(self.i).asdict())
 
     def test_nested_item(self):
-        i1 = self.item_class(name=u'Joseph\xa3', age='22')
-        i2 = self.item_class(name=u'Maria', age=i1)
-        i3 = self.item_class(name=u'Jesus', age=i2)
+        i1 = self.item_class(name='Joseph\xa3', age='22')
+        i2 = self.item_class(name='Maria', age=i1)
+        i3 = self.item_class(name='Jesus', age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
         exported = json.loads(to_unicode(self.output.getvalue()))
-        expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}}
+        expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}}
         self.assertEqual(exported, [expected])
 
     def test_nested_dict_item(self):
-        i1 = dict(name=u'Joseph\xa3', age='22')
-        i2 = self.item_class(name=u'Maria', age=i1)
-        i3 = dict(name=u'Jesus', age=i2)
+        i1 = dict(name='Joseph\xa3', age='22')
+        i2 = self.item_class(name='Maria', age=i1)
+        i3 = dict(name='Jesus', age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
         exported = json.loads(to_unicode(self.output.getvalue()))
-        expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}}
+        expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': i1}}
         self.assertEqual(exported, [expected])
 
     def test_nonstring_types_item(self):
@@ -595,14 +595,14 @@ class CustomExporterItemTest(unittest.TestCase):
                 else:
                     return super(CustomItemExporter, self).serialize_field(field, name, value)
 
-        i = self.item_class(name=u'John', age='22')
+        i = self.item_class(name='John', age='22')
         a = ItemAdapter(i)
         ie = CustomItemExporter()
 
         self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John')
         self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '23')
 
-        i2 = {'name': u'John', 'age': '22'}
+        i2 = {'name': 'John', 'age': '22'}
         self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John')
         self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23')
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index c49b2e92f..b57349848 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -874,7 +874,7 @@ class FeedExportTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_export_encoding(self):
-        items = [dict({'foo': u'Test\xd6'})]
+        items = [dict({'foo': 'Test\xd6'})]
 
         formats = {
             'json': '[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
@@ -919,7 +919,7 @@ class FeedExportTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_export_multiple_configs(self):
-        items = [dict({'foo': u'FOO', 'bar': u'BAR'})]
+        items = [dict({'foo': 'FOO', 'bar': 'BAR'})]
 
         formats = {
             'json': '[\n{"bar": "BAR"}\n]'.encode('utf-8'),
@@ -1393,7 +1393,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_export_multiple_configs(self):
-        items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]
+        items = [dict({'foo': 'FOO', 'bar': 'BAR'}), dict({'foo': 'FOO1', 'bar': 'BAR1'})]
 
         formats = {
             'json': ['[\n{"bar": "BAR"}\n]'.encode('utf-8'),
@@ -1442,7 +1442,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_batch_item_count_feeds_setting(self):
-        items = [dict({'foo': u'FOO'}), dict({'foo': u'FOO1'})]
+        items = [dict({'foo': 'FOO'}), dict({'foo': 'FOO1'})]
         formats = {
             'json': ['[{"foo": "FOO"}]'.encode('utf-8'),
                      '[{"foo": "FOO1"}]'.encode('utf-8')],
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index cf3fc8496..64ff7a73d 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -39,19 +39,19 @@ class HeadersTest(unittest.TestCase):
         assert h.getlist('X-Forwarded-For') is not hlist
 
     def test_encode_utf8(self):
-        h = Headers({u'key': u'\xa3'}, encoding='utf-8')
+        h = Headers({'key': '\xa3'}, encoding='utf-8')
         key, val = dict(h).popitem()
         assert isinstance(key, bytes), key
         assert isinstance(val[0], bytes), val[0]
         self.assertEqual(val[0], b'\xc2\xa3')
 
     def test_encode_latin1(self):
-        h = Headers({u'key': u'\xa3'}, encoding='latin1')
+        h = Headers({'key': '\xa3'}, encoding='latin1')
         key, val = dict(h).popitem()
         self.assertEqual(val[0], b'\xa3')
 
     def test_encode_multiple(self):
-        h = Headers({u'key': [u'\xa3']}, encoding='utf-8')
+        h = Headers({'key': ['\xa3']}, encoding='utf-8')
         key, val = dict(h).popitem()
         self.assertEqual(val[0], b'\xc2\xa3')
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 63014b22d..f5cf4e798 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -60,8 +60,8 @@ class RequestTest(unittest.TestCase):
         self.assertFalse(p.headers is r.headers)
 
         # headers must not be unicode
-        h = Headers({'key1': u'val1', u'key2': 'val2'})
-        h[u'newkey'] = u'newval'
+        h = Headers({'key1': 'val1', 'key2': 'val2'})
+        h['newkey'] = 'newval'
         for k, v in h.items():
             self.assertIsInstance(k, bytes)
             for s in v:
@@ -89,30 +89,30 @@ class RequestTest(unittest.TestCase):
         self.assertEqual(r.url, "http://www.scrapy.org/blank%20space")
 
     def test_url_encoding(self):
-        r = self.request_class(url=u"http://www.scrapy.org/price/£")
+        r = self.request_class(url="http://www.scrapy.org/price/£")
         self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
 
     def test_url_encoding_other(self):
         # encoding affects only query part of URI, not path
         # path part should always be UTF-8 encoded before percent-escaping
-        r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="utf-8")
+        r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8")
         self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
 
-        r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="latin1")
+        r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1")
         self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
 
     def test_url_encoding_query(self):
-        r1 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ")
+        r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ")
         self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5")
 
         # should be same as above
-        r2 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ", encoding="utf-8")
+        r2 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8")
         self.assertEqual(r2.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5")
 
     def test_url_encoding_query_latin1(self):
         # encoding is used for encoding query-string before percent-escaping;
         # path is still UTF-8 encoded before percent-escaping
-        r3 = self.request_class(url=u"http://www.scrapy.org/price/µ?currency=£", encoding="latin1")
+        r3 = self.request_class(url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1")
         self.assertEqual(r3.url, "http://www.scrapy.org/price/%C2%B5?currency=%A3")
 
     def test_url_encoding_nonutf8_untouched(self):
@@ -131,16 +131,16 @@ class RequestTest(unittest.TestCase):
         # characters.  Otherwise, in the future the IRI will be mapped to
         # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different
         # URI from "http://www.example.org/r%E9sum%E9.html".
-        r1 = self.request_class(url=u"http://www.scrapy.org/price/%a3")
+        r1 = self.request_class(url="http://www.scrapy.org/price/%a3")
         self.assertEqual(r1.url, "http://www.scrapy.org/price/%a3")
 
-        r2 = self.request_class(url=u"http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
+        r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
         self.assertEqual(r2.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
 
-        r3 = self.request_class(url=u"http://www.scrapy.org/résumé/%a3")
+        r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3")
         self.assertEqual(r3.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
 
-        r4 = self.request_class(url=u"http://www.example.org/r%E9sum%E9.html")
+        r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html")
         self.assertEqual(r4.url, "http://www.example.org/r%E9sum%E9.html")
 
     def test_body(self):
@@ -151,11 +151,11 @@ class RequestTest(unittest.TestCase):
         assert isinstance(r2.body, bytes)
         self.assertEqual(r2.encoding, 'utf-8')  # default encoding
 
-        r3 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8')
+        r3 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='utf-8')
         assert isinstance(r3.body, bytes)
         self.assertEqual(r3.body, b"Price: \xc2\xa3100")
 
-        r4 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='latin1')
+        r4 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='latin1')
         assert isinstance(r4.body, bytes)
         self.assertEqual(r4.body, b"Price: \xa3100")
 
@@ -164,7 +164,7 @@ class RequestTest(unittest.TestCase):
         r = self.request_class(url="http://www.example.com/ajax.html#!key=value")
         self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue")
         # unicode url
-        r = self.request_class(url=u"http://www.example.com/ajax.html#!key=value")
+        r = self.request_class(url="http://www.example.com/ajax.html#!key=value")
         self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue")
 
     def test_copy(self):
@@ -236,7 +236,7 @@ class RequestTest(unittest.TestCase):
         assert r4.dont_filter is False
 
     def test_method_always_str(self):
-        r = self.request_class("http://www.example.com", method=u"POST")
+        r = self.request_class("http://www.example.com", method="POST")
         assert isinstance(r.method, str)
 
     def test_immutable_attributes(self):
@@ -381,7 +381,7 @@ class FormRequestTest(RequestTest):
 
     def test_default_encoding_textual_data(self):
         # using default encoding (utf-8)
-        data = {u'µ one': u'two', u'price': u'£ 100'}
+        data = {'µ one': 'two', 'price': '£ 100'}
         r2 = self.request_class("http://www.example.com", formdata=data)
         self.assertEqual(r2.method, 'POST')
         self.assertEqual(r2.encoding, 'utf-8')
@@ -390,7 +390,7 @@ class FormRequestTest(RequestTest):
 
     def test_default_encoding_mixed_data(self):
         # using default encoding (utf-8)
-        data = {u'\u00b5one': b'two', b'price\xc2\xa3': u'\u00a3 100'}
+        data = {'\u00b5one': b'two', b'price\xc2\xa3': '\u00a3 100'}
         r2 = self.request_class("http://www.example.com", formdata=data)
         self.assertEqual(r2.method, 'POST')
         self.assertEqual(r2.encoding, 'utf-8')
@@ -406,14 +406,14 @@ class FormRequestTest(RequestTest):
         self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded')
 
     def test_custom_encoding_textual_data(self):
-        data = {'price': u'£ 100'}
+        data = {'price': '£ 100'}
         r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
         self.assertEqual(r3.encoding, 'latin1')
         self.assertEqual(r3.body, b'price=%A3+100')
 
     def test_multi_key_values(self):
         # using multiples values for a single key
-        data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']}
+        data = {'price': '\xa3 100', 'colours': ['red', 'blue', 'green']}
         r3 = self.request_class("http://www.example.com", formdata=data)
         self.assertQueryEqual(r3.body, b'colours=red&colours=blue&colours=green&price=%C2%A3+100')
 
@@ -450,10 +450,10 @@ class FormRequestTest(RequestTest):
         self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
         self.assertEqual(req.url, "http://www.example.com/this/post.php")
         fs = _qs(req, to_unicode=True)
-        self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})
-        self.assertEqual(set(fs[u'one']), {u'two', u'three'})
-        self.assertEqual(fs[u'test2'], [u'xxx µ'])
-        self.assertEqual(fs[u'six'], [u'seven'])
+        self.assertEqual(set(fs['test £']), {'val1', 'val2'})
+        self.assertEqual(set(fs['one']), {'two', 'three'})
+        self.assertEqual(fs['test2'], ['xxx µ'])
+        self.assertEqual(fs['six'], ['seven'])
 
     def test_from_response_post_nonascii_bytes_latin1(self):
         response = _buildresponse(
@@ -471,14 +471,14 @@ class FormRequestTest(RequestTest):
         self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
         self.assertEqual(req.url, "http://www.example.com/this/post.php")
         fs = _qs(req, to_unicode=True, encoding='latin1')
-        self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})
-        self.assertEqual(set(fs[u'one']), {u'two', u'three'})
-        self.assertEqual(fs[u'test2'], [u'xxx µ'])
-        self.assertEqual(fs[u'six'], [u'seven'])
+        self.assertEqual(set(fs['test £']), {'val1', 'val2'})
+        self.assertEqual(set(fs['one']), {'two', 'three'})
+        self.assertEqual(fs['test2'], ['xxx µ'])
+        self.assertEqual(fs['six'], ['seven'])
 
     def test_from_response_post_nonascii_unicode(self):
         response = _buildresponse(
-            u"""<form action="post.php" method="POST">
+            """<form action="post.php" method="POST">
             <input type="hidden" name="test £" value="val1">
             <input type="hidden" name="test £" value="val2">
             <input type="hidden" name="test2" value="xxx µ">
@@ -490,10 +490,10 @@ class FormRequestTest(RequestTest):
         self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
         self.assertEqual(req.url, "http://www.example.com/this/post.php")
         fs = _qs(req, to_unicode=True)
-        self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})
-        self.assertEqual(set(fs[u'one']), {u'two', u'three'})
-        self.assertEqual(fs[u'test2'], [u'xxx µ'])
-        self.assertEqual(fs[u'six'], [u'seven'])
+        self.assertEqual(set(fs['test £']), {'val1', 'val2'})
+        self.assertEqual(set(fs['one']), {'two', 'three'})
+        self.assertEqual(fs['test2'], ['xxx µ'])
+        self.assertEqual(fs['six'], ['seven'])
 
     def test_from_response_duplicate_form_key(self):
         response = _buildresponse(
@@ -685,7 +685,7 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="two" value="clicked2">
             </form>""")
         req = self.request_class.from_response(
-            response, clickdata={u'name': u'clickable', u'value': u'clicked2'}
+            response, clickdata={'name': 'clickable', 'value': 'clicked2'}
         )
         fs = _qs(req)
         self.assertEqual(fs[b'clickable'], [b'clicked2'])
@@ -694,21 +694,21 @@ class FormRequestTest(RequestTest):
 
     def test_from_response_unicode_clickdata(self):
         response = _buildresponse(
-            u"""<form action="get.php" method="GET">
+            """<form action="get.php" method="GET">
             <input type="submit" name="price in \u00a3" value="\u00a3 1000">
             <input type="submit" name="price in \u20ac" value="\u20ac 2000">
             <input type="hidden" name="poundsign" value="\u00a3">
             <input type="hidden" name="eurosign" value="\u20ac">
             </form>""")
         req = self.request_class.from_response(
-            response, clickdata={u'name': u'price in \u00a3'}
+            response, clickdata={'name': 'price in \u00a3'}
         )
         fs = _qs(req, to_unicode=True)
-        self.assertTrue(fs[u'price in \u00a3'])
+        self.assertTrue(fs['price in \u00a3'])
 
     def test_from_response_unicode_clickdata_latin1(self):
         response = _buildresponse(
-            u"""<form action="get.php" method="GET">
+            """<form action="get.php" method="GET">
             <input type="submit" name="price in \u00a3" value="\u00a3 1000">
             <input type="submit" name="price in \u00a5" value="\u00a5 2000">
             <input type="hidden" name="poundsign" value="\u00a3">
@@ -716,10 +716,10 @@ class FormRequestTest(RequestTest):
             </form>""",
             encoding='latin1')
         req = self.request_class.from_response(
-            response, clickdata={u'name': u'price in \u00a5'}
+            response, clickdata={'name': 'price in \u00a5'}
         )
         fs = _qs(req, to_unicode=True, encoding='latin1')
-        self.assertTrue(fs[u'price in \u00a5'])
+        self.assertTrue(fs['price in \u00a5'])
 
     def test_from_response_multiple_forms_clickdata(self):
         response = _buildresponse(
@@ -733,7 +733,7 @@ class FormRequestTest(RequestTest):
             </form>
             """)
         req = self.request_class.from_response(
-            response, formname='form2', clickdata={u'name': u'clickable'}
+            response, formname='form2', clickdata={'name': 'clickable'}
         )
         fs = _qs(req)
         self.assertEqual(fs[b'clickable'], [b'clicked2'])
@@ -1072,11 +1072,11 @@ class FormRequestTest(RequestTest):
 
     def test_from_response_unicode_xpath(self):
         response = _buildresponse(b'<form name="\xd1\x8a"></form>')
-        r = self.request_class.from_response(response, formxpath=u"//form[@name='\u044a']")
+        r = self.request_class.from_response(response, formxpath="//form[@name='\u044a']")
         fs = _qs(r)
         self.assertEqual(fs, {})
 
-        xpath = u"//form[@name='\u03b1']"
+        xpath = "//form[@name='\u03b1']"
         self.assertRaisesRegex(ValueError, re.escape(xpath),
                                self.request_class.from_response,
                                response, formxpath=xpath)
@@ -1246,13 +1246,13 @@ class XmlRpcRequestTest(RequestTest):
         self._test_request(params=('value',))
         self._test_request(params=('username', 'password'), methodname='login')
         self._test_request(params=('response', ), methodresponse='login')
-        self._test_request(params=(u'pas£',), encoding='utf-8')
+        self._test_request(params=('pas£',), encoding='utf-8')
         self._test_request(params=(None,), allow_none=1)
         self.assertRaises(TypeError, self._test_request)
         self.assertRaises(TypeError, self._test_request, params=(None,))
 
     def test_latin1(self):
-        self._test_request(params=(u'pas£',), encoding='latin1')
+        self._test_request(params=('pas£',), encoding='latin1')
 
 
 class JsonRequestTest(RequestTest):
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index e0ca3c0e6..56d017de6 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -318,28 +318,28 @@ class TextResponseTest(BaseResponseTest):
 
     def test_unicode_url(self):
         # instantiate with unicode url without encoding (should set default encoding)
-        resp = self.response_class(u"http://www.example.com/")
+        resp = self.response_class("http://www.example.com/")
         self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING)
 
         # make sure urls are converted to str
-        resp = self.response_class(url=u"http://www.example.com/", encoding='utf-8')
+        resp = self.response_class(url="http://www.example.com/", encoding='utf-8')
         assert isinstance(resp.url, str)
 
-        resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8')
+        resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='utf-8')
         self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
-        resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1')
+        resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='latin-1')
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
-        resp = self.response_class(u"http://www.example.com/price/\xa3",
+        resp = self.response_class("http://www.example.com/price/\xa3",
                                    headers={"Content-type": ["text/html; charset=utf-8"]})
         self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
-        resp = self.response_class(u"http://www.example.com/price/\xa3",
+        resp = self.response_class("http://www.example.com/price/\xa3",
                                    headers={"Content-type": ["text/html; charset=iso-8859-1"]})
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
 
     def test_unicode_body(self):
         unicode_string = ('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 '
                           '\u0442\u0435\u043a\u0441\u0442')
-        self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body=u'unicode body')
+        self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body='unicode body')
 
         original_string = unicode_string.encode('cp1251')
         r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251')
@@ -355,7 +355,7 @@ class TextResponseTest(BaseResponseTest):
     def test_encoding(self):
         r1 = self.response_class("http://www.example.com", body=b"\xc2\xa3",
                                  headers={"Content-type": ["text/html; charset=utf-8"]})
-        r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3")
+        r2 = self.response_class("http://www.example.com", encoding='utf-8', body="\xa3")
         r3 = self.response_class("http://www.example.com", body=b"\xa3",
                                  headers={"Content-type": ["text/html; charset=iso-8859-1"]})
         r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3")
@@ -376,14 +376,14 @@ class TextResponseTest(BaseResponseTest):
         self.assertEqual(r5._headers_encoding(), None)
         self._assert_response_encoding(r5, "utf-8")
         assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii'
-        self._assert_response_values(r1, 'utf-8', u"\xa3")
-        self._assert_response_values(r2, 'utf-8', u"\xa3")
-        self._assert_response_values(r3, 'iso-8859-1', u"\xa3")
-        self._assert_response_values(r6, 'gb18030', u"\u2015")
-        self._assert_response_values(r7, 'gb18030', u"\u2015")
+        self._assert_response_values(r1, 'utf-8', "\xa3")
+        self._assert_response_values(r2, 'utf-8', "\xa3")
+        self._assert_response_values(r3, 'iso-8859-1', "\xa3")
+        self._assert_response_values(r6, 'gb18030', "\u2015")
+        self._assert_response_values(r7, 'gb18030', "\u2015")
 
         # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies
-        self.assertRaises(TypeError, self.response_class, "http://www.example.com", body=u"\xa3")
+        self.assertRaises(TypeError, self.response_class, "http://www.example.com", body="\xa3")
 
     def test_declared_encoding_invalid(self):
         """Check that unknown declared encodings are ignored"""
@@ -391,14 +391,14 @@ class TextResponseTest(BaseResponseTest):
                                 headers={"Content-type": ["text/html; charset=UKNOWN"]},
                                 body=b"\xc2\xa3")
         self.assertEqual(r._declared_encoding(), None)
-        self._assert_response_values(r, 'utf-8', u"\xa3")
+        self._assert_response_values(r, 'utf-8', "\xa3")
 
     def test_utf16(self):
         """Test utf-16 because UnicodeDammit is known to have problems with"""
         r = self.response_class("http://www.example.com",
                                 body=b'\xff\xfeh\x00i\x00',
                                 encoding='utf-16')
-        self._assert_response_values(r, 'utf-16', u"hi")
+        self._assert_response_values(r, 'utf-16', "hi")
 
     def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self):
         r6 = self.response_class("http://www.example.com",
@@ -406,8 +406,8 @@ class TextResponseTest(BaseResponseTest):
                                  body=b"\xef\xbb\xbfWORD\xe3\xab")
         self.assertEqual(r6.encoding, 'utf-8')
         self.assertIn(r6.text, {
-            u'WORD\ufffd\ufffd',  # w3lib < 1.19.0
-            u'WORD\ufffd',        # w3lib >= 1.19.0
+            'WORD\ufffd\ufffd',  # w3lib < 1.19.0
+            'WORD\ufffd',        # w3lib >= 1.19.0
         })
 
     def test_bom_is_removed_from_body(self):
@@ -422,9 +422,9 @@ class TextResponseTest(BaseResponseTest):
         # Test response without content-type and BOM encoding
         response = self.response_class(url, body=body)
         self.assertEqual(response.encoding, 'utf-8')
-        self.assertEqual(response.text, u'WORD')
+        self.assertEqual(response.text, 'WORD')
         response = self.response_class(url, body=body)
-        self.assertEqual(response.text, u'WORD')
+        self.assertEqual(response.text, 'WORD')
         self.assertEqual(response.encoding, 'utf-8')
 
         # Body caching sideeffect isn't triggered when encoding is declared in
@@ -432,28 +432,28 @@ class TextResponseTest(BaseResponseTest):
         # body
         response = self.response_class(url, headers=headers, body=body)
         self.assertEqual(response.encoding, 'utf-8')
-        self.assertEqual(response.text, u'WORD')
+        self.assertEqual(response.text, 'WORD')
         response = self.response_class(url, headers=headers, body=body)
-        self.assertEqual(response.text, u'WORD')
+        self.assertEqual(response.text, 'WORD')
         self.assertEqual(response.encoding, 'utf-8')
 
     def test_replace_wrong_encoding(self):
         """Test invalid chars are replaced properly"""
         r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX')
         # XXX: Policy for replacing invalid chars may suffer minor variations
-        # but it should always contain the unicode replacement char (u'\ufffd')
-        assert u'\ufffd' in r.text, repr(r.text)
-        assert u'PREFIX' in r.text, repr(r.text)
-        assert u'SUFFIX' in r.text, repr(r.text)
+        # but it should always contain the unicode replacement char ('\ufffd')
+        assert '\ufffd' in r.text, repr(r.text)
+        assert 'PREFIX' in r.text, repr(r.text)
+        assert 'SUFFIX' in r.text, repr(r.text)
 
         # Do not destroy html tags due to encoding bugs
         r = self.response_class("http://example.com", encoding='utf-8',
                                 body=b'\xf0<span>value</span>')
-        assert u'<span>value</span>' in r.text, repr(r.text)
+        assert '<span>value</span>' in r.text, repr(r.text)
 
         # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse
         # r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX')
-        # assert u'\ufffd' in r.text, repr(r.text)
+        # assert '\ufffd' in r.text, repr(r.text)
 
     def test_selector(self):
         body = b"<html><head><title>Some page</title><body></body></html>"
@@ -466,15 +466,15 @@ class TextResponseTest(BaseResponseTest):
 
         self.assertEqual(
             response.selector.xpath("//title/text()").getall(),
-            [u'Some page']
+            ['Some page']
         )
         self.assertEqual(
             response.selector.css("title::text").getall(),
-            [u'Some page']
+            ['Some page']
         )
         self.assertEqual(
             response.selector.re("Some (.*)</title>"),
-            [u'page']
+            ['page']
         )
 
     def test_selector_shortcuts(self):
@@ -595,7 +595,7 @@ class TextResponseTest(BaseResponseTest):
         resp1 = self.response_class(
             'http://example.com',
             encoding='utf8',
-            body=u'<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8')
+            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8')
         )
         req = self._assert_followed_url(
             resp1.css('a')[0],
@@ -607,7 +607,7 @@ class TextResponseTest(BaseResponseTest):
         resp2 = self.response_class(
             'http://example.com',
             encoding='cp1251',
-            body=u'<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251')
+            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251')
         )
         req = self._assert_followed_url(
             resp2.css('a')[0],
@@ -681,8 +681,8 @@ class TextResponseTest(BaseResponseTest):
 
     def test_body_as_unicode_deprecation_warning(self):
         with catch_warnings(record=True) as warnings:
-            r1 = self.response_class("http://www.example.com", body=u'Hello', encoding='utf-8')
-            self.assertEqual(r1.body_as_unicode(), u'Hello')
+            r1 = self.response_class("http://www.example.com", body='Hello', encoding='utf-8')
+            self.assertEqual(r1.body_as_unicode(), 'Hello')
             self.assertEqual(len(warnings), 1)
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
 
@@ -787,7 +787,7 @@ class XmlResponseTest(TextResponseTest):
 
         self.assertEqual(
             response.selector.xpath("//elem/text()").getall(),
-            [u'value']
+            ['value']
         )
 
     def test_selector_shortcuts(self):
diff --git a/tests/test_item.py b/tests/test_item.py
index 60468971c..0ce78f8c0 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -20,8 +20,8 @@ class ItemTest(unittest.TestCase):
             name = Field()
 
         i = TestItem()
-        i['name'] = u'name'
-        self.assertEqual(i['name'], u'name')
+        i['name'] = 'name'
+        self.assertEqual(i['name'], 'name')
 
     def test_init(self):
         class TestItem(Item):
@@ -30,17 +30,17 @@ class ItemTest(unittest.TestCase):
         i = TestItem()
         self.assertRaises(KeyError, i.__getitem__, 'name')
 
-        i2 = TestItem(name=u'john doe')
-        self.assertEqual(i2['name'], u'john doe')
+        i2 = TestItem(name='john doe')
+        self.assertEqual(i2['name'], 'john doe')
 
-        i3 = TestItem({'name': u'john doe'})
-        self.assertEqual(i3['name'], u'john doe')
+        i3 = TestItem({'name': 'john doe'})
+        self.assertEqual(i3['name'], 'john doe')
 
         i4 = TestItem(i3)
-        self.assertEqual(i4['name'], u'john doe')
+        self.assertEqual(i4['name'], 'john doe')
 
-        self.assertRaises(KeyError, TestItem, {'name': u'john doe',
-                                               'other': u'foo'})
+        self.assertRaises(KeyError, TestItem, {'name': 'john doe',
+                                               'other': 'foo'})
 
     def test_invalid_field(self):
         class TestItem(Item):
@@ -56,7 +56,7 @@ class ItemTest(unittest.TestCase):
             number = Field()
 
         i = TestItem()
-        i['name'] = u'John Doe'
+        i['name'] = 'John Doe'
         i['number'] = 123
         itemrepr = repr(i)
 
@@ -101,9 +101,9 @@ class ItemTest(unittest.TestCase):
 
         i = TestItem()
         self.assertRaises(KeyError, i.get_name)
-        i['name'] = u'lala'
-        self.assertEqual(i.get_name(), u'lala')
-        i.change_name(u'other')
+        i['name'] = 'lala'
+        self.assertEqual(i.get_name(), 'lala')
+        i.change_name('other')
         self.assertEqual(i.get_name(), 'other')
 
     def test_metaclass(self):
@@ -113,22 +113,22 @@ class ItemTest(unittest.TestCase):
             values = Field()
 
         i = TestItem()
-        i['name'] = u'John'
+        i['name'] = 'John'
         self.assertEqual(list(i.keys()), ['name'])
         self.assertEqual(list(i.values()), ['John'])
 
-        i['keys'] = u'Keys'
-        i['values'] = u'Values'
+        i['keys'] = 'Keys'
+        i['values'] = 'Values'
         self.assertSortedEqual(list(i.keys()), ['keys', 'values', 'name'])
-        self.assertSortedEqual(list(i.values()), [u'Keys', u'Values', u'John'])
+        self.assertSortedEqual(list(i.values()), ['Keys', 'Values', 'John'])
 
     def test_metaclass_with_fields_attribute(self):
         class TestItem(Item):
             fields = {'new': Field(default='X')}
 
-        item = TestItem(new=u'New')
+        item = TestItem(new='New')
         self.assertSortedEqual(list(item.keys()), ['new'])
-        self.assertSortedEqual(list(item.values()), [u'New'])
+        self.assertSortedEqual(list(item.values()), ['New'])
 
     def test_metaclass_inheritance(self):
         class ParentItem(Item):
@@ -238,8 +238,8 @@ class ItemTest(unittest.TestCase):
             name = Field()
 
         i = TestItem()
-        i['name'] = u'John'
-        self.assertEqual(dict(i), {'name': u'John'})
+        i['name'] = 'John'
+        self.assertEqual(dict(i), {'name': 'John'})
 
     def test_copy(self):
         class TestItem(Item):
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 8d4538eed..a0bafa5e5 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -31,31 +31,31 @@ class Base:
             page4_url = 'http://example.com/page%204.html'
 
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
                 Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
-                Link(url='http://www.google.com/something', text=u''),
-                Link(url='http://example.com/innertag.html', text=u'inner tag'),
-                Link(url=page4_url, text=u'href with whitespaces'),
+                Link(url='http://www.google.com/something', text=''),
+                Link(url='http://example.com/innertag.html', text='inner tag'),
+                Link(url=page4_url, text='href with whitespaces'),
             ])
 
         def test_extract_filter_allow(self):
             lx = self.extractor_cls(allow=('sample', ))
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
                 Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment')
             ])
 
         def test_extract_filter_allow_with_duplicates(self):
             lx = self.extractor_cls(allow=('sample', ), unique=False)
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
+                Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
                 Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment')
             ])
 
@@ -63,10 +63,10 @@ class Base:
             lx = self.extractor_cls(allow=('sample', ), unique=False,
                                     canonicalize=True)
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
+                Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
                 Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment')
             ])
 
@@ -74,22 +74,22 @@ class Base:
             lx = self.extractor_cls(allow=('sample',), unique=True,
                                     canonicalize=True)
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
             ])
 
         def test_extract_filter_allow_and_deny(self):
             lx = self.extractor_cls(allow=('sample', ), deny=('3', ))
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
             ])
 
         def test_extract_filter_allowed_domains(self):
             lx = self.extractor_cls(allow_domains=('google.com', ))
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://www.google.com/something', text=u''),
+                Link(url='http://www.google.com/something', text=''),
             ])
 
         def test_extraction_using_single_values(self):
@@ -97,27 +97,27 @@ class Base:
 
             lx = self.extractor_cls(allow='sample')
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
                 Link(url='http://example.com/sample3.html#foo',
                      text='sample 3 repetition with fragment')
             ])
 
             lx = self.extractor_cls(allow='sample', deny='3')
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
             ])
 
             lx = self.extractor_cls(allow_domains='google.com')
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://www.google.com/something', text=u''),
+                Link(url='http://www.google.com/something', text=''),
             ])
 
             lx = self.extractor_cls(deny_domains='example.com')
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://www.google.com/something', text=u''),
+                Link(url='http://www.google.com/something', text=''),
             ])
 
         def test_nofollow(self):
@@ -145,11 +145,11 @@ class Base:
 
             lx = self.extractor_cls()
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.org/about.html', text=u'About us'),
-                Link(url='http://example.org/follow.html', text=u'Follow this link'),
-                Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True),
-                Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'),
-                Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+                Link(url='http://example.org/about.html', text='About us'),
+                Link(url='http://example.org/follow.html', text='Follow this link'),
+                Link(url='http://example.org/nofollow.html', text='Dont follow this one', nofollow=True),
+                Link(url='http://example.org/nofollow2.html', text='Choose to follow or not'),
+                Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
             ])
 
         def test_matches(self):
@@ -183,8 +183,8 @@ class Base:
         def test_restrict_xpaths(self):
             lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ))
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
             ])
 
         def test_restrict_xpaths_encoding(self):
@@ -202,14 +202,14 @@ class Base:
 
             lx = self.extractor_cls(restrict_xpaths="//div[@class='links']")
             self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.org/about.html', text=u'About us\xa3')])
+                             [Link(url='http://example.org/about.html', text='About us\xa3')])
 
         def test_restrict_xpaths_with_html_entities(self):
             html = b'<html><body><p><a href="/&hearts;/you?c=&euro;">text</a></p></body></html>'
             response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15')
             links = self.extractor_cls(restrict_xpaths='//p').extract_links(response)
             self.assertEqual(links,
-                             [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text=u'text')])
+                             [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text='text')])
 
         def test_restrict_xpaths_concat_in_handle_data(self):
             """html entities cause SGMLParser to call handle_data hook twice"""
@@ -217,22 +217,22 @@ class Base:
             response = HtmlResponse("http://example.org", body=body, encoding='gb18030')
             lx = self.extractor_cls(restrict_xpaths="//div")
             self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c',
+                             [Link(url='http://example.org/foo', text='>\u4eac<\u4e1c',
                                    fragment='', nofollow=False)])
 
         def test_restrict_css(self):
             lx = self.extractor_cls(restrict_css=('#subwrapper a',))
             self.assertEqual(lx.extract_links(self.response), [
-                Link(url='http://example.com/sample2.html', text=u'sample 2')
+                Link(url='http://example.com/sample2.html', text='sample 2')
             ])
 
         def test_restrict_css_and_restrict_xpaths_together(self):
             lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ),
                                     restrict_css=('#subwrapper + a', ))
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
             ])
 
         def test_area_tag_with_unicode_present(self):
@@ -243,7 +243,7 @@ class Base:
             lx.extract_links(response)
             lx.extract_links(response)
             self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.org/foo', text=u'',
+                             [Link(url='http://example.org/foo', text='',
                                    fragment='', nofollow=False)])
 
         def test_encoded_url(self):
@@ -251,7 +251,7 @@ class Base:
             response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
             lx = self.extractor_cls()
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
+                Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False),
             ])
 
         def test_encoded_url_in_restricted_xpath(self):
@@ -259,7 +259,7 @@ class Base:
             response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
             lx = self.extractor_cls(restrict_xpaths="//div")
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
+                Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False),
             ])
 
         def test_ignored_extensions(self):
@@ -268,7 +268,7 @@ class Base:
             response = HtmlResponse("http://example.org/", body=html)
             lx = self.extractor_cls()
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.org/page.html', text=u'asd'),
+                Link(url='http://example.org/page.html', text='asd'),
             ])
 
             # override denied extensions
@@ -308,25 +308,25 @@ class Base:
             page4_url = 'http://example.com/page%204.html'
 
             self.assertEqual(lx.extract_links(self.response), [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
                 Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
-                Link(url='http://www.google.com/something', text=u''),
-                Link(url='http://example.com/innertag.html', text=u'inner tag'),
-                Link(url=page4_url, text=u'href with whitespaces'),
+                Link(url='http://www.google.com/something', text=''),
+                Link(url='http://example.com/innertag.html', text='inner tag'),
+                Link(url=page4_url, text='href with whitespaces'),
             ])
 
             lx = self.extractor_cls(attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=())
             self.assertEqual(lx.extract_links(self.response), [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample2.jpg', text=u''),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample2.jpg', text=''),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
                 Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
-                Link(url='http://www.google.com/something', text=u''),
-                Link(url='http://example.com/innertag.html', text=u'inner tag'),
-                Link(url=page4_url, text=u'href with whitespaces'),
+                Link(url='http://www.google.com/something', text=''),
+                Link(url='http://example.com/innertag.html', text='inner tag'),
+                Link(url=page4_url, text='href with whitespaces'),
             ])
 
             lx = self.extractor_cls(attrs=None)
@@ -344,24 +344,24 @@ class Base:
 
             lx = self.extractor_cls()
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
             ])
 
             lx = self.extractor_cls(tags="area")
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/sample1.html', text=u''),
+                Link(url='http://example.com/sample1.html', text=''),
             ])
 
             lx = self.extractor_cls(tags="a")
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
             ])
 
             lx = self.extractor_cls(tags=("a", "img"), attrs=("href", "src"), deny_extensions=())
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample2.jpg', text=u''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample2.jpg', text=''),
             ])
 
         def test_tags_attrs(self):
@@ -375,14 +375,14 @@ class Base:
 
             lx = self.extractor_cls(tags='div', attrs='data-url')
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
-                Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
+                Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False),
+                Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False)
             ])
 
             lx = self.extractor_cls(tags=('div',), attrs=('data-url',))
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
-                Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
+                Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False),
+                Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False)
             ])
 
         def test_xhtml(self):
@@ -420,13 +420,13 @@ class Base:
             self.assertEqual(
                 lx.extract_links(response),
                 [
-                    Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
-                    Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
-                    Link(url='http://example.com/nofollow.html', text=u'Dont follow this one',
+                    Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False),
+                    Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False),
+                    Link(url='http://example.com/nofollow.html', text='Dont follow this one',
                          fragment='', nofollow=True),
-                    Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not',
+                    Link(url='http://example.com/nofollow2.html', text='Choose to follow or not',
                          fragment='', nofollow=False),
-                    Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+                    Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
                 ]
             )
 
@@ -436,13 +436,13 @@ class Base:
             self.assertEqual(
                 lx.extract_links(response),
                 [
-                    Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
-                    Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
-                    Link(url='http://example.com/nofollow.html', text=u'Dont follow this one',
+                    Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False),
+                    Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False),
+                    Link(url='http://example.com/nofollow.html', text='Dont follow this one',
                          fragment='', nofollow=True),
-                    Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not',
+                    Link(url='http://example.com/nofollow2.html', text='Choose to follow or not',
                          fragment='', nofollow=False),
-                    Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+                    Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
                 ]
             )
 
@@ -455,8 +455,8 @@ class Base:
             response = HtmlResponse("http://example.org/index.html", body=html)
             lx = self.extractor_cls()
             self.assertEqual([link for link in lx.extract_links(response)], [
-                Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
-                Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
+                Link(url='http://example.org/item1.html', text='Item 1', nofollow=False),
+                Link(url='http://example.org/item3.html', text='Item 3', nofollow=False),
             ])
 
         def test_ftp_links(self):
@@ -467,7 +467,7 @@ class Base:
             response = HtmlResponse("http://www.example.com/index.html", body=body, encoding='utf8')
             lx = self.extractor_cls()
             self.assertEqual(lx.extract_links(response), [
-                Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False),
+                Link(url='ftp://www.external.com/', text='An Item', fragment='', nofollow=False),
             ])
 
         def test_pickle_extractor(self):
@@ -487,8 +487,8 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
         response = HtmlResponse("http://example.org/index.html", body=html)
         lx = self.extractor_cls()
         self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
-            Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
+            Link(url='http://example.org/item1.html', text='Item 1', nofollow=False),
+            Link(url='http://example.org/item3.html', text='Item 3', nofollow=False),
         ])
 
     def test_link_restrict_text(self):
@@ -501,18 +501,18 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
         # Simple text inclusion test
         lx = self.extractor_cls(restrict_text='dog')
         self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
+            Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
         ])
         # Unique regex test
         lx = self.extractor_cls(restrict_text=r'of.*dog')
         self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
+            Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
         ])
         # Multiple regex test
         lx = self.extractor_cls(restrict_text=[r'of.*dog', r'of.*cat'])
         self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item1.html', text=u'Pic of a cat', nofollow=False),
-            Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
+            Link(url='http://example.org/item1.html', text='Pic of a cat', nofollow=False),
+            Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
         ])
 
     def test_restrict_xpaths_with_html_entities(self):
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 581183625..2ed6f365f 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -69,23 +69,23 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_add_value_on_unknown_field(self):
         il = TestItemLoader()
-        self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo'])
+        self.assertRaises(KeyError, il.add_value, 'wrong_field', ['lala', 'lolo'])
 
     def test_load_item_using_default_loader(self):
         i = TestItem()
-        i['summary'] = u'lala'
+        i['summary'] = 'lala'
         il = ItemLoader(item=i)
-        il.add_value('name', u'marta')
+        il.add_value('name', 'marta')
         item = il.load_item()
         assert item is i
-        self.assertEqual(item['summary'], [u'lala'])
-        self.assertEqual(item['name'], [u'marta'])
+        self.assertEqual(item['summary'], ['lala'])
+        self.assertEqual(item['name'], ['marta'])
 
     def test_load_item_using_custom_loader(self):
         il = TestItemLoader()
-        il.add_value('name', u'marta')
+        il.add_value('name', 'marta')
         item = il.load_item()
-        self.assertEqual(item['name'], [u'Marta'])
+        self.assertEqual(item['name'], ['Marta'])
 
 
 class InitializationTestMixin:
@@ -290,137 +290,137 @@ class SelectortemLoaderTest(unittest.TestCase):
         self.assertRaises(RuntimeError, l.get_css, '#name::text')
 
     def test_init_method_with_selector(self):
-        sel = Selector(text=u"<html><body><div>marta</div></body></html>")
+        sel = Selector(text="<html><body><div>marta</div></body></html>")
         l = TestItemLoader(selector=sel)
         self.assertIs(l.selector, sel)
 
         l.add_xpath('name', '//div/text()')
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
 
     def test_init_method_with_selector_css(self):
-        sel = Selector(text=u"<html><body><div>marta</div></body></html>")
+        sel = Selector(text="<html><body><div>marta</div></body></html>")
         l = TestItemLoader(selector=sel)
         self.assertIs(l.selector, sel)
 
         l.add_css('name', 'div::text')
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
 
     def test_init_method_with_response(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
 
         l.add_xpath('name', '//div/text()')
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
 
     def test_init_method_with_response_css(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
 
         l.add_css('name', 'div::text')
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
 
         l.add_css('url', 'a::attr(href)')
-        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
 
         # combining/accumulating CSS selectors and XPath expressions
         l.add_xpath('name', '//div/text()')
-        self.assertEqual(l.get_output_value('name'), [u'Marta', u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta', 'Marta'])
 
         l.add_xpath('url', '//img/@src')
-        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org', u'/images/logo.png'])
+        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org', '/images/logo.png'])
 
     def test_add_xpath_re(self):
         l = TestItemLoader(response=self.response)
         l.add_xpath('name', '//div/text()', re='ma')
-        self.assertEqual(l.get_output_value('name'), [u'Ma'])
+        self.assertEqual(l.get_output_value('name'), ['Ma'])
 
     def test_replace_xpath(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
         l.add_xpath('name', '//div/text()')
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
         l.replace_xpath('name', '//p/text()')
-        self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
+        self.assertEqual(l.get_output_value('name'), ['Paragraph'])
 
         l.replace_xpath('name', ['//p/text()', '//div/text()'])
-        self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta'])
 
     def test_get_xpath(self):
         l = TestItemLoader(response=self.response)
-        self.assertEqual(l.get_xpath('//p/text()'), [u'paragraph'])
-        self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), u'paragraph')
-        self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), u'pa')
+        self.assertEqual(l.get_xpath('//p/text()'), ['paragraph'])
+        self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), 'paragraph')
+        self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), 'pa')
 
-        self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), [u'paragraph', 'marta'])
+        self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), ['paragraph', 'marta'])
 
     def test_replace_xpath_multi_fields(self):
         l = TestItemLoader(response=self.response)
         l.add_xpath(None, '//div/text()', TakeFirst(), lambda x: {'name': x})
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
         l.replace_xpath(None, '//p/text()', TakeFirst(), lambda x: {'name': x})
-        self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
+        self.assertEqual(l.get_output_value('name'), ['Paragraph'])
 
     def test_replace_xpath_re(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
         l.add_xpath('name', '//div/text()')
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
         l.replace_xpath('name', '//div/text()', re='ma')
-        self.assertEqual(l.get_output_value('name'), [u'Ma'])
+        self.assertEqual(l.get_output_value('name'), ['Ma'])
 
     def test_add_css_re(self):
         l = TestItemLoader(response=self.response)
         l.add_css('name', 'div::text', re='ma')
-        self.assertEqual(l.get_output_value('name'), [u'Ma'])
+        self.assertEqual(l.get_output_value('name'), ['Ma'])
 
         l.add_css('url', 'a::attr(href)', re='http://(.+)')
-        self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['www.scrapy.org'])
 
     def test_replace_css(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
         l.add_css('name', 'div::text')
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
         l.replace_css('name', 'p::text')
-        self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
+        self.assertEqual(l.get_output_value('name'), ['Paragraph'])
 
         l.replace_css('name', ['p::text', 'div::text'])
-        self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta'])
 
         l.add_css('url', 'a::attr(href)', re='http://(.+)')
-        self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['www.scrapy.org'])
         l.replace_css('url', 'img::attr(src)')
-        self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])
+        self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
 
     def test_get_css(self):
         l = TestItemLoader(response=self.response)
-        self.assertEqual(l.get_css('p::text'), [u'paragraph'])
-        self.assertEqual(l.get_css('p::text', TakeFirst()), u'paragraph')
-        self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), u'pa')
+        self.assertEqual(l.get_css('p::text'), ['paragraph'])
+        self.assertEqual(l.get_css('p::text', TakeFirst()), 'paragraph')
+        self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), 'pa')
 
-        self.assertEqual(l.get_css(['p::text', 'div::text']), [u'paragraph', 'marta'])
+        self.assertEqual(l.get_css(['p::text', 'div::text']), ['paragraph', 'marta'])
         self.assertEqual(l.get_css(['a::attr(href)', 'img::attr(src)']),
-                         [u'http://www.scrapy.org', u'/images/logo.png'])
+                         ['http://www.scrapy.org', '/images/logo.png'])
 
     def test_replace_css_multi_fields(self):
         l = TestItemLoader(response=self.response)
         l.add_css(None, 'div::text', TakeFirst(), lambda x: {'name': x})
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
         l.replace_css(None, 'p::text', TakeFirst(), lambda x: {'name': x})
-        self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
+        self.assertEqual(l.get_output_value('name'), ['Paragraph'])
 
         l.add_css(None, 'a::attr(href)', TakeFirst(), lambda x: {'url': x})
-        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
         l.replace_css(None, 'img::attr(src)', TakeFirst(), lambda x: {'url': x})
-        self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])
+        self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
 
     def test_replace_css_re(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
         l.add_css('url', 'a::attr(href)')
-        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
         l.replace_css('url', 'a::attr(href)', re=r'http://www\.(.+)')
-        self.assertEqual(l.get_output_value('url'), [u'scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['scrapy.org'])
 
 
 class SubselectorLoaderTest(unittest.TestCase):
@@ -447,9 +447,9 @@ class SubselectorLoaderTest(unittest.TestCase):
         nl.add_css('name_div', '#id')
         nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
 
-        self.assertEqual(l.get_output_value('name'), [u'marta'])
-        self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
-        self.assertEqual(l.get_output_value('name_value'), [u'marta'])
+        self.assertEqual(l.get_output_value('name'), ['marta'])
+        self.assertEqual(l.get_output_value('name_div'), ['<div id="id">marta</div>'])
+        self.assertEqual(l.get_output_value('name_value'), ['marta'])
 
         self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
         self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
@@ -462,9 +462,9 @@ class SubselectorLoaderTest(unittest.TestCase):
         nl.add_css('name_div', '#id')
         nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
 
-        self.assertEqual(l.get_output_value('name'), [u'marta'])
-        self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
-        self.assertEqual(l.get_output_value('name_value'), [u'marta'])
+        self.assertEqual(l.get_output_value('name'), ['marta'])
+        self.assertEqual(l.get_output_value('name_div'), ['<div id="id">marta</div>'])
+        self.assertEqual(l.get_output_value('name_value'), ['marta'])
 
         self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
         self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
@@ -476,11 +476,11 @@ class SubselectorLoaderTest(unittest.TestCase):
         nl2 = nl1.nested_xpath('a')
 
         l.add_xpath('url', '//footer/a/@href')
-        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
         nl1.replace_xpath('url', 'img/@src')
-        self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])
+        self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
         nl2.replace_xpath('url', '@href')
-        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
 
     def test_nested_ordering(self):
         l = NestedItemLoader(response=self.response)
@@ -493,10 +493,10 @@ class SubselectorLoaderTest(unittest.TestCase):
         l.add_xpath('url', '//footer/a/@href')
 
         self.assertEqual(l.get_output_value('url'), [
-            u'/images/logo.png',
-            u'http://www.scrapy.org',
-            u'homepage',
-            u'http://www.scrapy.org',
+            '/images/logo.png',
+            'http://www.scrapy.org',
+            'homepage',
+            'http://www.scrapy.org',
         ])
 
     def test_nested_load_item(self):
@@ -514,9 +514,9 @@ class SubselectorLoaderTest(unittest.TestCase):
         assert item is nl1.item
         assert item is nl2.item
 
-        self.assertEqual(item['name'], [u'marta'])
-        self.assertEqual(item['url'], [u'http://www.scrapy.org'])
-        self.assertEqual(item['image'], [u'/images/logo.png'])
+        self.assertEqual(item['name'], ['marta'])
+        self.assertEqual(item['url'], ['http://www.scrapy.org'])
+        self.assertEqual(item['image'], ['/images/logo.png'])
 
 
 # Functions as processors
diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py
index d0a59e8cd..eb14de14f 100644
--- a/tests/test_loader_deprecated.py
+++ b/tests/test_loader_deprecated.py
@@ -51,19 +51,19 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_load_item_using_default_loader(self):
         i = TestItem()
-        i['summary'] = u'lala'
+        i['summary'] = 'lala'
         il = ItemLoader(item=i)
-        il.add_value('name', u'marta')
+        il.add_value('name', 'marta')
         item = il.load_item()
         assert item is i
-        self.assertEqual(item['summary'], [u'lala'])
-        self.assertEqual(item['name'], [u'marta'])
+        self.assertEqual(item['summary'], ['lala'])
+        self.assertEqual(item['name'], ['marta'])
 
     def test_load_item_using_custom_loader(self):
         il = TestItemLoader()
-        il.add_value('name', u'marta')
+        il.add_value('name', 'marta')
         item = il.load_item()
-        self.assertEqual(item['name'], [u'Marta'])
+        self.assertEqual(item['name'], ['Marta'])
 
     def test_load_item_ignore_none_field_values(self):
         def validate_sku(value):
@@ -76,23 +76,23 @@ class BasicItemLoaderTest(unittest.TestCase):
             price_out = Compose(TakeFirst(), float)
             sku_out = Compose(TakeFirst(), validate_sku)
 
-        valid_fragment = u'SKU: 1234'
-        invalid_fragment = u'SKU: not available'
+        valid_fragment = 'SKU: 1234'
+        invalid_fragment = 'SKU: not available'
         sku_re = 'SKU: (.+)'
 
         il = MyLoader(item={})
         # Should not return "sku: None".
         il.add_value('sku', [invalid_fragment], re=sku_re)
         # Should not ignore empty values.
-        il.add_value('name', u'')
-        il.add_value('price', [u'0'])
+        il.add_value('name', '')
+        il.add_value('price', ['0'])
         self.assertEqual(il.load_item(), {
-            'name': u'',
+            'name': '',
             'price': 0.0,
         })
 
         il.replace_value('sku', [valid_fragment], re=sku_re)
-        self.assertEqual(il.load_item()['sku'], u'1234')
+        self.assertEqual(il.load_item()['sku'], '1234')
 
     def test_self_referencing_loader(self):
         class MyLoader(ItemLoader):
@@ -117,19 +117,19 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_add_value(self):
         il = TestItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'Marta'])
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
-        il.add_value('name', u'pepe')
-        self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe'])
-        self.assertEqual(il.get_output_value('name'), [u'Marta', u'Pepe'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_collected_values('name'), ['Marta'])
+        self.assertEqual(il.get_output_value('name'), ['Marta'])
+        il.add_value('name', 'pepe')
+        self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe'])
+        self.assertEqual(il.get_output_value('name'), ['Marta', 'Pepe'])
 
         # test add object value
         il.add_value('summary', {'key': 1})
         self.assertEqual(il.get_collected_values('summary'), [{'key': 1}])
 
-        il.add_value(None, u'Jim', lambda x: {'name': x})
-        self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe', u'Jim'])
+        il.add_value(None, 'Jim', lambda x: {'name': x})
+        self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe', 'Jim'])
 
     def test_add_zero(self):
         il = NameItemLoader()
@@ -138,49 +138,49 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_replace_value(self):
         il = TestItemLoader()
-        il.replace_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'Marta'])
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
-        il.replace_value('name', u'pepe')
-        self.assertEqual(il.get_collected_values('name'), [u'Pepe'])
-        self.assertEqual(il.get_output_value('name'), [u'Pepe'])
+        il.replace_value('name', 'marta')
+        self.assertEqual(il.get_collected_values('name'), ['Marta'])
+        self.assertEqual(il.get_output_value('name'), ['Marta'])
+        il.replace_value('name', 'pepe')
+        self.assertEqual(il.get_collected_values('name'), ['Pepe'])
+        self.assertEqual(il.get_output_value('name'), ['Pepe'])
 
-        il.replace_value(None, u'Jim', lambda x: {'name': x})
-        self.assertEqual(il.get_collected_values('name'), [u'Jim'])
+        il.replace_value(None, 'Jim', lambda x: {'name': x})
+        self.assertEqual(il.get_collected_values('name'), ['Jim'])
 
     def test_get_value(self):
         il = NameItemLoader()
-        self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper))
-        self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$'))
-        self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$'))
+        self.assertEqual('FOO', il.get_value(['foo', 'bar'], TakeFirst(), str.upper))
+        self.assertEqual(['foo', 'bar'], il.get_value(['name:foo', 'name:bar'], re='name:(.*)$'))
+        self.assertEqual('foo', il.get_value(['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$'))
 
-        il.add_value('name', [u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')
-        self.assertEqual([u'foo'], il.get_collected_values('name'))
-        il.replace_value('name', u'name:bar', re=u'name:(.*)$')
-        self.assertEqual([u'bar'], il.get_collected_values('name'))
+        il.add_value('name', ['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$')
+        self.assertEqual(['foo'], il.get_collected_values('name'))
+        il.replace_value('name', 'name:bar', re='name:(.*)$')
+        self.assertEqual(['bar'], il.get_collected_values('name'))
 
     def test_iter_on_input_processor_input(self):
         class NameFirstItemLoader(NameItemLoader):
             name_in = TakeFirst()
 
         il = NameFirstItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_collected_values('name'), ['marta'])
         il = NameFirstItemLoader()
-        il.add_value('name', [u'marta', u'jose'])
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+        il.add_value('name', ['marta', 'jose'])
+        self.assertEqual(il.get_collected_values('name'), ['marta'])
 
         il = NameFirstItemLoader()
-        il.replace_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+        il.replace_value('name', 'marta')
+        self.assertEqual(il.get_collected_values('name'), ['marta'])
         il = NameFirstItemLoader()
-        il.replace_value('name', [u'marta', u'jose'])
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+        il.replace_value('name', ['marta', 'jose'])
+        self.assertEqual(il.get_collected_values('name'), ['marta'])
 
         il = NameFirstItemLoader()
-        il.add_value('name', u'marta')
-        il.add_value('name', [u'jose', u'pedro'])
-        self.assertEqual(il.get_collected_values('name'), [u'marta', u'jose'])
+        il.add_value('name', 'marta')
+        il.add_value('name', ['jose', 'pedro'])
+        self.assertEqual(il.get_collected_values('name'), ['marta', 'jose'])
 
     def test_map_compose_filter(self):
         def filter_world(x):
@@ -195,87 +195,87 @@ class BasicItemLoaderTest(unittest.TestCase):
             name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1])
 
         il = TestItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'Mart'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['Mart'])
         item = il.load_item()
-        self.assertEqual(item['name'], [u'Mart'])
+        self.assertEqual(item['name'], ['Mart'])
 
     def test_default_input_processor(self):
         il = DefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'mart'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['mart'])
 
     def test_inherited_default_input_processor(self):
         class InheritDefaultedItemLoader(DefaultedItemLoader):
             pass
 
         il = InheritDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'mart'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['mart'])
 
     def test_input_processor_inheritance(self):
         class ChildItemLoader(TestItemLoader):
             url_in = MapCompose(lambda v: v.lower())
 
         il = ChildItemLoader()
-        il.add_value('url', u'HTTP://scrapy.ORG')
-        self.assertEqual(il.get_output_value('url'), [u'http://scrapy.org'])
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
+        il.add_value('url', 'HTTP://scrapy.ORG')
+        self.assertEqual(il.get_output_value('url'), ['http://scrapy.org'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['Marta'])
 
         class ChildChildItemLoader(ChildItemLoader):
             url_in = MapCompose(lambda v: v.upper())
             summary_in = MapCompose(lambda v: v)
 
         il = ChildChildItemLoader()
-        il.add_value('url', u'http://scrapy.org')
-        self.assertEqual(il.get_output_value('url'), [u'HTTP://SCRAPY.ORG'])
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
+        il.add_value('url', 'http://scrapy.org')
+        self.assertEqual(il.get_output_value('url'), ['HTTP://SCRAPY.ORG'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['Marta'])
 
     def test_empty_map_compose(self):
         class IdentityDefaultedItemLoader(DefaultedItemLoader):
             name_in = MapCompose()
 
         il = IdentityDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'marta'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['marta'])
 
     def test_identity_input_processor(self):
         class IdentityDefaultedItemLoader(DefaultedItemLoader):
             name_in = Identity()
 
         il = IdentityDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'marta'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['marta'])
 
     def test_extend_custom_input_processors(self):
         class ChildItemLoader(TestItemLoader):
             name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
 
         il = ChildItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'mARTA'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['mARTA'])
 
     def test_extend_default_input_processors(self):
         class ChildDefaultedItemLoader(DefaultedItemLoader):
             name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase)
 
         il = ChildDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'MART'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['MART'])
 
     def test_output_processor_using_function(self):
         il = TestItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+        il.add_value('name', ['mar', 'ta'])
+        self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
 
         class TakeFirstItemLoader(TestItemLoader):
-            name_out = u" ".join
+            name_out = " ".join
 
         il = TakeFirstItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), u'Mar Ta')
+        il.add_value('name', ['mar', 'ta'])
+        self.assertEqual(il.get_output_value('name'), 'Mar Ta')
 
     def test_output_processor_error(self):
         class TestItemLoader(ItemLoader):
@@ -283,9 +283,9 @@ class BasicItemLoaderTest(unittest.TestCase):
             name_out = MapCompose(float)
 
         il = TestItemLoader()
-        il.add_value('name', [u'$10'])
+        il.add_value('name', ['$10'])
         try:
-            float(u'$10')
+            float('$10')
         except Exception as e:
             expected_exc_str = str(e)
 
@@ -303,53 +303,53 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_output_processor_using_classes(self):
         il = TestItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+        il.add_value('name', ['mar', 'ta'])
+        self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
 
         class TakeFirstItemLoader(TestItemLoader):
             name_out = Join()
 
         il = TakeFirstItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), u'Mar Ta')
+        il.add_value('name', ['mar', 'ta'])
+        self.assertEqual(il.get_output_value('name'), 'Mar Ta')
 
         class TakeFirstItemLoader(TestItemLoader):
             name_out = Join("<br>")
 
         il = TakeFirstItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), u'Mar<br>Ta')
+        il.add_value('name', ['mar', 'ta'])
+        self.assertEqual(il.get_output_value('name'), 'Mar<br>Ta')
 
     def test_default_output_processor(self):
         il = TestItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+        il.add_value('name', ['mar', 'ta'])
+        self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
 
         class LalaItemLoader(TestItemLoader):
             default_output_processor = Identity()
 
         il = LalaItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+        il.add_value('name', ['mar', 'ta'])
+        self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
 
     def test_loader_context_on_declaration(self):
         class ChildItemLoader(TestItemLoader):
-            url_in = MapCompose(processor_with_args, key=u'val')
+            url_in = MapCompose(processor_with_args, key='val')
 
         il = ChildItemLoader()
-        il.add_value('url', u'text')
+        il.add_value('url', 'text')
         self.assertEqual(il.get_output_value('url'), ['val'])
-        il.replace_value('url', u'text2')
+        il.replace_value('url', 'text2')
         self.assertEqual(il.get_output_value('url'), ['val'])
 
     def test_loader_context_on_instantiation(self):
         class ChildItemLoader(TestItemLoader):
             url_in = MapCompose(processor_with_args)
 
-        il = ChildItemLoader(key=u'val')
-        il.add_value('url', u'text')
+        il = ChildItemLoader(key='val')
+        il.add_value('url', 'text')
         self.assertEqual(il.get_output_value('url'), ['val'])
-        il.replace_value('url', u'text2')
+        il.replace_value('url', 'text2')
         self.assertEqual(il.get_output_value('url'), ['val'])
 
     def test_loader_context_on_assign(self):
@@ -357,10 +357,10 @@ class BasicItemLoaderTest(unittest.TestCase):
             url_in = MapCompose(processor_with_args)
 
         il = ChildItemLoader()
-        il.context['key'] = u'val'
-        il.add_value('url', u'text')
+        il.context['key'] = 'val'
+        il.add_value('url', 'text')
         self.assertEqual(il.get_output_value('url'), ['val'])
-        il.replace_value('url', u'text2')
+        il.replace_value('url', 'text2')
         self.assertEqual(il.get_output_value('url'), ['val'])
 
     def test_item_passed_to_input_processor_functions(self):
@@ -372,9 +372,9 @@ class BasicItemLoaderTest(unittest.TestCase):
 
         it = TestItem(name='marta')
         il = ChildItemLoader(item=it)
-        il.add_value('url', u'text')
+        il.add_value('url', 'text')
         self.assertEqual(il.get_output_value('url'), ['marta'])
-        il.replace_value('url', u'text2')
+        il.replace_value('url', 'text2')
         self.assertEqual(il.get_output_value('url'), ['marta'])
 
     def test_compose_processor(self):
@@ -382,10 +382,10 @@ class BasicItemLoaderTest(unittest.TestCase):
             name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1])
 
         il = TestItemLoader()
-        il.add_value('name', [u'marta', u'other'])
-        self.assertEqual(il.get_output_value('name'), u'Mart')
+        il.add_value('name', ['marta', 'other'])
+        self.assertEqual(il.get_output_value('name'), 'Mart')
         item = il.load_item()
-        self.assertEqual(item['name'], u'Mart')
+        self.assertEqual(item['name'], 'Mart')
 
     def test_partial_processor(self):
         def join(values, sep=None, loader_context=None, ignored=None):
@@ -402,13 +402,13 @@ class BasicItemLoaderTest(unittest.TestCase):
             summary_out = Compose(partial(join, ignored='foo'))
 
         il = TestItemLoader()
-        il.add_value('name', [u'rabbit', u'hole'])
-        il.add_value('url', [u'rabbit', u'hole'])
-        il.add_value('summary', [u'rabbit', u'hole'])
+        il.add_value('name', ['rabbit', 'hole'])
+        il.add_value('url', ['rabbit', 'hole'])
+        il.add_value('summary', ['rabbit', 'hole'])
         item = il.load_item()
-        self.assertEqual(item['name'], u'rabbit+hole')
-        self.assertEqual(item['url'], u'rabbit.hole')
-        self.assertEqual(item['summary'], u'rabbithole')
+        self.assertEqual(item['name'], 'rabbit+hole')
+        self.assertEqual(item['url'], 'rabbit.hole')
+        self.assertEqual(item['summary'], 'rabbithole')
 
     def test_error_input_processor(self):
         class TestItem(Item):
@@ -420,7 +420,7 @@ class BasicItemLoaderTest(unittest.TestCase):
 
         il = TestItemLoader()
         self.assertRaises(ValueError, il.add_value, 'name',
-                          [u'marta', u'other'])
+                          ['marta', 'other'])
 
     def test_error_output_processor(self):
         class TestItem(Item):
@@ -431,7 +431,7 @@ class BasicItemLoaderTest(unittest.TestCase):
             name_out = Compose(Join(), float)
 
         il = TestItemLoader()
-        il.add_value('name', u'marta')
+        il.add_value('name', 'marta')
         with self.assertRaises(ValueError):
             il.load_item()
 
@@ -444,7 +444,7 @@ class BasicItemLoaderTest(unittest.TestCase):
 
         il = TestItemLoader()
         self.assertRaises(ValueError, il.add_value, 'name',
-                          [u'marta', u'other'], Compose(float))
+                          ['marta', 'other'], Compose(float))
 
 
 class InitializationFromDictTest(unittest.TestCase):
@@ -608,8 +608,8 @@ class ProcessorsTest(unittest.TestCase):
     def test_join(self):
         proc = Join()
         self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
-        self.assertEqual(proc(['', 'hello', 'world']), u' hello world')
-        self.assertEqual(proc(['hello', 'world']), u'hello world')
+        self.assertEqual(proc(['', 'hello', 'world']), ' hello world')
+        self.assertEqual(proc(['hello', 'world']), 'hello world')
         self.assertIsInstance(proc(['hello', 'world']), str)
 
     def test_compose(self):
@@ -626,8 +626,8 @@ class ProcessorsTest(unittest.TestCase):
         def filter_world(x):
             return None if x == 'world' else x
         proc = MapCompose(filter_world, str.upper)
-        self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
-                         [u'HELLO', u'THIS', u'IS', u'SCRAPY'])
+        self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']),
+                         ['HELLO', 'THIS', 'IS', 'SCRAPY'])
         proc = MapCompose(filter_world, str.upper)
         self.assertEqual(proc(None), [])
         proc = MapCompose(filter_world, str.upper)
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 7064337ad..b771e7d79 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -56,13 +56,13 @@ class LogFormatterTestCase(unittest.TestCase):
 
     def test_dropped(self):
         item = {}
-        exception = Exception(u"\u2018")
+        exception = Exception("\u2018")
         response = Response("http://www.example.com")
         logkws = self.formatter.dropped(item, exception, response, self.spider)
         logline = logkws['msg'] % logkws['args']
         lines = logline.splitlines()
         assert all(isinstance(x, str) for x in lines)
-        self.assertEqual(lines, [u"Dropped: \u2018", '{}'])
+        self.assertEqual(lines, ["Dropped: \u2018", '{}'])
 
     def test_item_error(self):
         # In practice, the complete traceback is shown by passing the
@@ -72,7 +72,7 @@ class LogFormatterTestCase(unittest.TestCase):
         response = Response("http://www.example.com")
         logkws = self.formatter.item_error(item, exception, response, self.spider)
         logline = logkws['msg'] % logkws['args']
-        self.assertEqual(logline, u"Error processing {'key': 'value'}")
+        self.assertEqual(logline, "Error processing {'key': 'value'}")
 
     def test_spider_error(self):
         # In practice, the complete traceback is shown by passing the
@@ -107,20 +107,20 @@ class LogFormatterTestCase(unittest.TestCase):
 
     def test_scraped(self):
         item = CustomItem()
-        item['name'] = u'\xa3'
+        item['name'] = '\xa3'
         response = Response("http://www.example.com")
         logkws = self.formatter.scraped(item, response, self.spider)
         logline = logkws['msg'] % logkws['args']
         lines = logline.splitlines()
         assert all(isinstance(x, str) for x in lines)
-        self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3'])
+        self.assertEqual(lines, ["Scraped from <200 http://www.example.com>", 'name: \xa3'])
 
 
 class LogFormatterSubclass(LogFormatter):
     def crawled(self, request, response, spider):
         kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider)
         CRAWLEDMSG = (
-            u"Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
+            "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
         )
         log_args = kwargs['args']
         log_args['flags'] = str(request.flags)
diff --git a/tests/test_mail.py b/tests/test_mail.py
index 53dbc0686..9b248fbfa 100644
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@ -73,8 +73,8 @@ class MailSenderTest(unittest.TestCase):
         self.catched_msg = dict(**kwargs)
 
     def test_send_utf8(self):
-        subject = u'sübjèçt'
-        body = u'bödÿ-àéïöñß'
+        subject = 'sübjèçt'
+        body = 'bödÿ-àéïöñß'
         mailsender = MailSender(debug=True)
         mailsender.send(to=['test@scrapy.org'], subject=subject, body=body,
                         charset='utf-8', _callback=self._catch_mail_sent)
@@ -90,8 +90,8 @@ class MailSenderTest(unittest.TestCase):
         self.assertEqual(msg.get('Content-Type'), 'text/plain; charset="utf-8"')
 
     def test_send_attach_utf8(self):
-        subject = u'sübjèçt'
-        body = u'bödÿ-àéïöñß'
+        subject = 'sübjèçt'
+        body = 'bödÿ-àéïöñß'
         attach = BytesIO()
         attach.write(body.encode('utf-8'))
         attach.seek(0)
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index dd19a69d5..a175f88ca 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -23,11 +23,11 @@ class ResponseTypesTest(unittest.TestCase):
         mappings = [
             (b'attachment; filename="data.xml"', XmlResponse),
             (b'attachment; filename=data.xml', XmlResponse),
-            (u'attachment;filename=data£.tar.gz'.encode('utf-8'), Response),
-            (u'attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response),
-            (u'attachment;filename=data高.doc'.encode('gbk'), Response),
-            (u'attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse),
-            (u'attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse),
+            ('attachment;filename=data£.tar.gz'.encode('utf-8'), Response),
+            ('attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response),
+            ('attachment;filename=data高.doc'.encode('gbk'), Response),
+            ('attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse),
+            ('attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse),
 
         ]
         for source, cls in mappings:
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 24aaaf7ec..9d8c201dd 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -93,7 +93,7 @@ class BaseRobotParserTest:
         self.assertTrue(rp.allowed("https://site.local/disallowed", "*"))
 
     def test_unicode_url_and_useragent(self):
-        robotstxt_robotstxt_body = u"""
+        robotstxt_robotstxt_body = """
         User-Agent: *
         Disallow: /admin/
         Disallow: /static/
@@ -107,11 +107,11 @@ class BaseRobotParserTest:
         self.assertTrue(rp.allowed("https://site.local/", "*"))
         self.assertFalse(rp.allowed("https://site.local/admin/", "*"))
         self.assertFalse(rp.allowed("https://site.local/static/", "*"))
-        self.assertTrue(rp.allowed("https://site.local/admin/", u"UnicödeBöt"))
+        self.assertTrue(rp.allowed("https://site.local/admin/", "UnicödeBöt"))
         self.assertFalse(rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*"))
-        self.assertFalse(rp.allowed(u"https://site.local/wiki/Käyttäjä:", "*"))
+        self.assertFalse(rp.allowed("https://site.local/wiki/Käyttäjä:", "*"))
         self.assertTrue(rp.allowed("https://site.local/some/randome/page.html", "*"))
-        self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", u"UnicödeBöt"))
+        self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt"))
 
 
 class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 00e663c11..62036ad8c 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -25,19 +25,19 @@ class SelectorTestCase(unittest.TestCase):
         )
         self.assertEqual(
             [x.get() for x in sel.xpath("//input[@name='a']/@name")],
-            [u'a']
+            ['a']
         )
         self.assertEqual(
             [x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
-            [u'12.0']
+            ['12.0']
         )
         self.assertEqual(
             sel.xpath("concat('xpath', 'rules')").getall(),
-            [u'xpathrules']
+            ['xpathrules']
         )
         self.assertEqual(
             [x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
-            [u'12']
+            ['12']
         )
 
     def test_root_base_url(self):
@@ -52,30 +52,30 @@ class SelectorTestCase(unittest.TestCase):
         sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))
         self.assertEqual(sel.type, 'xml')
         self.assertEqual(sel.xpath("//div").getall(),
-                         [u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
+                         ['<div><img src="a.jpg"><p>Hello</p></img></div>'])
 
         sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8'))
         self.assertEqual(sel.type, 'html')
         self.assertEqual(sel.xpath("//div").getall(),
-                         [u'<div><img src="a.jpg"><p>Hello</p></div>'])
+                         ['<div><img src="a.jpg"><p>Hello</p></div>'])
 
     def test_http_header_encoding_precedence(self):
-        # u'\xa3'     = pound symbol in unicode
-        # u'\xc2\xa3' = pound symbol in utf-8
-        # u'\xa3'     = pound symbol in latin-1 (iso-8859-1)
+        # '\xa3'     = pound symbol in unicode
+        # '\xc2\xa3' = pound symbol in utf-8
+        # '\xa3'     = pound symbol in latin-1 (iso-8859-1)
 
-        meta = u'<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">'
-        head = u'<head>' + meta + u'</head>'
-        body_content = u'<span id="blank">\xa3</span>'
-        body = u'<body>' + body_content + u'</body>'
-        html = u'<html>' + head + body + u'</html>'
+        meta = '<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">'
+        head = '<head>' + meta + '</head>'
+        body_content = '<span id="blank">\xa3</span>'
+        body = '<body>' + body_content + '</body>'
+        html = '<html>' + head + body + '</html>'
         encoding = 'utf-8'
         html_utf8 = html.encode(encoding)
 
         headers = {'Content-Type': ['text/html; charset=utf-8']}
         response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
         x = Selector(response)
-        self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3'])
+        self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), ['\xa3'])
 
     def test_badly_encoded_body(self):
         # \xe9 alone isn't valid utf8 sequence
@@ -92,4 +92,4 @@ class SelectorTestCase(unittest.TestCase):
 
     def test_selector_bad_args(self):
         with self.assertRaisesRegex(ValueError, 'received both response and text'):
-            Selector(TextResponse(url='http://example.com', body=b''), text=u'')
+            Selector(TextResponse(url='http://example.com', body=b''), text='')
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 83c10a3c3..78157a9b9 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -153,13 +153,13 @@ class XMLFeedSpiderTest(SpiderTest):
             output = list(spider._parse(response))
             self.assertEqual(len(output), 2, iterator)
             self.assertEqual(output, [
-                {'loc': [u'http://www.example.com/Special-Offers.html'],
-                 'updated': [u'2009-08-16'],
-                 'custom': [u'fuu'],
-                 'other': [u'bar']},
+                {'loc': ['http://www.example.com/Special-Offers.html'],
+                 'updated': ['2009-08-16'],
+                 'custom': ['fuu'],
+                 'other': ['bar']},
                 {'loc': [],
-                 'updated': [u'2009-08-16'],
-                 'other': [u'foo'],
+                 'updated': ['2009-08-16'],
+                 'other': ['foo'],
                  'custom': []},
             ], iterator)
 
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index d17bb2cbc..298178f08 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -54,7 +54,7 @@ class XmliterTestCase(unittest.TestCase):
 
     def test_xmliter_unicode(self):
         # example taken from https://github.com/scrapy/scrapy/issues/1665
-        body = u"""<?xml version="1.0" encoding="UTF-8"?>
+        body = """<?xml version="1.0" encoding="UTF-8"?>
             <þingflokkar>
                <þingflokkur id="26">
                   <heiti />
@@ -97,15 +97,15 @@ class XmliterTestCase(unittest.TestCase):
             XmlResponse(url="http://example.com", body=body, encoding='utf-8'),
         ):
             attrs = []
-            for x in self.xmliter(r, u'þingflokkur'):
+            for x in self.xmliter(r, 'þingflokkur'):
                 attrs.append((x.attrib['id'],
-                              x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(),
-                              x.xpath(u'./tímabil/fyrstaþing/text()').getall()))
+                              x.xpath('./skammstafanir/stuttskammstöfun/text()').getall(),
+                              x.xpath('./tímabil/fyrstaþing/text()').getall()))
 
             self.assertEqual(attrs,
-                             [(u'26', [u'-'], [u'80']),
-                              (u'21', [u'Ab'], [u'76']),
-                              (u'27', [u'A'], [u'27'])])
+                             [('26', ['-'], ['80']),
+                              ('21', ['Ab'], ['76']),
+                              ('27', ['A'], ['27'])])
 
     def test_xmliter_text(self):
         body = (
@@ -114,7 +114,7 @@ class XmliterTestCase(unittest.TestCase):
         )
 
         self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')],
-                         [[u'one'], [u'two']])
+                         [['one'], ['two']])
 
     def test_xmliter_namespaces(self):
         body = b"""
@@ -179,7 +179,7 @@ class XmliterTestCase(unittest.TestCase):
         response = XmlResponse('http://www.example.com', body=body)
         self.assertEqual(
             next(self.xmliter(response, 'item')).get(),
-            u'<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'
+            '<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'
         )
 
 
@@ -265,10 +265,10 @@ class UtilsCsvTestCase(unittest.TestCase):
 
         result = [row for row in csv]
         self.assertEqual(result,
-                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
-                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty', u'value': u''}])
+                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '4', 'name': 'empty', 'value': ''}])
 
         # explicit type check cuz' we no like stinkin' autocasting! yarrr
         for result_row in result:
@@ -281,10 +281,10 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv = csviter(response, delimiter='\t')
 
         self.assertEqual([row for row in csv],
-                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
-                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty', u'value': u''}])
+                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '4', 'name': 'empty', 'value': ''}])
 
     def test_csviter_quotechar(self):
         body1 = get_testdata('feeds', 'feed-sample6.csv')
@@ -294,19 +294,19 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv1 = csviter(response1, quotechar="'")
 
         self.assertEqual([row for row in csv1],
-                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
-                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty', u'value': u''}])
+                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '4', 'name': 'empty', 'value': ''}])
 
         response2 = TextResponse(url="http://example.com/", body=body2)
         csv2 = csviter(response2, delimiter="|", quotechar="'")
 
         self.assertEqual([row for row in csv2],
-                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
-                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty', u'value': u''}])
+                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '4', 'name': 'empty', 'value': ''}])
 
     def test_csviter_wrong_quotechar(self):
         body = get_testdata('feeds', 'feed-sample6.csv')
@@ -314,10 +314,10 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv = csviter(response)
 
         self.assertEqual([row for row in csv],
-                         [{u"'id'": u"1", u"'name'": u"'alpha'", u"'value'": u"'foobar'"},
-                          {u"'id'": u"2", u"'name'": u"'unicode'", u"'value'": u"'\xfan\xedc\xf3d\xe9\u203d'"},
-                          {u"'id'": u"'3'", u"'name'": u"'multi'", u"'value'": u"'foo"},
-                          {u"'id'": u"4", u"'name'": u"'empty'", u"'value'": u""}])
+                         [{"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"},
+                          {"'id'": "2", "'name'": "'unicode'", "'value'": "'\xfan\xedc\xf3d\xe9\u203d'"},
+                          {"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"},
+                          {"'id'": "4", "'name'": "'empty'", "'value'": ""}])
 
     def test_csviter_delimiter_binary_response_assume_utf8_encoding(self):
         body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
@@ -325,10 +325,10 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv = csviter(response, delimiter='\t')
 
         self.assertEqual([row for row in csv],
-                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
-                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty', u'value': u''}])
+                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '4', 'name': 'empty', 'value': ''}])
 
     def test_csviter_headers(self):
         sample = get_testdata('feeds', 'feed-sample3.csv').splitlines()
@@ -338,10 +338,10 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv = csviter(response, headers=[h.decode('utf-8') for h in headers])
 
         self.assertEqual([row for row in csv],
-                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
-                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'},
-                          {u'id': u'4', u'name': u'empty', u'value': u''}])
+                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+                          {'id': '3', 'name': 'multi', 'value': 'foo\nbar'},
+                          {'id': '4', 'name': 'empty', 'value': ''}])
 
     def test_csviter_falserow(self):
         body = get_testdata('feeds', 'feed-sample3.csv')
@@ -351,10 +351,10 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv = csviter(response)
 
         self.assertEqual([row for row in csv],
-                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
-                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty', u'value': u''}])
+                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '4', 'name': 'empty', 'value': ''}])
 
     def test_csviter_exception(self):
         body = get_testdata('feeds', 'feed-sample3.csv')
@@ -377,8 +377,8 @@ class UtilsCsvTestCase(unittest.TestCase):
         self.assertEqual(
             list(csv),
             [
-                {u'id': u'1', u'name': u'latin1', u'value': u'test'},
-                {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'},
+                {'id': '1', 'name': 'latin1', 'value': 'test'},
+                {'id': '2', 'name': 'something', 'value': '\xf1\xe1\xe9\xf3'},
             ]
         )
 
@@ -387,8 +387,8 @@ class UtilsCsvTestCase(unittest.TestCase):
         self.assertEqual(
             list(csv),
             [
-                {u'id': u'1', u'name': u'cp852', u'value': u'test'},
-                {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'},
+                {'id': '1', 'name': 'cp852', 'value': 'test'},
+                {'id': '2', 'name': 'something', 'value': '\u255a\u2569\u2569\u2569\u2550\u2550\u2557'},
             ]
         )
 
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index ebce3c079..3f93f509e 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -34,13 +34,13 @@ class MutableChainTest(unittest.TestCase):
 
 class ToUnicodeTest(unittest.TestCase):
     def test_converting_an_utf8_encoded_string_to_unicode(self):
-        self.assertEqual(to_unicode(b'lel\xc3\xb1e'), u'lel\xf1e')
+        self.assertEqual(to_unicode(b'lel\xc3\xb1e'), 'lel\xf1e')
 
     def test_converting_a_latin_1_encoded_string_to_unicode(self):
-        self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), u'lel\xf1e')
+        self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), 'lel\xf1e')
 
     def test_converting_a_unicode_to_unicode_should_return_the_same_object(self):
-        self.assertEqual(to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e')
+        self.assertEqual(to_unicode('\xf1e\xf1e\xf1e'), '\xf1e\xf1e\xf1e')
 
     def test_converting_a_strange_object_should_raise_TypeError(self):
         self.assertRaises(TypeError, to_unicode, 423)
@@ -48,16 +48,16 @@ class ToUnicodeTest(unittest.TestCase):
     def test_errors_argument(self):
         self.assertEqual(
             to_unicode(b'a\xedb', 'utf-8', errors='replace'),
-            u'a\ufffdb'
+            'a\ufffdb'
         )
 
 
 class ToBytesTest(unittest.TestCase):
     def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self):
-        self.assertEqual(to_bytes(u'\xa3 49'), b'\xc2\xa3 49')
+        self.assertEqual(to_bytes('\xa3 49'), b'\xc2\xa3 49')
 
     def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self):
-        self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), b'\xa3 49')
+        self.assertEqual(to_bytes('\xa3 49', 'latin-1'), b'\xa3 49')
 
     def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self):
         self.assertEqual(to_bytes(b'lel\xf1e'), b'lel\xf1e')
@@ -67,7 +67,7 @@ class ToBytesTest(unittest.TestCase):
 
     def test_errors_argument(self):
         self.assertEqual(
-            to_bytes(u'a\ufffdb', 'latin-1', errors='replace'),
+            to_bytes('a\ufffdb', 'latin-1', errors='replace'),
             b'a?b'
         )
 
@@ -96,7 +96,7 @@ class BinaryIsTextTest(unittest.TestCase):
         assert binary_is_text(b"hello")
 
     def test_utf_16_strings_contain_null_bytes(self):
-        assert binary_is_text(u"hello".encode('utf-16'))
+        assert binary_is_text("hello".encode('utf-16'))
 
     def test_one_with_encoding(self):
         assert binary_is_text(b"<div>Price \xa3</div>")
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 450e4bdca..de94ec960 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -22,7 +22,7 @@ class RequestSerializationTest(unittest.TestCase):
             method="POST",
             body=b"some body",
             headers={'content-encoding': 'text/html; charset=latin-1'},
-            cookies={'currency': u'руб'},
+            cookies={'currency': 'руб'},
             encoding='latin-1',
             priority=20,
             meta={'a': 'b'},
diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py
index 5a52dd695..5ff2e41ef 100644
--- a/tests/test_utils_template.py
+++ b/tests/test_utils_template.py
@@ -19,8 +19,8 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase):
     def test_simple_render(self):
 
         context = dict(project_name='proj', name='spi', classname='TheSpider')
-        template = u'from ${project_name}.spiders.${name} import ${classname}'
-        rendered = u'from proj.spiders.spi import TheSpider'
+        template = 'from ${project_name}.spiders.${name} import ${classname}'
+        rendered = 'from proj.spiders.spi import TheSpider'
 
         template_path = os.path.join(self.tmp_path, 'templ.py.tmpl')
         render_path = os.path.join(self.tmp_path, 'templ.py')

From 3600582f56071dd9f40f31ed44e09294a78dc13f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 4 Aug 2020 20:05:56 +0200
Subject: [PATCH 236/243] Cover Scrapy 2.2.1 and 2.3 in the release notes
 (#4708)

---
 docs/news.rst                   | 133 ++++++++++++++++++++++++++++++++
 docs/topics/commands.rst        |   2 +
 docs/topics/developer-tools.rst |   6 +-
 docs/topics/feed-exports.rst    |  36 ++++++++-
 scrapy/utils/curl.py            |   3 +-
 5 files changed, 176 insertions(+), 4 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 80d130e4a..850b323ef 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,139 @@
 Release notes
 =============
 
+.. _release-2.3.0:
+
+Scrapy 2.3.0 (2020-08-04)
+-------------------------
+
+Highlights:
+
+*   :ref:`Feed exports <topics-feed-exports>` now support :ref:`Google Cloud
+    Storage <topics-feed-storage-gcs>` as a storage backend
+
+*   The new :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting allows to deliver
+    output items in batches of up to the specified number of items.
+
+    It also serves as a workaround for :ref:`delayed file delivery
+    <delayed-file-delivery>`, which causes Scrapy to only start item delivery
+    after the crawl has finished when using certain storage backends
+    (:ref:`S3 <topics-feed-storage-s3>`, :ref:`FTP <topics-feed-storage-ftp>`,
+    and now :ref:`GCS <topics-feed-storage-gcs>`).
+
+*   The base implementation of :ref:`item loaders <topics-loaders>` has been
+    moved into a separate library, :doc:`itemloaders <itemloaders:index>`,
+    allowing usage from outside Scrapy and a separate release schedule
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+*   Removed the following classes and their parent modules from
+    ``scrapy.linkextractors``:
+
+    *   ``htmlparser.HtmlParserLinkExtractor``
+    *   ``regex.RegexLinkExtractor``
+    *   ``sgml.BaseSgmlLinkExtractor``
+    *   ``sgml.SgmlLinkExtractor``
+
+    Use
+    :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
+    instead (:issue:`4356`, :issue:`4679`)
+
+
+Deprecations
+~~~~~~~~~~~~
+
+*   The ``scrapy.utils.python.retry_on_eintr`` function is now deprecated
+    (:issue:`4683`)
+
+
+New features
+~~~~~~~~~~~~
+
+*   :ref:`Feed exports <topics-feed-exports>` support :ref:`Google Cloud
+    Storage <topics-feed-storage-gcs>` (:issue:`685`, :issue:`3608`)
+
+*   New :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting for batch deliveries
+    (:issue:`4250`, :issue:`4434`)
+
+*   The :command:`parse` command now allows specifying an output file
+    (:issue:`4317`, :issue:`4377`)
+
+*   :meth:`Request.from_curl <scrapy.http.Request.from_curl>` and
+    :func:`~scrapy.utils.curl.curl_to_request_kwargs` now also support
+    ``--data-raw`` (:issue:`4612`)
+
+*   A ``parse`` callback may now be used in built-in spider subclasses, such
+    as :class:`~scrapy.spiders.CrawlSpider` (:issue:`712`, :issue:`732`,
+    :issue:`781`, :issue:`4254` )
+
+
+Bug fixes
+~~~~~~~~~
+
+*   Fixed the :ref:`CSV exporting <topics-feed-format-csv>` of
+    :ref:`dataclass items <dataclass-items>` and :ref:`attr.s items
+    <attrs-items>` (:issue:`4667`, :issue:`4668`)
+
+*   :meth:`Request.from_curl <scrapy.http.Request.from_curl>` and
+    :func:`~scrapy.utils.curl.curl_to_request_kwargs` now set the request
+    method to ``POST`` when a request body is specified and no request method
+    is specified (:issue:`4612`)
+
+*   The processing of ANSI escape sequences in enabled in Windows 10.0.14393
+    and later, where it is required for colored output (:issue:`4393`,
+    :issue:`4403`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+*   Updated the `OpenSSL cipher list format`_ link in the documentation about
+    the :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` setting (:issue:`4653`)
+
+*   Simplified the code example in :ref:`topics-loaders-dataclass`
+    (:issue:`4652`)
+
+.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT
+
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+*   The base implementation of :ref:`item loaders <topics-loaders>` has been
+    moved into :doc:`itemloaders <itemloaders:index>` (:issue:`4005`,
+    :issue:`4516`)
+
+*   Fixed a silenced error in some scheduler tests (:issue:`4644`,
+    :issue:`4645`)
+
+*   Renewed the localhost certificate used for SSL tests (:issue:`4650`)
+
+*   Removed cookie-handling code specific to Python 2 (:issue:`4682`)
+
+*   Stopped using Python 2 unicode literal syntax (:issue:`4704`)
+
+*   Stopped using a backlash for line continuation (:issue:`4673`)
+
+*   Removed unneeded entries from the MyPy exception list (:issue:`4690`)
+
+*   Automated tests now pass on Windows as part of our continuous integration
+    system (:issue:`4458`)
+
+*   Automated tests now pass on the latest PyPy version for supported Python
+    versions in our continuous integration system (:issue:`4504`)
+
+
+.. _release-2.2.1:
+
+Scrapy 2.2.1 (2020-07-17)
+-------------------------
+
+*   The :command:`startproject` command no longer makes unintended changes to
+    the permissions of files in the destination folder, such as removing
+    execution permissions (:issue:`4662`, :issue:`4666`)
+
+
 .. _release-2.2.0:
 
 Scrapy 2.2.0 (2020-06-24)
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 4fce51abc..9638a2322 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -493,6 +493,8 @@ Supported options:
 
 * ``--output`` or ``-o``: dump scraped items to a file
 
+  .. versionadded:: 2.3
+
 .. skip: start
 
 Usage example::
diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index 4e87a00f2..101aa159c 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -289,8 +289,10 @@ request::
         "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'")
 
 Alternatively, if you want to know the arguments needed to recreate that
-request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs`
-function to get a dictionary with the equivalent arguments.
+request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs`
+function to get a dictionary with the equivalent arguments:
+
+.. autofunction:: scrapy.utils.curl.curl_to_request_kwargs
 
 Note that to translate a cURL command into a Scrapy request,
 you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index dd4eb3c61..37b7096f6 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -100,6 +100,7 @@ The storages backends supported out of the box are:
  * :ref:`topics-feed-storage-fs`
  * :ref:`topics-feed-storage-ftp`
  * :ref:`topics-feed-storage-s3` (requires botocore_)
+ * :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
  * :ref:`topics-feed-storage-stdout`
 
 Some storage backends may be unavailable if the required external libraries are
@@ -169,6 +170,9 @@ FTP supports two different connection modes: `active or passive
 mode by default. To use the active connection mode instead, set the
 :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
 
+This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
+
+
 .. _topics-feed-storage-s3:
 
 S3
@@ -194,11 +198,16 @@ You can also define a custom ACL for exported feeds using this setting:
 
  * :setting:`FEED_STORAGE_S3_ACL`
 
+This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
+
+
 .. _topics-feed-storage-gcs:
 
 Google Cloud Storage (GCS)
 --------------------------
 
+.. versionadded:: 2.3
+
 The feeds are stored on `Google Cloud Storage`_.
 
  * URI scheme: ``gs``
@@ -206,7 +215,7 @@ The feeds are stored on `Google Cloud Storage`_.
 
    * ``gs://mybucket/path/to/export.csv``
 
- * Required external libraries: `google-cloud-storage <https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python>`_.
+ * Required external libraries: `google-cloud-storage`_.
 
 For more information about authentication, please refer to `Google Cloud documentation <https://cloud.google.com/docs/authentication/production>`_.
 
@@ -215,6 +224,11 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following
  * :setting:`FEED_STORAGE_GCS_ACL`
  * :setting:`GCS_PROJECT_ID`
 
+This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
+
+.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
+
+
 .. _topics-feed-storage-stdout:
 
 Standard output
@@ -227,6 +241,26 @@ The feeds are written to the standard output of the Scrapy process.
  * Required external libraries: none
 
 
+.. _delayed-file-delivery:
+
+Delayed file delivery
+---------------------
+
+As indicated above, some of the described storage backends use delayed file
+delivery.
+
+These storage backends do not upload items to the feed URI as those items are
+scraped. Instead, Scrapy writes items into a temporary local file, and only
+once all the file contents have been written (i.e. at the end of the crawl) is
+that file uploaded to the feed URI.
+
+If you want item delivery to start earlier when using one of these storage
+backends, use :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` to split the output items
+in multiple files, with the specified maximum item count per file. That way, as
+soon as a file reaches the maximum item count, that file is delivered to the
+feed URI, allowing item delivery to start way before the end of the crawl.
+
+
 Settings
 ========
 
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index aa681522f..9c0efcec4 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -39,7 +39,8 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
 
     :param str curl_command: string containing the curl command
     :param bool ignore_unknown_options: If true, only a warning is emitted when
-    cURL options are unknown. Otherwise raises an error. (default: True)
+                                        cURL options are unknown. Otherwise
+                                        raises an error. (default: True)
     :return: dictionary of Request kwargs
     """
 

From 1278e76d9093b1c5c9ec810768d1066772a8a134 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 4 Aug 2020 20:07:02 +0200
Subject: [PATCH 237/243] =?UTF-8?q?Bump=20version:=202.2.0=20=E2=86=92=202?=
 =?UTF-8?q?.3.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 8d4d74bc5..3c1c8f891 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.2.0
+current_version = 2.3.0
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index ccbccc3dc..276cbf9e2 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-2.2.0
+2.3.0

From 4ee538e44b2650c49054b1f7f4c87ac70350471a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 4 Aug 2020 20:34:11 +0200
Subject: [PATCH 238/243] Update unicode references from Python 2 times in the
 documentation (#4703)

---
 docs/topics/exporters.rst        |  8 ++------
 docs/topics/loaders.rst          |  4 ++--
 docs/topics/request-response.rst | 30 ++++++++++++++++--------------
 docs/topics/selectors.rst        |  7 ++++---
 4 files changed, 24 insertions(+), 25 deletions(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index e5c99e5b1..8c84b85fc 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -166,8 +166,7 @@ BaseItemExporter
       By default, this method looks for a serializer :ref:`declared in the item
       field <topics-exporters-serializers>` and returns the result of applying
       that serializer to the value. If no serializer is found, it returns the
-      value unchanged except for ``unicode`` values which are encoded to
-      ``str`` using the encoding declared in the :attr:`encoding` attribute.
+      value unchanged.
 
       :param field: the field being serialized. If the source :ref:`item object
           <item-types>` does not define field metadata, *field* is an empty
@@ -217,10 +216,7 @@ BaseItemExporter
 
    .. attribute:: encoding
 
-      The encoding that will be used to encode unicode values. This only
-      affects unicode values (which are always serialized to str using this
-      encoding). Other value types are passed unchanged to the specific
-      serialization library.
+      The output character encoding.
 
    .. attribute:: indent
 
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 29d9c5805..c0f534493 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -193,10 +193,10 @@ Item Loaders are declared using a class definition syntax. Here is an example::
 
         default_output_processor = TakeFirst()
 
-        name_in = MapCompose(unicode.title)
+        name_in = MapCompose(str.title)
         name_out = Join()
 
-        price_in = MapCompose(unicode.strip)
+        price_in = MapCompose(str.strip)
 
         # ...
 
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index fbd8e4b73..1dffd1d55 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -51,12 +51,12 @@ Request objects
        given, the dict passed in this parameter will be shallow copied.
     :type meta: dict
 
-    :param body: the request body. If a ``unicode`` is passed, then it's encoded to
-      ``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If
-      ``body`` is not given, an empty string is stored. Regardless of the
-      type of this argument, the final value stored will be a ``str`` (never
-      ``unicode`` or ``None``).
-    :type body: str or unicode
+    :param body: the request body. If a string is passed, then it's encoded as
+      bytes using the ``encoding`` passed (which defaults to ``utf-8``). If
+      ``body`` is not given, an empty bytes object is stored. Regardless of the
+      type of this argument, the final value stored will be a bytes object
+      (never a string or ``None``).
+    :type body: bytes or str
 
     :param headers: the headers of this request. The dict values can be strings
        (for single valued headers) or lists (for multi-valued headers). If
@@ -106,7 +106,7 @@ Request objects
 
     :param encoding: the encoding of this request (defaults to ``'utf-8'``).
        This encoding will be used to percent-encode the URL and to convert the
-       body to ``str`` (if given as ``unicode``).
+       body to bytes (if given as a string).
     :type encoding: string
 
     :param priority: the priority of this request (defaults to ``0``).
@@ -721,7 +721,7 @@ Response objects
     .. attribute:: Response.body
 
         The body of this Response. Keep in mind that Response.body
-        is always a bytes object. If you want the unicode version use
+        is always a bytes object. If you want the string version use
         :attr:`TextResponse.text` (only available in :class:`TextResponse`
         and subclasses).
 
@@ -842,9 +842,9 @@ TextResponse objects
     is the same as for the :class:`Response` class and is not documented here.
 
     :param encoding: is a string which contains the encoding to use for this
-       response. If you create a :class:`TextResponse` object with a unicode
+       response. If you create a :class:`TextResponse` object with a string as
        body, it will be encoded using this encoding (remember the body attribute
-       is always a string). If ``encoding`` is ``None`` (default value), the
+       is always a bytes object). If ``encoding`` is ``None`` (default value), the
        encoding will be looked up in the response headers and body instead.
     :type encoding: string
 
@@ -853,7 +853,7 @@ TextResponse objects
 
     .. attribute:: TextResponse.text
 
-       Response body, as unicode.
+       Response body, as a string.
 
        The same as ``response.body.decode(response.encoding)``, but the
        result is cached after the first call, so you can access
@@ -861,9 +861,11 @@ TextResponse objects
 
        .. note::
 
-            ``unicode(response.body)`` is not a correct way to convert response
-            body to unicode: you would be using the system default encoding
-            (typically ``ascii``) instead of the response encoding.
+            ``str(response.body)`` is not a correct way to convert the response
+            body into a string:
+
+            >>> str(b'body')
+            "b'body'"
 
 
     .. attribute:: TextResponse.encoding
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 5014df6ac..9e2c6ba42 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -64,7 +64,8 @@ more shortcuts: ``response.xpath()`` and ``response.css()``:
 
 Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
 constructed by passing either :class:`~scrapy.http.TextResponse` object or
-markup as an unicode string (in ``text`` argument).
+markup as a string (in ``text`` argument).
+
 Usually there is no need to construct Scrapy selectors manually:
 ``response`` object is available in Spider callbacks, so in most cases
 it is more convenient to use ``response.css()`` and ``response.xpath()``
@@ -383,7 +384,7 @@ Using selectors with regular expressions
 
 :class:`~scrapy.selector.Selector` also has a ``.re()`` method for extracting
 data using regular expressions. However, unlike using ``.xpath()`` or
-``.css()`` methods, ``.re()`` returns a list of unicode strings. So you
+``.css()`` methods, ``.re()`` returns a list of strings. So you
 can't construct nested ``.re()`` calls.
 
 Here's an example used to extract image names from the :ref:`HTML code
@@ -989,7 +990,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this::
       sel.xpath("//h1")
 
 2. Extract the text of all ``<h1>`` elements from an HTML response body,
-   returning a list of unicode strings::
+   returning a list of strings::
 
       sel.xpath("//h1").getall()         # this includes the h1 tag
       sel.xpath("//h1/text()").getall()  # this excludes the h1 tag

From 336f19f5cc6edd0392c77f38b857d3e40bf565da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com>
Date: Tue, 4 Aug 2020 20:42:01 +0200
Subject: [PATCH 239/243] Change super syntax (#4707)

---
 extras/qpsclient.py                          |  2 +-
 scrapy/commands/view.py                      |  2 +-
 scrapy/contracts/default.py                  |  2 +-
 scrapy/core/downloader/contextfactory.py     |  4 ++--
 scrapy/core/downloader/handlers/http11.py    | 10 +++++-----
 scrapy/core/downloader/tls.py                |  2 +-
 scrapy/core/spidermw.py                      |  2 +-
 scrapy/crawler.py                            |  2 +-
 scrapy/downloadermiddlewares/redirect.py     |  2 +-
 scrapy/exceptions.py                         |  4 ++--
 scrapy/exporters.py                          |  2 +-
 scrapy/http/headers.py                       |  8 ++++----
 scrapy/http/request/form.py                  |  2 +-
 scrapy/http/request/json_request.py          |  4 ++--
 scrapy/http/request/rpc.py                   |  2 +-
 scrapy/http/response/text.py                 | 10 +++++-----
 scrapy/item.py                               | 10 +++++-----
 scrapy/linkextractors/__init__.py            |  2 +-
 scrapy/linkextractors/lxmlhtml.py            |  2 +-
 scrapy/pipelines/files.py                    |  2 +-
 scrapy/pipelines/images.py                   |  3 +--
 scrapy/resolver.py                           | 10 +++++-----
 scrapy/selector/unified.py                   |  2 +-
 scrapy/settings/__init__.py                  |  2 +-
 scrapy/spidermiddlewares/httperror.py        |  2 +-
 scrapy/spiders/crawl.py                      |  4 ++--
 scrapy/spiders/init.py                       |  2 +-
 scrapy/spiders/sitemap.py                    |  2 +-
 scrapy/squeues.py                            | 12 ++++++------
 scrapy/statscollectors.py                    |  2 +-
 scrapy/utils/datatypes.py                    | 14 +++++++-------
 scrapy/utils/deprecate.py                    |  8 ++++----
 scrapy/utils/log.py                          |  2 +-
 scrapy/utils/serialize.py                    |  2 +-
 scrapy/utils/testsite.py                     |  4 ++--
 tests/spiders.py                             | 20 ++++++++++----------
 tests/test_command_parse.py                  |  2 +-
 tests/test_commands.py                       |  4 ++--
 tests/test_contracts.py                      |  2 +-
 tests/test_downloader_handlers.py            |  4 ++--
 tests/test_downloadermiddleware_httpcache.py |  4 ++--
 tests/test_downloadermiddleware_robotstxt.py |  4 ++--
 tests/test_exporters.py                      |  2 +-
 tests/test_http_request.py                   |  4 ++--
 tests/test_http_response.py                  |  2 +-
 tests/test_item.py                           |  2 +-
 tests/test_linkextractors.py                 |  2 +-
 tests/test_loader.py                         |  2 +-
 tests/test_loader_deprecated.py              |  2 +-
 tests/test_logformatter.py                   |  2 +-
 tests/test_middleware.py                     |  2 +-
 tests/test_pipeline_media.py                 | 12 ++++++------
 tests/test_request_left.py                   |  2 +-
 tests/test_robotstxt_interface.py            |  8 ++++----
 tests/test_scheduler.py                      |  4 ++--
 tests/test_spidermiddleware_httperror.py     |  2 +-
 56 files changed, 117 insertions(+), 118 deletions(-)

diff --git a/extras/qpsclient.py b/extras/qpsclient.py
index 7554f7eec..fe1f96cbb 100644
--- a/extras/qpsclient.py
+++ b/extras/qpsclient.py
@@ -27,7 +27,7 @@ class QPSSpider(Spider):
     slots = 1
 
     def __init__(self, *a, **kw):
-        super(QPSSpider, self).__init__(*a, **kw)
+        super().__init__(*a, **kw)
         if self.qps is not None:
             self.qps = float(self.qps)
             self.download_delay = 1 / self.qps
diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py
index 908bee966..c8f873334 100644
--- a/scrapy/commands/view.py
+++ b/scrapy/commands/view.py
@@ -11,7 +11,7 @@ class Command(fetch.Command):
         return "Fetch a URL using the Scrapy downloader and show its contents in a browser"
 
     def add_options(self, parser):
-        super(Command, self).add_options(parser)
+        super().add_options(parser)
         parser.remove_option("--headers")
 
     def _print_response(self, response, opts):
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 34f0d36d4..cfdcc7c25 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -56,7 +56,7 @@ class ReturnsContract(Contract):
     }
 
     def __init__(self, *args, **kwargs):
-        super(ReturnsContract, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
         if len(self.args) not in [1, 2, 3]:
             raise ValueError(
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 452242d47..8a7d656a1 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -20,7 +20,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
     """
 
     def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
-        super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self._ssl_method = method
         self.tls_verbose_logging = tls_verbose_logging
         if tls_ciphers:
@@ -45,7 +45,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
         #   (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
         #
         # * getattr() for `_ssl_method` attribute for context factories
-        #   not calling super(..., self).__init__
+        #   not calling super().__init__
         return CertificateOptions(
             verify=False,
             method=getattr(self, 'method', getattr(self, '_ssl_method', None)),
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 22c9ac520..fb04d1fb7 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -126,7 +126,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
 
     def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None):
         proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
-        super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
+        super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
         self._tunnelReadyDeferred = defer.Deferred()
         self._tunneledHost = host
         self._tunneledPort = port
@@ -178,7 +178,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
 
     def connect(self, protocolFactory):
         self._protocolFactory = protocolFactory
-        connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory)
+        connectDeferred = super().connect(protocolFactory)
         connectDeferred.addCallback(self.requestTunnel)
         connectDeferred.addErrback(self.connectFailed)
         return self._tunnelReadyDeferred
@@ -215,7 +215,7 @@ class TunnelingAgent(Agent):
 
     def __init__(self, reactor, proxyConf, contextFactory=None,
                  connectTimeout=None, bindAddress=None, pool=None):
-        super(TunnelingAgent, self).__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
+        super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
         self._proxyConf = proxyConf
         self._contextFactory = contextFactory
 
@@ -235,7 +235,7 @@ class TunnelingAgent(Agent):
         # otherwise, same remote host connection request could reuse
         # a cached tunneled connection to a different proxy
         key = key + self._proxyConf
-        return super(TunnelingAgent, self)._requestWithEndpoint(
+        return super()._requestWithEndpoint(
             key=key,
             endpoint=endpoint,
             method=method,
@@ -249,7 +249,7 @@ class TunnelingAgent(Agent):
 class ScrapyProxyAgent(Agent):
 
     def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None):
-        super(ScrapyProxyAgent, self).__init__(
+        super().__init__(
             reactor=reactor,
             connectTimeout=connectTimeout,
             bindAddress=bindAddress,
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index e43a3c83e..d9f3750d5 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -47,7 +47,7 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
     """
 
     def __init__(self, hostname, ctx, verbose_logging=False):
-        super(ScrapyClientTLSOptions, self).__init__(hostname, ctx)
+        super().__init__(hostname, ctx)
         self.verbose_logging = verbose_logging
 
     def _identityVerifyingInfoCallback(self, connection, where, ret):
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 35264a92b..5a99b96be 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -34,7 +34,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
 
     def _add_middleware(self, mw):
-        super(SpiderMiddlewareManager, self)._add_middleware(mw)
+        super()._add_middleware(mw)
         if hasattr(mw, 'process_spider_input'):
             self.methods['process_spider_input'].append(mw.process_spider_input)
         if hasattr(mw, 'process_start_requests'):
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 6f43771e2..48f19424c 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -277,7 +277,7 @@ class CrawlerProcess(CrawlerRunner):
     """
 
     def __init__(self, settings=None, install_root_handler=True):
-        super(CrawlerProcess, self).__init__(settings)
+        super().__init__(settings)
         install_shutdown_handlers(self._signal_shutdown)
         configure_logging(self.settings, install_root_handler)
         log_scrapy_info(self.settings)
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 366d60dcb..4053fecc5 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -92,7 +92,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
     enabled_setting = 'METAREFRESH_ENABLED'
 
     def __init__(self, settings):
-        super(MetaRefreshMiddleware, self).__init__(settings)
+        super().__init__(settings)
         self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS')
         self._maxdelay = settings.getint('METAREFRESH_MAXDELAY')
 
diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py
index 45f152321..0c410f035 100644
--- a/scrapy/exceptions.py
+++ b/scrapy/exceptions.py
@@ -37,7 +37,7 @@ class CloseSpider(Exception):
     """Raise this from callbacks to request the spider to be closed"""
 
     def __init__(self, reason='cancelled'):
-        super(CloseSpider, self).__init__()
+        super().__init__()
         self.reason = reason
 
 
@@ -74,7 +74,7 @@ class UsageError(Exception):
 
     def __init__(self, *a, **kw):
         self.print_help = kw.pop('print_help', True)
-        super(UsageError, self).__init__(*a, **kw)
+        super().__init__(*a, **kw)
 
 
 class ScrapyDeprecationWarning(Warning):
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 0aba1c904..95518b3ac 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -301,7 +301,7 @@ class PythonItemExporter(BaseItemExporter):
 
     def _configure(self, options, dont_fail=False):
         self.binary = options.pop('binary', True)
-        super(PythonItemExporter, self)._configure(options, dont_fail)
+        super()._configure(options, dont_fail)
         if self.binary:
             warnings.warn(
                 "PythonItemExporter will drop support for binary export in the future",
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index dcaaeddfa..6bf9e5346 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -8,7 +8,7 @@ class Headers(CaselessDict):
 
     def __init__(self, seq=None, encoding='utf-8'):
         self.encoding = encoding
-        super(Headers, self).__init__(seq)
+        super().__init__(seq)
 
     def normkey(self, key):
         """Normalize key to bytes"""
@@ -37,19 +37,19 @@ class Headers(CaselessDict):
 
     def __getitem__(self, key):
         try:
-            return super(Headers, self).__getitem__(key)[-1]
+            return super().__getitem__(key)[-1]
         except IndexError:
             return None
 
     def get(self, key, def_val=None):
         try:
-            return super(Headers, self).get(key, def_val)[-1]
+            return super().get(key, def_val)[-1]
         except IndexError:
             return None
 
     def getlist(self, key, def_val=None):
         try:
-            return super(Headers, self).__getitem__(key)
+            return super().__getitem__(key)
         except KeyError:
             if def_val is not None:
                 return self.normvalue(def_val)
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index a260798ac..59af81321 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -24,7 +24,7 @@ class FormRequest(Request):
         if formdata and kwargs.get('method') is None:
             kwargs['method'] = 'POST'
 
-        super(FormRequest, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
         if formdata:
             items = formdata.items() if isinstance(formdata, dict) else formdata
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index f08b25280..eae3f9f6b 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -32,7 +32,7 @@ class JsonRequest(Request):
             if 'method' not in kwargs:
                 kwargs['method'] = 'POST'
 
-        super(JsonRequest, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.headers.setdefault('Content-Type', 'application/json')
         self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
 
@@ -47,7 +47,7 @@ class JsonRequest(Request):
         elif not body_passed and data_passed:
             kwargs['body'] = self._dumps(data)
 
-        return super(JsonRequest, self).replace(*args, **kwargs)
+        return super().replace(*args, **kwargs)
 
     def _dumps(self, data):
         """Convert to JSON """
diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py
index 811d3ad6b..c70912e49 100644
--- a/scrapy/http/request/rpc.py
+++ b/scrapy/http/request/rpc.py
@@ -31,5 +31,5 @@ class XmlRpcRequest(Request):
         if encoding is not None:
             kwargs['encoding'] = encoding
 
-        super(XmlRpcRequest, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.headers.setdefault('Content-Type', 'text/xml')
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 0f300c8da..a7bb34d48 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -35,13 +35,13 @@ class TextResponse(Response):
         self._cached_benc = None
         self._cached_ubody = None
         self._cached_selector = None
-        super(TextResponse, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
     def _set_url(self, url):
         if isinstance(url, str):
             self._url = to_unicode(url, self.encoding)
         else:
-            super(TextResponse, self)._set_url(url)
+            super()._set_url(url)
 
     def _set_body(self, body):
         self._body = b''  # used by encoding detection
@@ -51,7 +51,7 @@ class TextResponse(Response):
                                 type(self).__name__)
             self._body = body.encode(self._encoding)
         else:
-            super(TextResponse, self)._set_body(body)
+            super()._set_body(body)
 
     def replace(self, *args, **kwargs):
         kwargs.setdefault('encoding', self.encoding)
@@ -166,7 +166,7 @@ class TextResponse(Response):
         elif isinstance(url, parsel.SelectorList):
             raise ValueError("SelectorList is not supported")
         encoding = self.encoding if encoding is None else encoding
-        return super(TextResponse, self).follow(
+        return super().follow(
             url=url,
             callback=callback,
             method=method,
@@ -226,7 +226,7 @@ class TextResponse(Response):
             for sel in selectors:
                 with suppress(_InvalidSelector):
                     urls.append(_url_from_selector(sel))
-        return super(TextResponse, self).follow_all(
+        return super().follow_all(
             urls=urls,
             callback=callback,
             method=method,
diff --git a/scrapy/item.py b/scrapy/item.py
index 4ab83d1a0..c262a153c 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -39,7 +39,7 @@ class BaseItem(_BaseItem, metaclass=_BaseItemMeta):
         if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)):
             warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
                  ScrapyDeprecationWarning, stacklevel=2)
-        return super(BaseItem, cls).__new__(cls, *args, **kwargs)
+        return super().__new__(cls, *args, **kwargs)
 
 
 class Field(dict):
@@ -55,7 +55,7 @@ class ItemMeta(_BaseItemMeta):
     def __new__(mcs, class_name, bases, attrs):
         classcell = attrs.pop('__classcell__', None)
         new_bases = tuple(base._class for base in bases if hasattr(base, '_class'))
-        _class = super(ItemMeta, mcs).__new__(mcs, 'x_' + class_name, new_bases, attrs)
+        _class = super().__new__(mcs, 'x_' + class_name, new_bases, attrs)
 
         fields = getattr(_class, 'fields', {})
         new_attrs = {}
@@ -70,7 +70,7 @@ class ItemMeta(_BaseItemMeta):
         new_attrs['_class'] = _class
         if classcell is not None:
             new_attrs['__classcell__'] = classcell
-        return super(ItemMeta, mcs).__new__(mcs, class_name, bases, new_attrs)
+        return super().__new__(mcs, class_name, bases, new_attrs)
 
 
 class DictItem(MutableMapping, BaseItem):
@@ -81,7 +81,7 @@ class DictItem(MutableMapping, BaseItem):
         if issubclass(cls, DictItem) and not issubclass(cls, Item):
             warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead',
                  ScrapyDeprecationWarning, stacklevel=2)
-        return super(DictItem, cls).__new__(cls, *args, **kwargs)
+        return super().__new__(cls, *args, **kwargs)
 
     def __init__(self, *args, **kwargs):
         self._values = {}
@@ -109,7 +109,7 @@ class DictItem(MutableMapping, BaseItem):
     def __setattr__(self, name, value):
         if not name.startswith('_'):
             raise AttributeError("Use item[%r] = %r to set field value" % (name, value))
-        super(DictItem, self).__setattr__(name, value)
+        super().__setattr__(name, value)
 
     def __len__(self):
         return len(self._values)
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 984a5c4e1..08a6ca1e8 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -65,7 +65,7 @@ class FilteringLinkExtractor:
             warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
                  'please use scrapy.linkextractors.LinkExtractor instead',
                  ScrapyDeprecationWarning, stacklevel=2)
-        return super(FilteringLinkExtractor, cls).__new__(cls)
+        return super().__new__(cls)
 
     def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,
                  restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text):
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 8b9f961ee..e941c4321 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -126,7 +126,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
             strip=strip,
             canonicalized=canonicalize
         )
-        super(LxmlLinkExtractor, self).__init__(
+        super().__init__(
             link_extractor=lx,
             allow=allow,
             deny=deny,
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 487382a38..6bc5d46eb 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -376,7 +376,7 @@ class FilesPipeline(MediaPipeline):
             resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD
         )
 
-        super(FilesPipeline, self).__init__(download_func=download_func, settings=settings)
+        super().__init__(download_func=download_func, settings=settings)
 
     @classmethod
     def from_settings(cls, settings):
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 46f2bfb58..e2dd70215 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -45,8 +45,7 @@ class ImagesPipeline(FilesPipeline):
     DEFAULT_IMAGES_RESULT_FIELD = 'images'
 
     def __init__(self, store_uri, download_func=None, settings=None):
-        super(ImagesPipeline, self).__init__(store_uri, settings=settings,
-                                             download_func=download_func)
+        super().__init__(store_uri, settings=settings, download_func=download_func)
 
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index f69894b1e..f191deac6 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -17,7 +17,7 @@ class CachingThreadedResolver(ThreadedResolver):
     """
 
     def __init__(self, reactor, cache_size, timeout):
-        super(CachingThreadedResolver, self).__init__(reactor)
+        super().__init__(reactor)
         dnscache.limit = cache_size
         self.timeout = timeout
 
@@ -40,7 +40,7 @@ class CachingThreadedResolver(ThreadedResolver):
         # so the input argument above is simply overridden
         # to enforce Scrapy's DNS_TIMEOUT setting's value
         timeout = (self.timeout,)
-        d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
+        d = super().getHostByName(name, timeout)
         if dnscache.limit:
             d.addCallback(self._cache_result, name)
         return d
@@ -80,16 +80,16 @@ class CachingHostnameResolver:
         class CachingResolutionReceiver(resolutionReceiver):
 
             def resolutionBegan(self, resolution):
-                super(CachingResolutionReceiver, self).resolutionBegan(resolution)
+                super().resolutionBegan(resolution)
                 self.resolution = resolution
                 self.resolved = False
 
             def addressResolved(self, address):
-                super(CachingResolutionReceiver, self).addressResolved(address)
+                super().addressResolved(address)
                 self.resolved = True
 
             def resolutionComplete(self):
-                super(CachingResolutionReceiver, self).resolutionComplete()
+                super().resolutionComplete()
                 if self.resolved:
                     dnscache[hostName] = self.resolution
 
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 85a9bb526..f12c61081 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -79,4 +79,4 @@ class Selector(_ParselSelector, object_ref):
             kwargs.setdefault('base_url', response.url)
 
         self.response = response
-        super(Selector, self).__init__(text=text, type=st, root=root, **kwargs)
+        super().__init__(text=text, type=st, root=root, **kwargs)
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index ff8317cd1..b8ae32d7c 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -439,7 +439,7 @@ class Settings(BaseSettings):
         # Do not pass kwarg values here. We don't want to promote user-defined
         # dicts, and we want to update, not replace, default dicts with the
         # values given by the user
-        super(Settings, self).__init__()
+        super().__init__()
         self.setmodule(default_settings, 'default')
         # Promote default dictionaries to BaseSettings instances for per-key
         # priorities
diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py
index 375042340..db9d0f2ae 100644
--- a/scrapy/spidermiddlewares/httperror.py
+++ b/scrapy/spidermiddlewares/httperror.py
@@ -15,7 +15,7 @@ class HttpError(IgnoreRequest):
 
     def __init__(self, response, *args, **kwargs):
         self.response = response
-        super(HttpError, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
 
 class HttpErrorMiddleware:
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index cb7260892..c9fbce08d 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -75,7 +75,7 @@ class CrawlSpider(Spider):
     rules = ()
 
     def __init__(self, *a, **kw):
-        super(CrawlSpider, self).__init__(*a, **kw)
+        super().__init__(*a, **kw)
         self._compile_rules()
 
     def _parse(self, response, **kwargs):
@@ -145,6 +145,6 @@ class CrawlSpider(Spider):
 
     @classmethod
     def from_crawler(cls, crawler, *args, **kwargs):
-        spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs)
+        spider = super().from_crawler(crawler, *args, **kwargs)
         spider._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)
         return spider
diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py
index fd41133ea..fe8c94e78 100644
--- a/scrapy/spiders/init.py
+++ b/scrapy/spiders/init.py
@@ -6,7 +6,7 @@ class InitSpider(Spider):
     """Base Spider with initialization facilities"""
 
     def start_requests(self):
-        self._postinit_reqs = super(InitSpider, self).start_requests()
+        self._postinit_reqs = super().start_requests()
         return iterate_spider_output(self.init_request())
 
     def initialized(self, response=None):
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index c5360bfa7..1f72e76b7 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -18,7 +18,7 @@ class SitemapSpider(Spider):
     sitemap_alternate_links = False
 
     def __init__(self, *a, **kw):
-        super(SitemapSpider, self).__init__(*a, **kw)
+        super().__init__(*a, **kw)
         self._cbs = []
         for r, c in self.sitemap_rules:
             if isinstance(c, str):
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index c7ad4d53d..77ffda6f7 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -20,7 +20,7 @@ def _with_mkdir(queue_class):
             if not os.path.exists(dirname):
                 os.makedirs(dirname, exist_ok=True)
 
-            super(DirectoriesCreated, self).__init__(path, *args, **kwargs)
+            super().__init__(path, *args, **kwargs)
 
     return DirectoriesCreated
 
@@ -31,10 +31,10 @@ def _serializable_queue(queue_class, serialize, deserialize):
 
         def push(self, obj):
             s = serialize(obj)
-            super(SerializableQueue, self).push(s)
+            super().push(s)
 
         def pop(self):
-            s = super(SerializableQueue, self).pop()
+            s = super().pop()
             if s:
                 return deserialize(s)
 
@@ -47,7 +47,7 @@ def _scrapy_serialization_queue(queue_class):
 
         def __init__(self, crawler, key):
             self.spider = crawler.spider
-            super(ScrapyRequestQueue, self).__init__(key)
+            super().__init__(key)
 
         @classmethod
         def from_crawler(cls, crawler, key, *args, **kwargs):
@@ -55,10 +55,10 @@ def _scrapy_serialization_queue(queue_class):
 
         def push(self, request):
             request = request_to_dict(request, self.spider)
-            return super(ScrapyRequestQueue, self).push(request)
+            return super().push(request)
 
         def pop(self):
-            request = super(ScrapyRequestQueue, self).pop()
+            request = super().pop()
 
             if not request:
                 return None
diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py
index 579c60180..ba7d1a6bf 100644
--- a/scrapy/statscollectors.py
+++ b/scrapy/statscollectors.py
@@ -54,7 +54,7 @@ class StatsCollector:
 class MemoryStatsCollector(StatsCollector):
 
     def __init__(self, crawler):
-        super(MemoryStatsCollector, self).__init__(crawler)
+        super().__init__(crawler)
         self.spider_stats = {}
 
     def _persist_stats(self, stats, spider):
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 2a92d0588..e31284a7f 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -15,7 +15,7 @@ class CaselessDict(dict):
     __slots__ = ()
 
     def __init__(self, seq=None):
-        super(CaselessDict, self).__init__()
+        super().__init__()
         if seq:
             self.update(seq)
 
@@ -53,7 +53,7 @@ class CaselessDict(dict):
     def update(self, seq):
         seq = seq.items() if isinstance(seq, Mapping) else seq
         iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
-        super(CaselessDict, self).update(iseq)
+        super().update(iseq)
 
     @classmethod
     def fromkeys(cls, keys, value=None):
@@ -70,14 +70,14 @@ class LocalCache(collections.OrderedDict):
     """
 
     def __init__(self, limit=None):
-        super(LocalCache, self).__init__()
+        super().__init__()
         self.limit = limit
 
     def __setitem__(self, key, value):
         if self.limit:
             while len(self) >= self.limit:
                 self.popitem(last=False)
-        super(LocalCache, self).__setitem__(key, value)
+        super().__setitem__(key, value)
 
 
 class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
@@ -93,18 +93,18 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
     """
 
     def __init__(self, limit=None):
-        super(LocalWeakReferencedCache, self).__init__()
+        super().__init__()
         self.data = LocalCache(limit=limit)
 
     def __setitem__(self, key, value):
         try:
-            super(LocalWeakReferencedCache, self).__setitem__(key, value)
+            super().__setitem__(key, value)
         except TypeError:
             pass  # key is not weak-referenceable, skip caching
 
     def __getitem__(self, key):
         try:
-            return super(LocalWeakReferencedCache, self).__getitem__(key)
+            return super().__getitem__(key)
         except (TypeError, KeyError):
             return None  # key is either not weak-referenceable or not cached
 
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index 3dbea5fee..3c8e3c8b5 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -57,7 +57,7 @@ def create_deprecated_class(
         warned_on_subclass = False
 
         def __new__(metacls, name, bases, clsdict_):
-            cls = super(DeprecatedClass, metacls).__new__(metacls, name, bases, clsdict_)
+            cls = super().__new__(metacls, name, bases, clsdict_)
             if metacls.deprecated_class is None:
                 metacls.deprecated_class = cls
             return cls
@@ -73,7 +73,7 @@ def create_deprecated_class(
                 if warn_once:
                     msg += ' (warning only on first subclass, there may be others)'
                 warnings.warn(msg, warn_category, stacklevel=2)
-            super(DeprecatedClass, cls).__init__(name, bases, clsdict_)
+            super().__init__(name, bases, clsdict_)
 
         # see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass
         # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks
@@ -88,7 +88,7 @@ def create_deprecated_class(
                 # is the deprecated class itself - subclasses of the
                 # deprecated class should not use custom `__subclasscheck__`
                 # method.
-                return super(DeprecatedClass, cls).__subclasscheck__(sub)
+                return super().__subclasscheck__(sub)
 
             if not inspect.isclass(sub):
                 raise TypeError("issubclass() arg 1 must be a class")
@@ -102,7 +102,7 @@ def create_deprecated_class(
                 msg = instance_warn_message.format(cls=_clspath(cls, old_class_path),
                                                    new=_clspath(new_class, new_class_path))
                 warnings.warn(msg, warn_category, stacklevel=2)
-            return super(DeprecatedClass, cls).__call__(*args, **kwargs)
+            return super().__call__(*args, **kwargs)
 
     deprecated_cls = DeprecatedClass(name, (new_class,), clsdict or {})
 
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 51d276097..1d6a2c39d 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -176,7 +176,7 @@ class LogCounterHandler(logging.Handler):
     """Record log levels count into a crawler stats"""
 
     def __init__(self, crawler, *args, **kwargs):
-        super(LogCounterHandler, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.crawler = crawler
 
     def emit(self, record):
diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py
index dc9604578..cc3263602 100644
--- a/scrapy/utils/serialize.py
+++ b/scrapy/utils/serialize.py
@@ -33,7 +33,7 @@ class ScrapyJSONEncoder(json.JSONEncoder):
         elif isinstance(o, Response):
             return "<%s %s %s>" % (type(o).__name__, o.status, o.url)
         else:
-            return super(ScrapyJSONEncoder, self).default(o)
+            return super().default(o)
 
 
 class ScrapyJSONDecoder(json.JSONDecoder):
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index 66930ad2c..397e54703 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -7,12 +7,12 @@ class SiteTest:
 
     def setUp(self):
         from twisted.internet import reactor
-        super(SiteTest, self).setUp()
+        super().setUp()
         self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
         self.baseurl = "http://localhost:%d/" % self.site.getHost().port
 
     def tearDown(self):
-        super(SiteTest, self).tearDown()
+        super().tearDown()
         self.site.stopListening()
 
     def url(self, path):
diff --git a/tests/spiders.py b/tests/spiders.py
index 3eb681819..63bd726fb 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -19,7 +19,7 @@ from scrapy.utils.test import get_from_asyncio_queue
 
 class MockServerSpider(Spider):
     def __init__(self, mockserver=None, *args, **kwargs):
-        super(MockServerSpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.mockserver = mockserver
 
 
@@ -28,7 +28,7 @@ class MetaSpider(MockServerSpider):
     name = 'meta'
 
     def __init__(self, *args, **kwargs):
-        super(MetaSpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.meta = {}
 
     def closed(self, reason):
@@ -41,7 +41,7 @@ class FollowAllSpider(MetaSpider):
     link_extractor = LinkExtractor()
 
     def __init__(self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs):
-        super(FollowAllSpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.urls_visited = []
         self.times = []
         qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency}
@@ -60,7 +60,7 @@ class DelaySpider(MetaSpider):
     name = 'delay'
 
     def __init__(self, n=1, b=0, *args, **kwargs):
-        super(DelaySpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.n = n
         self.b = b
         self.t1 = self.t2 = self.t2_err = 0
@@ -82,7 +82,7 @@ class SimpleSpider(MetaSpider):
     name = 'simple'
 
     def __init__(self, url="http://localhost:8998", *args, **kwargs):
-        super(SimpleSpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.start_urls = [url]
 
     def parse(self, response):
@@ -153,7 +153,7 @@ class ItemSpider(FollowAllSpider):
     name = 'item'
 
     def parse(self, response):
-        for request in super(ItemSpider, self).parse(response):
+        for request in super().parse(response):
             yield request
             yield Item()
             yield {}
@@ -172,7 +172,7 @@ class ErrorSpider(FollowAllSpider):
         raise self.exception_cls('Expected exception')
 
     def parse(self, response):
-        for request in super(ErrorSpider, self).parse(response):
+        for request in super().parse(response):
             yield request
             self.raise_exception()
 
@@ -183,7 +183,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
     fail_yielding = False
 
     def __init__(self, *a, **kw):
-        super(BrokenStartRequestsSpider, self).__init__(*a, **kw)
+        super().__init__(*a, **kw)
         self.seedsseen = []
 
     def start_requests(self):
@@ -201,7 +201,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
 
     def parse(self, response):
         self.seedsseen.append(response.meta.get('seed'))
-        for req in super(BrokenStartRequestsSpider, self).parse(response):
+        for req in super().parse(response):
             yield req
 
 
@@ -243,7 +243,7 @@ class DuplicateStartRequestsSpider(MockServerSpider):
                 yield Request(url, dont_filter=self.dont_filter)
 
     def __init__(self, url="http://localhost:8998", *args, **kwargs):
-        super(DuplicateStartRequestsSpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.visited = 0
 
     def parse(self, response):
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 5754a5478..e115f420f 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -17,7 +17,7 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
     command = 'parse'
 
     def setUp(self):
-        super(ParseCommandTest, self).setUp()
+        super().setUp()
         self.spider_name = 'parse_spider'
         fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py'))
         with open(fname, 'w') as f:
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 42091ab00..8938156fc 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -151,7 +151,7 @@ def get_permissions_dict(path, renamings=None, ignore=None):
 class StartprojectTemplatesTest(ProjectTest):
 
     def setUp(self):
-        super(StartprojectTemplatesTest, self).setUp()
+        super().setUp()
         self.tmpl = join(self.temp_path, 'templates')
         self.tmpl_proj = join(self.tmpl, 'project')
 
@@ -315,7 +315,7 @@ class StartprojectTemplatesTest(ProjectTest):
 class CommandTest(ProjectTest):
 
     def setUp(self):
-        super(CommandTest, self).setUp()
+        super().setUp()
         self.call('startproject', self.project_name)
         self.cwd = join(self.temp_path, self.project_name)
         self.env['SCRAPY_SETTINGS_MODULE'] = '%s.settings' % self.project_name
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 99120b128..2e7e3ccc4 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -378,7 +378,7 @@ class ContractsManagerTest(unittest.TestCase):
             name = 'test_same_url'
 
             def __init__(self, *args, **kwargs):
-                super(TestSameUrlSpider, self).__init__(*args, **kwargs)
+                super().__init__(*args, **kwargs)
                 self.visited = 0
 
             def start_requests(s):
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 57d4cdd6b..13063d106 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -530,7 +530,7 @@ class Https11InvalidDNSId(Https11TestCase):
     """Connect to HTTPS hosts with IP while certificate uses domain names IDs."""
 
     def setUp(self):
-        super(Https11InvalidDNSId, self).setUp()
+        super().setUp()
         self.host = '127.0.0.1'
 
 
@@ -549,7 +549,7 @@ class Https11InvalidDNSPattern(Https11TestCase):
             'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", '
             'subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
         )
-        super(Https11InvalidDNSPattern, self).setUp()
+        super().setUp()
 
 
 class Https11CustomCiphers(unittest.TestCase):
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 9b77c97a8..299fb0eb8 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -134,7 +134,7 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest):
 
     def _get_settings(self, **new_settings):
         new_settings.setdefault('HTTPCACHE_DBM_MODULE', self.dbm_module)
-        return super(DbmStorageWithCustomDbmModuleTest, self)._get_settings(**new_settings)
+        return super()._get_settings(**new_settings)
 
     def test_custom_dbm_module_loaded(self):
         # make sure our dbm module has been loaded
@@ -151,7 +151,7 @@ class FilesystemStorageGzipTest(FilesystemStorageTest):
 
     def _get_settings(self, **new_settings):
         new_settings.setdefault('HTTPCACHE_GZIP', True)
-        return super(FilesystemStorageTest, self)._get_settings(**new_settings)
+        return super()._get_settings(**new_settings)
 
 
 class DummyPolicyTest(_BaseTest):
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index f9936baba..858138f81 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -189,7 +189,7 @@ class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest):
         skip = "Rerp parser is not installed"
 
     def setUp(self):
-        super(RobotsTxtMiddlewareWithRerpTest, self).setUp()
+        super().setUp()
         self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.RerpRobotParser')
 
 
@@ -198,5 +198,5 @@ class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest):
         skip = "Reppy parser is not installed"
 
     def setUp(self):
-        super(RobotsTxtMiddlewareWithReppyTest, self).setUp()
+        super().setUp()
         self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.ReppyRobotParser')
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 660c99ce1..6c25a0064 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -593,7 +593,7 @@ class CustomExporterItemTest(unittest.TestCase):
                 if name == 'age':
                     return str(int(value) + 1)
                 else:
-                    return super(CustomItemExporter, self).serialize_field(field, name, value)
+                    return super().serialize_field(field, name, value)
 
         i = self.item_class(name='John', age='22')
         a = ItemAdapter(i)
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index f5cf4e798..0a303dbe2 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -1265,7 +1265,7 @@ class JsonRequestTest(RequestTest):
 
     def setUp(self):
         warnings.simplefilter("always")
-        super(JsonRequestTest, self).setUp()
+        super().setUp()
 
     def test_data(self):
         r1 = self.request_class(url="http://www.example.com/")
@@ -1419,7 +1419,7 @@ class JsonRequestTest(RequestTest):
 
     def tearDown(self):
         warnings.resetwarnings()
-        super(JsonRequestTest, self).tearDown()
+        super().tearDown()
 
 
 if __name__ == "__main__":
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 56d017de6..f831ef5dc 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -305,7 +305,7 @@ class TextResponseTest(BaseResponseTest):
     response_class = TextResponse
 
     def test_replace(self):
-        super(TextResponseTest, self).test_replace()
+        super().test_replace()
         r1 = self.response_class("http://www.example.com", body="hello", encoding="cp852")
         r2 = r1.replace(url="http://www.example.com/other")
         r3 = r1.replace(url="http://www.example.com/other", encoding="latin1")
diff --git a/tests/test_item.py b/tests/test_item.py
index 0ce78f8c0..66fa761f0 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -312,7 +312,7 @@ class ItemMetaClassCellRegression(unittest.TestCase):
                 # requirement. When not done properly raises an error:
                 # TypeError: __class__ set to <class '__main__.MyItem'>
                 # defining 'MyItem' as <class '__main__.MyItem'>
-                super(MyItem, self).__init__(*args, **kwargs)
+                super().__init__(*args, **kwargs)
 
 
 class DictItemTest(unittest.TestCase):
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index a0bafa5e5..6f133d77a 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -516,7 +516,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
         ])
 
     def test_restrict_xpaths_with_html_entities(self):
-        super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities()
+        super().test_restrict_xpaths_with_html_entities()
 
     def test_filteringlinkextractor_deprecation_warning(self):
         """Make sure the FilteringLinkExtractor deprecation warning is not
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 2ed6f365f..b0bc82f4e 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -250,7 +250,7 @@ class TestOutputProcessorItem(unittest.TestCase):
             temp = Field()
 
             def __init__(self, *args, **kwargs):
-                super(TempItem, self).__init__(self, *args, **kwargs)
+                super().__init__(self, *args, **kwargs)
                 self.setdefault('temp', 0.3)
 
         class TempLoader(ItemLoader):
diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py
index eb14de14f..624dd9ab8 100644
--- a/tests/test_loader_deprecated.py
+++ b/tests/test_loader_deprecated.py
@@ -579,7 +579,7 @@ class TestOutputProcessorDict(unittest.TestCase):
 
         class TempDict(dict):
             def __init__(self, *args, **kwargs):
-                super(TempDict, self).__init__(self, *args, **kwargs)
+                super().__init__(self, *args, **kwargs)
                 self.setdefault('temp', 0.3)
 
         class TempLoader(ItemLoader):
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index b771e7d79..41ff3651d 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -118,7 +118,7 @@ class LogFormatterTestCase(unittest.TestCase):
 
 class LogFormatterSubclass(LogFormatter):
     def crawled(self, request, response, spider):
-        kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider)
+        kwargs = super().crawled(request, response, spider)
         CRAWLEDMSG = (
             "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
         )
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index 3364d2258..b2b75ef20 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -53,7 +53,7 @@ class TestMiddlewareManager(MiddlewareManager):
         return ['tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']]
 
     def _add_middleware(self, mw):
-        super(TestMiddlewareManager, self)._add_middleware(mw)
+        super()._add_middleware(mw)
         if hasattr(mw, 'process'):
             self.methods['process'].append(mw.process)
 
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 19ff00350..4f130c0c9 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -162,18 +162,18 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
 class MockedMediaPipeline(MediaPipeline):
 
     def __init__(self, *args, **kwargs):
-        super(MockedMediaPipeline, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self._mockcalled = []
 
     def download(self, request, info):
         self._mockcalled.append('download')
-        return super(MockedMediaPipeline, self).download(request, info)
+        return super().download(request, info)
 
     def media_to_download(self, request, info):
         self._mockcalled.append('media_to_download')
         if 'result' in request.meta:
             return request.meta.get('result')
-        return super(MockedMediaPipeline, self).media_to_download(request, info)
+        return super().media_to_download(request, info)
 
     def get_media_requests(self, item, info):
         self._mockcalled.append('get_media_requests')
@@ -181,15 +181,15 @@ class MockedMediaPipeline(MediaPipeline):
 
     def media_downloaded(self, response, request, info):
         self._mockcalled.append('media_downloaded')
-        return super(MockedMediaPipeline, self).media_downloaded(response, request, info)
+        return super().media_downloaded(response, request, info)
 
     def media_failed(self, failure, request, info):
         self._mockcalled.append('media_failed')
-        return super(MockedMediaPipeline, self).media_failed(failure, request, info)
+        return super().media_failed(failure, request, info)
 
     def item_completed(self, results, item, info):
         self._mockcalled.append('item_completed')
-        item = super(MockedMediaPipeline, self).item_completed(results, item, info)
+        item = super().item_completed(results, item, info)
         item['results'] = results
         return item
 
diff --git a/tests/test_request_left.py b/tests/test_request_left.py
index 5cfef8e7d..373b2e49c 100644
--- a/tests/test_request_left.py
+++ b/tests/test_request_left.py
@@ -10,7 +10,7 @@ class SignalCatcherSpider(Spider):
     name = 'signal_catcher'
 
     def __init__(self, crawler, url, *args, **kwargs):
-        super(SignalCatcherSpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         crawler.signals.connect(self.on_request_left,
                                 signal=request_left_downloader)
         self.caught_times = 0
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 9d8c201dd..4b15d0fab 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -117,7 +117,7 @@ class BaseRobotParserTest:
 class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
     def setUp(self):
         from scrapy.robotstxt import PythonRobotParser
-        super(PythonRobotParserTest, self)._setUp(PythonRobotParser)
+        super()._setUp(PythonRobotParser)
 
     def test_length_based_precedence(self):
         raise unittest.SkipTest("RobotFileParser does not support length based directives precedence.")
@@ -132,7 +132,7 @@ class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase):
 
     def setUp(self):
         from scrapy.robotstxt import ReppyRobotParser
-        super(ReppyRobotParserTest, self)._setUp(ReppyRobotParser)
+        super()._setUp(ReppyRobotParser)
 
     def test_order_based_precedence(self):
         raise unittest.SkipTest("Reppy does not support order based directives precedence.")
@@ -144,7 +144,7 @@ class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):
 
     def setUp(self):
         from scrapy.robotstxt import RerpRobotParser
-        super(RerpRobotParserTest, self)._setUp(RerpRobotParser)
+        super()._setUp(RerpRobotParser)
 
     def test_length_based_precedence(self):
         raise unittest.SkipTest("Rerp does not support length based directives precedence.")
@@ -156,7 +156,7 @@ class ProtegoRobotParserTest(BaseRobotParserTest, unittest.TestCase):
 
     def setUp(self):
         from scrapy.robotstxt import ProtegoRobotParser
-        super(ProtegoRobotParserTest, self)._setUp(ProtegoRobotParser)
+        super()._setUp(ProtegoRobotParser)
 
     def test_order_based_precedence(self):
         raise unittest.SkipTest("Protego does not support order based directives precedence.")
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 2b6cb0902..512a7460e 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -53,7 +53,7 @@ class MockCrawler(Crawler):
             JOBDIR=jobdir,
             DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter',
         )
-        super(MockCrawler, self).__init__(Spider, settings)
+        super().__init__(Spider, settings)
         self.engine = MockEngine(downloader=MockDownloader())
 
 
@@ -296,7 +296,7 @@ class StartUrlsSpider(Spider):
 
     def __init__(self, start_urls):
         self.start_urls = start_urls
-        super(StartUrlsSpider, self).__init__(name='StartUrlsSpider')
+        super().__init__(name='StartUrlsSpider')
 
     def parse(self, response):
         pass
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index e032b247c..e449cd706 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -19,7 +19,7 @@ class _HttpErrorSpider(MockServerSpider):
     bypass_status_codes = set()
 
     def __init__(self, *args, **kwargs):
-        super(_HttpErrorSpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.start_urls = [
             self.mockserver.url("/status?n=200"),
             self.mockserver.url("/status?n=404"),

From 9d84289109b2368d5929d8b60ce583529c19fe4c Mon Sep 17 00:00:00 2001
From: Kshitij Sharma <ikshitijsharma@gmail.com>
Date: Wed, 5 Aug 2020 09:11:59 +0530
Subject: [PATCH 240/243] deprecated weakkeycache by specifying in __init__

---
 scrapy/utils/python.py     |  4 +++-
 scrapy/utils/tester.py     |  3 +++
 tests/test_utils_python.py | 22 +++++++++++++++++++++-
 3 files changed, 27 insertions(+), 2 deletions(-)
 create mode 100644 scrapy/utils/tester.py

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index c8f921ff3..4756b07b6 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -7,10 +7,12 @@ import inspect
 import re
 import sys
 import weakref
+import warnings
 from functools import partial, wraps
 from itertools import chain
 
 from scrapy.utils.decorators import deprecated
+from scrapy.exceptions import ScrapyDeprecationWarning
 
 
 def flatten(x):
@@ -275,10 +277,10 @@ def equal_attributes(obj1, obj2, attributes):
     return True
 
 
-@deprecated
 class WeakKeyCache:
 
     def __init__(self, default_factory):
+        warnings.warn("Call to deprecated Class WeakKeyCache", category=ScrapyDeprecationWarning, stacklevel=2)
         self.default_factory = default_factory
         self._weakdict = weakref.WeakKeyDictionary()
 
diff --git a/scrapy/utils/tester.py b/scrapy/utils/tester.py
new file mode 100644
index 000000000..691e9bc1a
--- /dev/null
+++ b/scrapy/utils/tester.py
@@ -0,0 +1,3 @@
+from scrapy.utils.decorators import deprecated
+
+
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 5a53d89e4..ebce3c079 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -1,15 +1,18 @@
 import functools
+import gc
 import operator
 import platform
 import unittest
+from itertools import count
 from sys import version_info
 from warnings import catch_warnings
 
 from scrapy.utils.python import (
     memoizemethod_noargs, binary_is_text, equal_attributes,
-    get_func_args, to_bytes, to_unicode,
+    WeakKeyCache, get_func_args, to_bytes, to_unicode,
     without_none_values, MutableChain)
 
+
 __doctests__ = ['scrapy.utils.python']
 
 
@@ -152,6 +155,23 @@ class UtilsPythonTestCase(unittest.TestCase):
         a.meta['z'] = 2
         self.assertFalse(equal_attributes(a, b, [compare_z, 'x']))
 
+    def test_weakkeycache(self):
+        class _Weakme:
+            pass
+
+        _values = count()
+        wk = WeakKeyCache(lambda k: next(_values))
+        k = _Weakme()
+        v = wk[k]
+        self.assertEqual(v, wk[k])
+        self.assertNotEqual(v, wk[_Weakme()])
+        self.assertEqual(v, wk[k])
+        del k
+        for _ in range(100):
+            if wk._weakdict:
+                gc.collect()
+        self.assertFalse(len(wk._weakdict))
+
     def test_get_func_args(self):
         def f1(a, b, c):
             pass

From b35d1f2b2c430f4d12cb8f9d408dfa0c0051746d Mon Sep 17 00:00:00 2001
From: Kshitij Sharma <ikshitijsharma@gmail.com>
Date: Wed, 5 Aug 2020 09:14:04 +0530
Subject: [PATCH 241/243] deleted tester.py

---
 scrapy/utils/tester.py | 3 ---
 1 file changed, 3 deletions(-)
 delete mode 100644 scrapy/utils/tester.py

diff --git a/scrapy/utils/tester.py b/scrapy/utils/tester.py
deleted file mode 100644
index 691e9bc1a..000000000
--- a/scrapy/utils/tester.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from scrapy.utils.decorators import deprecated
-
-

From 983b7ddf2e39c480efc6d104054f92f570714ac8 Mon Sep 17 00:00:00 2001
From: Kshitij Sharma <ikshitijsharma@gmail.com>
Date: Wed, 5 Aug 2020 16:13:52 +0530
Subject: [PATCH 242/243] aesthetic fixes

---
 scrapy/utils/python.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 4756b07b6..59f1b8371 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -6,13 +6,13 @@ import gc
 import inspect
 import re
 import sys
-import weakref
 import warnings
+import weakref
 from functools import partial, wraps
 from itertools import chain
 
-from scrapy.utils.decorators import deprecated
 from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.decorators import deprecated
 
 
 def flatten(x):
@@ -280,7 +280,7 @@ def equal_attributes(obj1, obj2, attributes):
 class WeakKeyCache:
 
     def __init__(self, default_factory):
-        warnings.warn("Call to deprecated Class WeakKeyCache", category=ScrapyDeprecationWarning, stacklevel=2)
+        warnings.warn("The WeakKeyCache class is deprecated", category=ScrapyDeprecationWarning, stacklevel=2)
         self.default_factory = default_factory
         self._weakdict = weakref.WeakKeyDictionary()
 

From 4dc09f09aa9698b02f2cbf2e3001202388eba043 Mon Sep 17 00:00:00 2001
From: linchiwei123 <40888469+linchiwei123@users.noreply.github.com>
Date: Wed, 5 Aug 2020 22:23:19 +0800
Subject: [PATCH 243/243] Update setup.py

---
 setup.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/setup.py b/setup.py
index 58090f7a2..d0880051f 100644
--- a/setup.py
+++ b/setup.py
@@ -23,7 +23,6 @@ install_requires = [
     'cryptography>=2.0',
     'cssselect>=0.9.1',
     'itemloaders>=1.0.1',
-    'lxml>=3.5.0',
     'parsel>=1.5.0',
     'PyDispatcher>=2.0.5',
     'pyOpenSSL>=16.2.0',