From 0641ba0faa97498ca5bee39c4e8faec58d5f0522 Mon Sep 17 00:00:00 2001
From: faizan2700 <syedfaizan824@gmail.com>
Date: Sun, 2 Feb 2020 16:54:22 +0530
Subject: [PATCH 01/57] SCRAPY_CHECK will be set while running contact

---
 scrapy/commands/check.py | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index 9d4437a47..09a76ca7a 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -78,19 +78,19 @@ class Command(ScrapyCommand):
                 elif tested_methods:
                     self.crawler_process.crawl(spidercls)
 
-        # start checks
-        if opts.list:
-            for spider, methods in sorted(contract_reqs.items()):
-                if not methods and not opts.verbose:
-                    continue
-                print(spider)
-                for method in sorted(methods):
-                    print('  * %s' % method)
-        else:
-            start = time.time()
-            self.crawler_process.start()
-            stop = time.time()
+            # start checks
+            if opts.list:
+                for spider, methods in sorted(contract_reqs.items()):
+                    if not methods and not opts.verbose:
+                        continue
+                    print(spider)
+                    for method in sorted(methods):
+                        print('  * %s' % method)
+            else:
+                start = time.time()
+                self.crawler_process.start()
+                stop = time.time()
 
-            result.printErrors()
-            result.printSummary(start, stop)
-            self.exitcode = int(not result.wasSuccessful())
+                result.printErrors()
+                result.printSummary(start, stop)
+                self.exitcode = int(not result.wasSuccessful())

From e5b23f4b00962df76d8302ebf869ef4a4319e142 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Wed, 18 Mar 2020 11:26:59 +0300
Subject: [PATCH 02/57] fix #4250: add batch deliveries

---
 scrapy/extensions/feedexport.py     | 54 +++++++++++++++++++++--------
 scrapy/settings/default_settings.py |  1 +
 2 files changed, 41 insertions(+), 14 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 998d2a5d1..906f99fee 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -241,6 +241,7 @@ class FeedExporter:
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
+        self.storage_batch = self.settings.getint('FEED_STORAGE_BATCH')
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
@@ -250,19 +251,7 @@ class FeedExporter:
     def open_spider(self, spider):
         for uri, feed in self.feeds.items():
             uri = uri % self._get_uri_params(spider, feed['uri_params'])
-            storage = self._get_storage(uri)
-            file = storage.open(spider)
-            exporter = self._get_exporter(
-                file=file,
-                format=feed['format'],
-                fields_to_export=feed['fields'],
-                encoding=feed['encoding'],
-                indent=feed['indent'],
-            )
-            slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'])
-            self.slots.append(slot)
-            if slot.store_empty:
-                slot.start_exporting()
+            self.slots.append(self._start_new_batch(None, uri, feed, spider))
 
     def close_spider(self, spider):
         deferred_list = []
@@ -285,11 +274,48 @@ class FeedExporter:
             deferred_list.append(d)
         return defer.DeferredList(deferred_list) if deferred_list else None
 
+    def _start_new_batch(self, previous_batch_slot, uri, feed, spider):
+        """
+        Redirect the output data stream to a new file.
+        Execute multiple times if 'FEED_STORAGE_BATCH' setting is greater than zero.
+        """
+        if previous_batch_slot is not None:
+            previous_batch_slot.exporter.finish_exporting()
+            previous_batch_slot.storage.store(previous_batch_slot.file)
+        storage = self._get_storage(uri)
+        file = storage.open(spider)
+        exporter = self._get_exporter(
+            file=file,
+            format=feed['format'],
+            fields_to_export=feed['fields'],
+            encoding=feed['encoding'],
+            indent=feed['indent']
+        )
+        slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'])
+        if slot.store_empty:
+            slot.start_exporting()
+        return slot
+
+    def _get_uri_of_partial(self, slot, feed, spider):
+        """Get uri for each partial using datetime.now().isoformat()"""
+        uri = (slot.uri % self._get_uri_params(spider, feed['uri_params'])).split('.')[0] + '.'
+        uri = uri + datetime.now().isoformat() + '.' + feed['format']
+        return uri
+
     def item_scraped(self, item, spider):
-        for slot in self.slots:
+        slots = []
+        for idx, slot in enumerate(self.slots):
             slot.start_exporting()
             slot.exporter.export_item(item)
             slot.itemcount += 1
+            if self.storage_batch and slot.itemcount % self.storage_batch == 0:
+                uri = self._get_uri_of_partial(slot, self.feeds[slot.uri], spider)
+                slots.append(self._start_new_batch(slot, uri, self.feeds[slot.uri], spider))
+                self.feeds[uri] = self.feeds[slot.uri]
+                self.feeds.pop(slot.uri)
+                self.slots[idx] = None
+        self.slots = [slot for slot in self.slots if slot is not None]
+        self.slots.extend(slots)
 
     def _load_components(self, setting_prefix):
         conf = without_none_values(self.settings.getwithbase(setting_prefix))
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 077317c81..690e044c5 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -146,6 +146,7 @@ FEED_STORAGES_BASE = {
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
 }
+FEED_STORAGE_BATCH = 0
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
     'json': 'scrapy.exporters.JsonItemExporter',

From 8b4566ff93843cdf17ada069dc09261a99971d26 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Wed, 18 Mar 2020 14:21:21 +0300
Subject: [PATCH 03/57] fix wrong name of first file in partial deliveries

---
 scrapy/extensions/feedexport.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 906f99fee..4f7c6bf07 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -249,6 +249,8 @@ class FeedExporter:
                 raise NotConfigured
 
     def open_spider(self, spider):
+        if self.storage_batch:
+            self.feeds = {self._get_uri_of_partial(uri, feed, spider): feed for uri, feed in self.feeds.items()}
         for uri, feed in self.feeds.items():
             uri = uri % self._get_uri_params(spider, feed['uri_params'])
             self.slots.append(self._start_new_batch(None, uri, feed, spider))
@@ -296,11 +298,11 @@ class FeedExporter:
             slot.start_exporting()
         return slot
 
-    def _get_uri_of_partial(self, slot, feed, spider):
+    def _get_uri_of_partial(self, template_uri, feed, spider):
         """Get uri for each partial using datetime.now().isoformat()"""
-        uri = (slot.uri % self._get_uri_params(spider, feed['uri_params'])).split('.')[0] + '.'
-        uri = uri + datetime.now().isoformat() + '.' + feed['format']
-        return uri
+        template_uri = (template_uri % self._get_uri_params(spider, feed['uri_params']))
+        uri_name = template_uri.split('.')[0]
+        return '{}.{}.{}'.format(uri_name, datetime.now().isoformat(), feed["format"])
 
     def item_scraped(self, item, spider):
         slots = []
@@ -309,11 +311,12 @@ class FeedExporter:
             slot.exporter.export_item(item)
             slot.itemcount += 1
             if self.storage_batch and slot.itemcount % self.storage_batch == 0:
-                uri = self._get_uri_of_partial(slot, self.feeds[slot.uri], spider)
+                uri = self._get_uri_of_partial(slot.uri, self.feeds[slot.uri], spider)
                 slots.append(self._start_new_batch(slot, uri, self.feeds[slot.uri], spider))
                 self.feeds[uri] = self.feeds[slot.uri]
                 self.feeds.pop(slot.uri)
                 self.slots[idx] = None
+
         self.slots = [slot for slot in self.slots if slot is not None]
         self.slots.extend(slots)
 

From 0723e3f4f9777a87d0df3b2e2fddfeac9099dd3b Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 19 Mar 2020 21:17:02 +0300
Subject: [PATCH 04/57] add batch_id, add error if uri is specified incorrectly

---
 scrapy/extensions/feedexport.py     | 73 ++++++++++++++++++++---------
 scrapy/settings/default_settings.py |  2 +-
 2 files changed, 52 insertions(+), 23 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 4f7c6bf07..38b25bf4a 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -180,14 +180,16 @@ class FTPFeedStorage(BlockingFeedStorage):
 
 
 class _FeedSlot:
-    def __init__(self, file, exporter, storage, uri, format, store_empty):
+    def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, template_uri):
         self.file = file
         self.exporter = exporter
         self.storage = storage
         # feed params
-        self.uri = uri
+        self.batch_id = batch_id
         self.format = format
         self.store_empty = store_empty
+        self.template_uri = template_uri
+        self.uri = uri
         # flags
         self.itemcount = 0
         self._exporting = False
@@ -241,19 +243,28 @@ class FeedExporter:
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
-        self.storage_batch = self.settings.getint('FEED_STORAGE_BATCH')
+        self.storage_batch_size = self.settings.getint('FEED_STORAGE_BATCH_SIZE')
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
+            if not self._batch_deliveries_supported(uri):
+                raise NotConfigured
             if not self._exporter_supported(feed['format']):
                 raise NotConfigured
 
     def open_spider(self, spider):
-        if self.storage_batch:
-            self.feeds = {self._get_uri_of_partial(uri, feed, spider): feed for uri, feed in self.feeds.items()}
         for uri, feed in self.feeds.items():
-            uri = uri % self._get_uri_params(spider, feed['uri_params'])
-            self.slots.append(self._start_new_batch(None, uri, feed, spider))
+            batch_id = 1
+            uri_params = self._get_uri_params(spider, feed['uri_params'])
+            uri_params['batch_id'] = batch_id
+            self.slots.append(self._start_new_batch(
+                previous_batch_slot=None,
+                uri=uri % uri_params,
+                feed=feed,
+                spider=spider,
+                batch_id=batch_id,
+                template_uri=uri
+            ))
 
     def close_spider(self, spider):
         deferred_list = []
@@ -276,10 +287,17 @@ class FeedExporter:
             deferred_list.append(d)
         return defer.DeferredList(deferred_list) if deferred_list else None
 
-    def _start_new_batch(self, previous_batch_slot, uri, feed, spider):
+    def _start_new_batch(self, previous_batch_slot, uri, feed, spider, batch_id, template_uri):
         """
         Redirect the output data stream to a new file.
         Execute multiple times if 'FEED_STORAGE_BATCH' setting is greater than zero.
+        :param previous_batch_slot: slot of previous batch. We need to call slot.storage.store
+        to get the file properly closed.
+        :param uri: uri of the new batch to start
+        :param feed: dict with parameters of feed
+        :param spider: user spider
+        :param batch_id: sequential batch id starting at 1
+        :param template_uri: template uri which contains %(time)s or %(batch_id)s to create new uri
         """
         if previous_batch_slot is not None:
             previous_batch_slot.exporter.finish_exporting()
@@ -293,30 +311,30 @@ class FeedExporter:
             encoding=feed['encoding'],
             indent=feed['indent']
         )
-        slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'])
+        slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'], batch_id, template_uri)
         if slot.store_empty:
             slot.start_exporting()
         return slot
 
-    def _get_uri_of_partial(self, template_uri, feed, spider):
-        """Get uri for each partial using datetime.now().isoformat()"""
-        template_uri = (template_uri % self._get_uri_params(spider, feed['uri_params']))
-        uri_name = template_uri.split('.')[0]
-        return '{}.{}.{}'.format(uri_name, datetime.now().isoformat(), feed["format"])
-
     def item_scraped(self, item, spider):
         slots = []
         for idx, slot in enumerate(self.slots):
             slot.start_exporting()
             slot.exporter.export_item(item)
             slot.itemcount += 1
-            if self.storage_batch and slot.itemcount % self.storage_batch == 0:
-                uri = self._get_uri_of_partial(slot.uri, self.feeds[slot.uri], spider)
-                slots.append(self._start_new_batch(slot, uri, self.feeds[slot.uri], spider))
-                self.feeds[uri] = self.feeds[slot.uri]
-                self.feeds.pop(slot.uri)
+            if self.storage_batch_size and slot.itemcount % self.storage_batch_size == 0:
+                batch_id = slot.batch_id + 1
+                uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'])
+                uri_params['batch_id'] = batch_id
+                self.slots.append(self._start_new_batch(
+                    previous_batch_slot=slot,
+                    uri=slot.template_uri % uri_params,
+                    feed=self.feeds[slot.template_uri],
+                    spider=spider,
+                    batch_id=batch_id,
+                    template_uri=slot.template_uri
+                ))
                 self.slots[idx] = None
-
         self.slots = [slot for slot in self.slots if slot is not None]
         self.slots.extend(slots)
 
@@ -335,6 +353,17 @@ class FeedExporter:
             return True
         logger.error("Unknown feed format: %(format)s", {'format': format})
 
+    def _batch_deliveries_supported(self, uri):
+        """
+        If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time)s or %(batch_id)s
+        to distinguish different files of partial output
+        """
+        if not self.storage_batch_size:
+            return True
+        if '%(time)s' in uri or '%(batch_id)s' in uri:
+            return True
+        logger.error('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
+
     def _storage_supported(self, uri):
         scheme = urlparse(uri).scheme
         if scheme in self.storages:
@@ -364,7 +393,7 @@ class FeedExporter:
         params = {}
         for k in dir(spider):
             params[k] = getattr(spider, k)
-        ts = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')
+        ts = datetime.utcnow().isoformat().replace(':', '-')
         params['time'] = ts
         uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
         uripar_function(params, spider)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 690e044c5..7f90a2280 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -146,7 +146,7 @@ FEED_STORAGES_BASE = {
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
 }
-FEED_STORAGE_BATCH = 0
+FEED_STORAGE_BATCH_SIZE = 0
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
     'json': 'scrapy.exporters.JsonItemExporter',

From d11411b402ae68874c6ccc2883836be0b9cf8326 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Sat, 21 Mar 2020 10:48:13 +0300
Subject: [PATCH 05/57] fix comments

---
 scrapy/extensions/feedexport.py     | 31 ++++++++++++++++++-----------
 scrapy/settings/default_settings.py |  2 +-
 2 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 38b25bf4a..ab0a0de37 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -25,7 +25,6 @@ from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import without_none_values
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -243,7 +242,7 @@ class FeedExporter:
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
-        self.storage_batch_size = self.settings.getint('FEED_STORAGE_BATCH_SIZE')
+        self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_SIZE', None)
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
@@ -263,7 +262,7 @@ class FeedExporter:
                 feed=feed,
                 spider=spider,
                 batch_id=batch_id,
-                template_uri=uri
+                template_uri=uri,
             ))
 
     def close_spider(self, spider):
@@ -290,7 +289,7 @@ class FeedExporter:
     def _start_new_batch(self, previous_batch_slot, uri, feed, spider, batch_id, template_uri):
         """
         Redirect the output data stream to a new file.
-        Execute multiple times if 'FEED_STORAGE_BATCH' setting is greater than zero.
+        Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified.
         :param previous_batch_slot: slot of previous batch. We need to call slot.storage.store
         to get the file properly closed.
         :param uri: uri of the new batch to start
@@ -309,9 +308,18 @@ class FeedExporter:
             format=feed['format'],
             fields_to_export=feed['fields'],
             encoding=feed['encoding'],
-            indent=feed['indent']
+            indent=feed['indent'],
+        )
+        slot = _FeedSlot(
+            file=file,
+            exporter=exporter,
+            storage=storage,
+            uri=uri,
+            format=feed['format'],
+            store_empty=feed['store_empty'],
+            batch_id=batch_id,
+            template_uri=template_uri,
         )
-        slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'], batch_id, template_uri)
         if slot.store_empty:
             slot.start_exporting()
         return slot
@@ -326,13 +334,13 @@ class FeedExporter:
                 batch_id = slot.batch_id + 1
                 uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'])
                 uri_params['batch_id'] = batch_id
-                self.slots.append(self._start_new_batch(
+                slots.append(self._start_new_batch(
                     previous_batch_slot=slot,
                     uri=slot.template_uri % uri_params,
                     feed=self.feeds[slot.template_uri],
                     spider=spider,
                     batch_id=batch_id,
-                    template_uri=slot.template_uri
+                    template_uri=slot.template_uri,
                 ))
                 self.slots[idx] = None
         self.slots = [slot for slot in self.slots if slot is not None]
@@ -358,11 +366,10 @@ class FeedExporter:
         If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time)s or %(batch_id)s
         to distinguish different files of partial output
         """
-        if not self.storage_batch_size:
+        if self.storage_batch_size is None or '%(time)s' in uri or '%(batch_id)s' in uri:
             return True
-        if '%(time)s' in uri or '%(batch_id)s' in uri:
-            return True
-        logger.error('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
+        logger.warning('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
+        return False
 
     def _storage_supported(self, uri):
         scheme = urlparse(uri).scheme
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 7f90a2280..c3463a505 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -146,7 +146,7 @@ FEED_STORAGES_BASE = {
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
 }
-FEED_STORAGE_BATCH_SIZE = 0
+FEED_STORAGE_BATCH_SIZE = None
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
     'json': 'scrapy.exporters.JsonItemExporter',

From 39d0d13d3f7bd671d5b29646b209c62e23373fab Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 26 Mar 2020 14:18:35 +0300
Subject: [PATCH 06/57] Add partial deliveries tests

---
 tests/test_feedexport.py | 195 +++++++++++++++++++++++++++++++--------
 1 file changed, 159 insertions(+), 36 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index c5589e52f..1ebe44e12 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -6,6 +6,7 @@ import shutil
 import string
 import tempfile
 import warnings
+from abc import ABC, abstractmethod
 from io import BytesIO
 from pathlib import Path
 from string import ascii_letters, digits
@@ -21,8 +22,9 @@ from zope.interface.verify import verifyObject
 
 import scrapy
 from scrapy.crawler import CrawlerRunner
+from scrapy.exceptions import NotConfigured
 from scrapy.exporters import CsvItemExporter
-from scrapy.extensions.feedexport import (BlockingFeedStorage, FileFeedStorage, FTPFeedStorage,
+from scrapy.extensions.feedexport import (BlockingFeedStorage, FeedExporter, FileFeedStorage, FTPFeedStorage,
                                           IFeedStorage, S3FeedStorage, StdoutFeedStorage)
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
@@ -76,6 +78,7 @@ class FTPFeedStorageTest(unittest.TestCase):
     def get_test_spider(self, settings=None):
         class TestSpider(scrapy.Spider):
             name = 'test_spider'
+
         crawler = get_crawler(settings_dict=settings)
         spider = TestSpider.from_crawler(crawler)
         return spider
@@ -129,6 +132,7 @@ class BlockingFeedStorageTest(unittest.TestCase):
     def get_test_spider(self, settings=None):
         class TestSpider(scrapy.Spider):
             name = 'test_spider'
+
         crawler = get_crawler(settings_dict=settings)
         spider = TestSpider.from_crawler(crawler)
         return spider
@@ -390,23 +394,63 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):
     pass
 
 
-class FeedExportTest(unittest.TestCase):
+class FeedExportTestBase(ABC, unittest.TestCase):
+    __test__ = False
 
     class MyItem(scrapy.Item):
         foo = scrapy.Field()
         egg = scrapy.Field()
         baz = scrapy.Field()
 
+    def _random_temp_filename(self, inter_dir=''):
+        chars = [random.choice(ascii_letters + digits) for _ in range(15)]
+        filename = ''.join(chars)
+        return os.path.join(self.temp_dir, inter_dir, filename)
+
     def setUp(self):
         self.temp_dir = tempfile.mkdtemp()
 
     def tearDown(self):
         shutil.rmtree(self.temp_dir, ignore_errors=True)
 
-    def _random_temp_filename(self):
-        chars = [random.choice(ascii_letters + digits) for _ in range(15)]
-        filename = ''.join(chars)
-        return os.path.join(self.temp_dir, filename)
+    @defer.inlineCallbacks
+    def exported_data(self, items, settings):
+        """
+        Return exported data which a spider yielding ``items`` would return.
+        """
+
+        class TestSpider(scrapy.Spider):
+            name = 'testspider'
+
+            def parse(self, response):
+                for item in items:
+                    yield item
+
+        data = yield self.run_and_export(TestSpider, settings)
+        defer.returnValue(data)
+
+    @defer.inlineCallbacks
+    def exported_no_data(self, settings):
+        """
+        Return exported data which a spider yielding no ``items`` would return.
+        """
+
+        class TestSpider(scrapy.Spider):
+            name = 'testspider'
+
+            def parse(self, response):
+                pass
+
+        data = yield self.run_and_export(TestSpider, settings)
+        defer.returnValue(data)
+
+    @abstractmethod
+    def run_and_export(self, spider_cls, settings):
+        pass
+
+
+class FeedExportTest(FeedExportTestBase):
+    __test__ = True
 
     @defer.inlineCallbacks
     def run_and_export(self, spider_cls, settings):
@@ -417,7 +461,6 @@ class FeedExportTest(unittest.TestCase):
             urljoin('file:', pathname2url(str(file_path))): feed
             for file_path, feed in FEEDS.items()
         }
-
         content = {}
         try:
             with MockServer() as s:
@@ -435,35 +478,6 @@ class FeedExportTest(unittest.TestCase):
 
         defer.returnValue(content)
 
-    @defer.inlineCallbacks
-    def exported_data(self, items, settings):
-        """
-        Return exported data which a spider yielding ``items`` would return.
-        """
-        class TestSpider(scrapy.Spider):
-            name = 'testspider'
-
-            def parse(self, response):
-                for item in items:
-                    yield item
-
-        data = yield self.run_and_export(TestSpider, settings)
-        defer.returnValue(data)
-
-    @defer.inlineCallbacks
-    def exported_no_data(self, settings):
-        """
-        Return exported data which a spider yielding no ``items`` would return.
-        """
-        class TestSpider(scrapy.Spider):
-            name = 'testspider'
-
-            def parse(self, response):
-                pass
-
-        data = yield self.run_and_export(TestSpider, settings)
-        defer.returnValue(data)
-
     @defer.inlineCallbacks
     def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
         settings = settings or {}
@@ -970,3 +984,112 @@ class FeedExportTest(unittest.TestCase):
         }
         data = yield self.exported_no_data(settings)
         self.assertEqual(data['csv'], b'')
+
+
+class PartialDeliveriesTest(FeedExportTestBase):
+    __test__ = True
+    _file_mark = '_%(time)s_#%(batch_id)s'
+
+    @defer.inlineCallbacks
+    def run_and_export(self, spider_cls, settings):
+        """ Run spider with specified settings; return exported data. """
+
+        FEEDS = settings.get('FEEDS') or {}
+        settings['FEEDS'] = {
+            urljoin('file:', file_path): feed
+            for file_path, feed in FEEDS.items()
+        }
+        from collections import defaultdict
+        content = defaultdict(list)
+        try:
+            with MockServer() as s:
+                runner = CrawlerRunner(Settings(settings))
+                spider_cls.start_urls = [s.url('/')]
+                yield runner.crawl(spider_cls)
+
+            for path, feed in FEEDS.items():
+                dir_name = os.path.dirname(path)
+                for file in sorted(os.listdir(dir_name)):
+                    with open(os.path.join(dir_name, file), 'rb') as f:
+                        data = f.read()
+                        content[feed['format']].append(data)
+        finally:
+            pass
+        defer.returnValue(content)
+
+    @defer.inlineCallbacks
+    def assertPartialExported(self, items, rows, settings=None):
+        settings = settings or {}
+        settings.update({
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
+            },
+        })
+        data = yield self.exported_data(items, settings)
+        data['jl'] = b''.join(data['jl'])
+        parsed = [json.loads(to_unicode(line)) for line in data['jl'].splitlines()]
+
+        rows = [{k: v for k, v in row.items() if v} for row in rows]
+        self.assertEqual(rows, parsed)
+
+    @defer.inlineCallbacks
+    def test_partial_deliveries(self):
+        items = [
+            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+            self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
+            self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
+        ]
+        rows = [
+            {'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
+            {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'},
+            {'foo': 'bar3', 'baz': 'quux3'}
+        ]
+        settings = {
+            'FEED_STORAGE_BATCH_SIZE': 1
+        }
+        yield self.assertPartialExported(items, rows, settings=settings)
+
+    def test_wrong_path(self):
+        settings = {
+            'FEEDS': {
+                self._random_temp_filename(): {'format': 'xml'},
+            },
+            'FEED_STORAGE_BATCH_SIZE': 1
+        }
+        crawler = get_crawler(settings_dict=settings)
+        self.assertRaises(NotConfigured, FeedExporter, crawler)
+
+    @defer.inlineCallbacks
+    def test_export_no_items_not_store_empty(self):
+        for fmt in ('json', 'jsonlines', 'xml', 'csv'):
+            settings = {
+                'FEEDS': {
+                    os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
+                },
+                'FEED_STORAGE_BATCH_SIZE': 1
+            }
+            data = yield self.exported_no_data(settings)
+            data[fmt] = b''.join(data[fmt])
+            self.assertEqual(data[fmt], b'')
+
+    @defer.inlineCallbacks
+    def test_export_no_items_store_empty(self):
+        formats = (
+            ('json', b'[]'),
+            ('jsonlines', b''),
+            ('xml', b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),
+            ('csv', b''),
+        )
+
+        for fmt, expctd in formats:
+            settings = {
+                'FEEDS': {
+                    os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
+                },
+                'FEED_STORE_EMPTY': True,
+                'FEED_EXPORT_INDENT': None,
+                'FEED_STORAGE_BATCH_SIZE': 1
+            }
+            data = yield self.exported_no_data(settings)
+            data[fmt] = b''.join(data[fmt])
+            self.assertEqual(data[fmt], expctd)

From ffa8a533e74478a5c81fbf453f2c65601bb1d244 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Sat, 28 Mar 2020 11:40:16 +0300
Subject: [PATCH 07/57] Set batch_id in _get_uri_params

---
 scrapy/extensions/feedexport.py | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index ab0a0de37..06ea6c5b2 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -253,15 +253,12 @@ class FeedExporter:
 
     def open_spider(self, spider):
         for uri, feed in self.feeds.items():
-            batch_id = 1
-            uri_params = self._get_uri_params(spider, feed['uri_params'])
-            uri_params['batch_id'] = batch_id
+            uri_params = self._get_uri_params(spider, feed['uri_params'], None)
             self.slots.append(self._start_new_batch(
                 previous_batch_slot=None,
                 uri=uri % uri_params,
                 feed=feed,
                 spider=spider,
-                batch_id=batch_id,
                 template_uri=uri,
             ))
 
@@ -286,7 +283,7 @@ class FeedExporter:
             deferred_list.append(d)
         return defer.DeferredList(deferred_list) if deferred_list else None
 
-    def _start_new_batch(self, previous_batch_slot, uri, feed, spider, batch_id, template_uri):
+    def _start_new_batch(self, previous_batch_slot, uri, feed, spider, template_uri):
         """
         Redirect the output data stream to a new file.
         Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified.
@@ -295,12 +292,15 @@ class FeedExporter:
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
         :param spider: user spider
-        :param batch_id: sequential batch id starting at 1
         :param template_uri: template uri which contains %(time)s or %(batch_id)s to create new uri
         """
         if previous_batch_slot is not None:
+            previous_batch_id = previous_batch_slot.batch_id
             previous_batch_slot.exporter.finish_exporting()
             previous_batch_slot.storage.store(previous_batch_slot.file)
+        else:
+            previous_batch_id = 0
+
         storage = self._get_storage(uri)
         file = storage.open(spider)
         exporter = self._get_exporter(
@@ -317,7 +317,7 @@ class FeedExporter:
             uri=uri,
             format=feed['format'],
             store_empty=feed['store_empty'],
-            batch_id=batch_id,
+            batch_id=previous_batch_id + 1,
             template_uri=template_uri,
         )
         if slot.store_empty:
@@ -331,15 +331,12 @@ class FeedExporter:
             slot.exporter.export_item(item)
             slot.itemcount += 1
             if self.storage_batch_size and slot.itemcount % self.storage_batch_size == 0:
-                batch_id = slot.batch_id + 1
-                uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'])
-                uri_params['batch_id'] = batch_id
+                uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot)
                 slots.append(self._start_new_batch(
                     previous_batch_slot=slot,
                     uri=slot.template_uri % uri_params,
                     feed=self.feeds[slot.template_uri],
                     spider=spider,
-                    batch_id=batch_id,
                     template_uri=slot.template_uri,
                 ))
                 self.slots[idx] = None
@@ -396,12 +393,12 @@ class FeedExporter:
     def _get_storage(self, uri):
         return self._get_instance(self.storages[urlparse(uri).scheme], uri)
 
-    def _get_uri_params(self, spider, uri_params):
+    def _get_uri_params(self, spider, uri_params, slot):
         params = {}
         for k in dir(spider):
             params[k] = getattr(spider, k)
-        ts = datetime.utcnow().isoformat().replace(':', '-')
-        params['time'] = ts
+        params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
+        params['time'] = datetime.utcnow().isoformat().replace(':', '-')
         uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
         uripar_function(params, spider)
         return params

From 963580463b96315eb58319e6d35b4cd52672371a Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Wed, 15 Apr 2020 20:14:33 +0300
Subject: [PATCH 08/57] Update tests

---
 tests/test_feedexport.py | 203 ++++++++++++++++++++++++++++++---------
 1 file changed, 159 insertions(+), 44 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 1ebe44e12..c6cd867b1 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -7,6 +7,7 @@ import string
 import tempfile
 import warnings
 from abc import ABC, abstractmethod
+from collections import defaultdict
 from io import BytesIO
 from pathlib import Path
 from string import ascii_letters, digits
@@ -444,10 +445,31 @@ class FeedExportTestBase(ABC, unittest.TestCase):
         data = yield self.run_and_export(TestSpider, settings)
         defer.returnValue(data)
 
+    @defer.inlineCallbacks
+    def assertExported(self, items, header, rows, settings=None, ordered=True):
+        yield self.assertExportedCsv(items, header, rows, settings, ordered)
+        yield self.assertExportedJsonLines(items, rows, settings)
+        yield self.assertExportedXml(items, rows, settings)
+        yield self.assertExportedPickle(items, rows, settings)
+        yield self.assertExportedMarshal(items, rows, settings)
+        yield self.assertExportedMultiple(items, rows, settings)
+
     @abstractmethod
     def run_and_export(self, spider_cls, settings):
         pass
 
+    def _load_until_eof(self, data, load_func):
+        result = []
+        with tempfile.TemporaryFile() as temp:
+            temp.write(data)
+            temp.seek(0)
+            while True:
+                try:
+                    result.append(load_func(temp))
+                except EOFError:
+                    break
+        return result
+
 
 class FeedExportTest(FeedExportTestBase):
     __test__ = True
@@ -478,6 +500,22 @@ class FeedExportTest(FeedExportTestBase):
 
         defer.returnValue(content)
 
+    @defer.inlineCallbacks
+    def exported_data(self, items, settings):
+        """
+        Return exported data which a spider yielding ``items`` would return.
+        """
+
+        class TestSpider(scrapy.Spider):
+            name = 'testspider'
+
+            def parse(self, response):
+                for item in items:
+                    yield item
+
+        data = yield self.run_and_export(TestSpider, settings)
+        defer.returnValue(data)
+
     @defer.inlineCallbacks
     def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
         settings = settings or {}
@@ -543,18 +581,6 @@ class FeedExportTest(FeedExportTestBase):
         json_rows = json.loads(to_unicode(data['json']))
         self.assertEqual(rows, json_rows)
 
-    def _load_until_eof(self, data, load_func):
-        result = []
-        with tempfile.TemporaryFile() as temp:
-            temp.write(data)
-            temp.seek(0)
-            while True:
-                try:
-                    result.append(load_func(temp))
-                except EOFError:
-                    break
-        return result
-
     @defer.inlineCallbacks
     def assertExportedPickle(self, items, rows, settings=None):
         settings = settings or {}
@@ -583,15 +609,6 @@ class FeedExportTest(FeedExportTestBase):
         result = self._load_until_eof(data['marshal'], load_func=marshal.load)
         self.assertEqual(expected, result)
 
-    @defer.inlineCallbacks
-    def assertExported(self, items, header, rows, settings=None, ordered=True):
-        yield self.assertExportedCsv(items, header, rows, settings, ordered)
-        yield self.assertExportedJsonLines(items, rows, settings)
-        yield self.assertExportedXml(items, rows, settings)
-        yield self.assertExportedPickle(items, rows, settings)
-        yield self.assertExportedMarshal(items, rows, settings)
-        yield self.assertExportedMultiple(items, rows, settings)
-
     @defer.inlineCallbacks
     def test_export_items(self):
         # feed exporters use field names from Item
@@ -615,7 +632,7 @@ class FeedExportTest(FeedExportTestBase):
                 },
             }
             data = yield self.exported_no_data(settings)
-            self.assertEqual(data[fmt], b'')
+            self.assertEqual(b'', data[fmt])
 
     @defer.inlineCallbacks
     def test_export_no_items_store_empty(self):
@@ -635,7 +652,7 @@ class FeedExportTest(FeedExportTestBase):
                 'FEED_EXPORT_INDENT': None,
             }
             data = yield self.exported_no_data(settings)
-            self.assertEqual(data[fmt], expctd)
+            self.assertEqual(expctd, data[fmt])
 
     @defer.inlineCallbacks
     def test_export_multiple_item_classes(self):
@@ -734,7 +751,8 @@ class FeedExportTest(FeedExportTestBase):
         formats = {
             'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
             'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'),
-            'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'),
+            'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode(
+                'utf-8'),
             'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),
         }
 
@@ -751,7 +769,8 @@ class FeedExportTest(FeedExportTestBase):
         formats = {
             'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'),
             'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'),
-            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'),
+            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode(
+                'latin-1'),
             'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'),
         }
 
@@ -772,7 +791,8 @@ class FeedExportTest(FeedExportTestBase):
 
         formats = {
             'json': u'[\n{"bar": "BAR"}\n]'.encode('utf-8'),
-            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'.encode('latin-1'),
+            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'.encode(
+                'latin-1'),
             'csv': u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
         }
 
@@ -988,7 +1008,7 @@ class FeedExportTest(FeedExportTestBase):
 
 class PartialDeliveriesTest(FeedExportTestBase):
     __test__ = True
-    _file_mark = '_%(time)s_#%(batch_id)s'
+    _file_mark = '_%(time)s_#%(batch_id)s_'
 
     @defer.inlineCallbacks
     def run_and_export(self, spider_cls, settings):
@@ -999,7 +1019,6 @@ class PartialDeliveriesTest(FeedExportTestBase):
             urljoin('file:', file_path): feed
             for file_path, feed in FEEDS.items()
         }
-        from collections import defaultdict
         content = defaultdict(list)
         try:
             with MockServer() as s:
@@ -1014,26 +1033,120 @@ class PartialDeliveriesTest(FeedExportTestBase):
                         data = f.read()
                         content[feed['format']].append(data)
         finally:
-            pass
+            self.tearDown()
         defer.returnValue(content)
 
     @defer.inlineCallbacks
-    def assertPartialExported(self, items, rows, settings=None):
+    def assertExportedJsonLines(self, items, rows, settings=None):
         settings = settings or {}
         settings.update({
             'FEEDS': {
                 os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
             },
         })
-        data = yield self.exported_data(items, settings)
-        data['jl'] = b''.join(data['jl'])
-        parsed = [json.loads(to_unicode(line)) for line in data['jl'].splitlines()]
-
+        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
         rows = [{k: v for k, v in row.items() if v} for row in rows]
-        self.assertEqual(rows, parsed)
+        data = yield self.exported_data(items, settings)
+        for batch in data['jl']:
+            got_batch = [json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines()]
+            expected_batch, rows = rows[:batch_size], rows[batch_size:]
+            self.assertEqual(expected_batch, got_batch)
 
     @defer.inlineCallbacks
-    def test_partial_deliveries(self):
+    def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
+        settings = settings or {}
+        settings.update({
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
+            },
+        })
+        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        data = yield self.exported_data(items, settings)
+        for batch in data['csv']:
+            got_batch = csv.DictReader(to_unicode(batch).splitlines())
+            self.assertEqual(list(header), got_batch.fieldnames)
+            expected_batch, rows = rows[:batch_size], rows[batch_size:]
+            self.assertEqual(expected_batch, list(got_batch))
+
+    @defer.inlineCallbacks
+    def assertExportedXml(self, items, rows, settings=None):
+        settings = settings or {}
+        settings.update({
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
+            },
+        })
+        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        rows = [{k: v for k, v in row.items() if v} for row in rows]
+        data = yield self.exported_data(items, settings)
+        for batch in data['xml']:
+            root = lxml.etree.fromstring(batch)
+            got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')]
+            expected_batch, rows = rows[:batch_size], rows[batch_size:]
+            self.assertEqual(expected_batch, got_batch)
+
+    @defer.inlineCallbacks
+    def assertExportedMultiple(self, items, rows, settings=None):
+        settings = settings or {}
+        settings.update({
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
+                os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
+            },
+        })
+        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        rows = [{k: v for k, v in row.items() if v} for row in rows]
+        data = yield self.exported_data(items, settings)
+        # XML
+        xml_rows = rows.copy()
+        for batch in data['xml']:
+            root = lxml.etree.fromstring(batch)
+            got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')]
+            expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:]
+            self.assertEqual(expected_batch, got_batch)
+        # JSON
+        json_rows = rows.copy()
+        for batch in data['json']:
+            got_batch = json.loads(batch)
+            expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:]
+            self.assertEqual(expected_batch, got_batch)
+
+    @defer.inlineCallbacks
+    def assertExportedPickle(self, items, rows, settings=None):
+        settings = settings or {}
+        settings.update({
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
+            },
+        })
+        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        rows = [{k: v for k, v in row.items() if v} for row in rows]
+        data = yield self.exported_data(items, settings)
+        import pickle
+        for batch in data['pickle']:
+            got_batch = self._load_until_eof(batch, load_func=pickle.load)
+            expected_batch, rows = rows[:batch_size], rows[batch_size:]
+            self.assertEqual(expected_batch, got_batch)
+
+    @defer.inlineCallbacks
+    def assertExportedMarshal(self, items, rows, settings=None):
+        settings = settings or {}
+        settings.update({
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
+            },
+        })
+        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        rows = [{k: v for k, v in row.items() if v} for row in rows]
+        data = yield self.exported_data(items, settings)
+        import marshal
+        for batch in data['marshal']:
+            got_batch = self._load_until_eof(batch, load_func=marshal.load)
+            expected_batch, rows = rows[:batch_size], rows[batch_size:]
+            self.assertEqual(expected_batch, got_batch)
+
+    @defer.inlineCallbacks
+    def test_export_items(self):
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
             self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
@@ -1042,14 +1155,16 @@ class PartialDeliveriesTest(FeedExportTestBase):
         rows = [
             {'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
             {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'},
-            {'foo': 'bar3', 'baz': 'quux3'}
+            {'foo': 'bar3', 'baz': 'quux3', 'egg': ''}
         ]
         settings = {
-            'FEED_STORAGE_BATCH_SIZE': 1
+            'FEED_STORAGE_BATCH_SIZE': 2
         }
-        yield self.assertPartialExported(items, rows, settings=settings)
+        header = self.MyItem.fields.keys()
+        yield self.assertExported(items, header, rows, settings=settings)
 
     def test_wrong_path(self):
+        """If path without %(time)s or %(batch_id)s an exception must be raised"""
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},
@@ -1069,8 +1184,8 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 'FEED_STORAGE_BATCH_SIZE': 1
             }
             data = yield self.exported_no_data(settings)
-            data[fmt] = b''.join(data[fmt])
-            self.assertEqual(data[fmt], b'')
+            data = dict(data)
+            self.assertEqual(b'', data[fmt][0])
 
     @defer.inlineCallbacks
     def test_export_no_items_store_empty(self):
@@ -1088,8 +1203,8 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 },
                 'FEED_STORE_EMPTY': True,
                 'FEED_EXPORT_INDENT': None,
-                'FEED_STORAGE_BATCH_SIZE': 1
+                'FEED_STORAGE_BATCH_SIZE': 1,
             }
             data = yield self.exported_no_data(settings)
-            data[fmt] = b''.join(data[fmt])
-            self.assertEqual(data[fmt], expctd)
+            data = dict(data)
+            self.assertEqual(expctd, data[fmt][0])

From cac1f3a6adedc32977e0fb1830917a5e7d758bef Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 16 Apr 2020 10:06:56 +0300
Subject: [PATCH 09/57] Update documentation

---
 docs/topics/feed-exports.rst | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 9e5968a29..0bba03a7c 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -220,6 +220,7 @@ These are the settings used for configuring the feed exports:
  * :setting:`FEED_STORAGE_FTP_ACTIVE`
  * :setting:`FEED_STORAGE_S3_ACL`
  * :setting:`FEED_EXPORTERS`
+ * :setting:`FEED_EXPORT_BATCH_SIZE`
 
 .. currentmodule:: scrapy.extensions.feedexport
 
@@ -429,3 +430,37 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 .. _Amazon S3: https://aws.amazon.com/s3/
 .. _botocore: https://github.com/boto/botocore
 .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+
+.. setting:: FEED_EXPORT_BATCH_SIZE
+
+FEED_EXPORT_BATCH_SIZE
+----------------------
+Default: ``None``
+
+An integer number which represent number of scraped items stored in each output
+file. Whenever the number of items exceeds this setting, a new file
+creates and output redirects to it.  The name of the new file will be selected
+based on timestamp when the feed is being created and/or batch sequence number.
+Therefore you must specify %(time)s or %(batch_id)s or both in the file path.
+
+* ``%(time)s`` - gets replaced by a timestamp when the feed is being created
+* ``%(batch_id)s`` - gets replaced by sequence number of batch
+
+For instance::
+
+    FEED_EXPORT_BATCH_SIZE=100
+
+Your request can be like::
+
+  scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time)s.json
+
+The result directory tree of above can be like::
+
+->projectname
+-->dirname
+--->1-filename2020-03-28T14-45-08.237134.json
+--->2-filename2020-03-28T14-45-09.148903.json
+--->3-filename2020-03-28T14-45-10.046092.json
+
+Where first and second files contain exactly 100 items. The last one contains
+<= 100 items.
\ No newline at end of file

From 5980ae72c6cb177f47fbb41d17837e8d98d50025 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 16 Apr 2020 10:13:39 +0300
Subject: [PATCH 10/57] Some minor fixes and refactoring

---
 tests/test_feedexport.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 60e19d1df..e97e50e8e 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -428,7 +428,7 @@ class FeedExportTestBase(ABC, unittest.TestCase):
                     yield item
 
         data = yield self.run_and_export(TestSpider, settings)
-        defer.returnValue(data)
+        return data
 
     @defer.inlineCallbacks
     def exported_no_data(self, settings):
@@ -443,7 +443,7 @@ class FeedExportTestBase(ABC, unittest.TestCase):
                 pass
 
         data = yield self.run_and_export(TestSpider, settings)
-        defer.returnValue(data)
+        return data
 
     @defer.inlineCallbacks
     def assertExported(self, items, header, rows, settings=None, ordered=True):
@@ -735,8 +735,7 @@ class FeedExportTest(FeedExportTestBase):
         formats = {
             'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
             'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'),
-            'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode(
-                'utf-8'),
+            'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'),
             'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),
         }
 
@@ -753,8 +752,7 @@ class FeedExportTest(FeedExportTestBase):
         formats = {
             'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'),
             'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'),
-            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode(
-                'latin-1'),
+            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'),
             'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'),
         }
 
@@ -775,8 +773,7 @@ class FeedExportTest(FeedExportTestBase):
 
         formats = {
             'json': u'[\n{"bar": "BAR"}\n]'.encode('utf-8'),
-            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'.encode(
-                'latin-1'),
+            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'.encode('latin-1'),
             'csv': u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
         }
 
@@ -1148,7 +1145,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
         yield self.assertExported(items, header, rows, settings=settings)
 
     def test_wrong_path(self):
-        """If path without %(time)s or %(batch_id)s an exception must be raised"""
+        """If path is without %(time)s or %(batch_id)s an exception must be raised"""
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},

From ec76445dd6753074c1531571f66467eecf22b498 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Sat, 18 Apr 2020 09:29:23 +0300
Subject: [PATCH 11/57] Update tests

---
 tests/test_feedexport.py | 66 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 65 insertions(+), 1 deletion(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index e97e50e8e..8e03a91c8 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1128,6 +1128,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_export_items(self):
+        """ Test partial deliveries in all supported formats """
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
             self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
@@ -1145,7 +1146,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
         yield self.assertExported(items, header, rows, settings=settings)
 
     def test_wrong_path(self):
-        """If path is without %(time)s or %(batch_id)s an exception must be raised"""
+        """ If path is without %(time)s or %(batch_id)s an exception must be raised """
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},
@@ -1189,3 +1190,66 @@ class PartialDeliveriesTest(FeedExportTestBase):
             data = yield self.exported_no_data(settings)
             data = dict(data)
             self.assertEqual(expctd, data[fmt][0])
+
+    @defer.inlineCallbacks
+    def test_export_multiple_configs(self):
+        items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]
+
+        formats = {
+            'json': [u'[\n{"bar": "BAR"}\n]'.encode('utf-8'),
+                     u'[\n{"bar": "BAR1"}\n]'.encode('utf-8')],
+            'xml': [u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'.encode('latin-1'),
+                    u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO1</foo>\n  </item>\n</items>'.encode('latin-1')],
+            'csv': [u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
+                    u'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')],
+        }
+
+        settings = {
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'json', self._file_mark): {
+                    'format': 'json',
+                    'indent': 0,
+                    'fields': ['bar'],
+                    'encoding': 'utf-8',
+                },
+                os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {
+                    'format': 'xml',
+                    'indent': 2,
+                    'fields': ['foo'],
+                    'encoding': 'latin-1',
+                },
+                os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {
+                    'format': 'csv',
+                    'indent': None,
+                    'fields': ['bar', 'foo'],
+                    'encoding': 'utf-8',
+                },
+            },
+            'FEED_STORAGE_BATCH_SIZE': 1,
+        }
+        data = yield self.exported_data(items, settings)
+        for fmt, expected in formats.items():
+            for expected_batch, got_batch in zip(expected, data[fmt]):
+                self.assertEqual(expected_batch, got_batch)
+
+    @defer.inlineCallbacks
+    def test_batch_path_differ(self):
+        """
+        Test that the name of all batch files differ from each other.
+        So %(time)s replaced with the current date.
+        """
+        items = [
+            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+            self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
+            self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
+        ]
+        settings = {
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), '%(time)s'): {
+                    'format': 'json',
+                },
+            },
+            'FEED_STORAGE_BATCH_SIZE': 1,
+        }
+        data = yield self.exported_data(items, settings)
+        self.assertEqual(len(items) + 1, len(data['json']))

From f0f1be76d1e6cef65ac9a01d13c5d5060a03f648 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Mon, 27 Apr 2020 09:56:57 +0300
Subject: [PATCH 12/57] Using time_id instead of time as a timestamp

---
 docs/topics/feed-exports.rst    |  6 +++---
 scrapy/extensions/feedexport.py | 11 ++++++-----
 tests/test_feedexport.py        |  8 ++++----
 3 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 0bba03a7c..2017be78f 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -441,9 +441,9 @@ An integer number which represent number of scraped items stored in each output
 file. Whenever the number of items exceeds this setting, a new file
 creates and output redirects to it.  The name of the new file will be selected
 based on timestamp when the feed is being created and/or batch sequence number.
-Therefore you must specify %(time)s or %(batch_id)s or both in the file path.
+Therefore you must specify %(time_id)s or %(batch_id)s or both in the file path.
 
-* ``%(time)s`` - gets replaced by a timestamp when the feed is being created
+* ``%(time_id)s`` - gets replaced by a timestamp when the feed is being created
 * ``%(batch_id)s`` - gets replaced by sequence number of batch
 
 For instance::
@@ -452,7 +452,7 @@ For instance::
 
 Your request can be like::
 
-  scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time)s.json
+  scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time_id)s.json
 
 The result directory tree of above can be like::
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 06ea6c5b2..72baa6269 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -292,7 +292,7 @@ class FeedExporter:
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
         :param spider: user spider
-        :param template_uri: template uri which contains %(time)s or %(batch_id)s to create new uri
+        :param template_uri: template uri which contains %(time_id)s or %(batch_id)s to create new uri
         """
         if previous_batch_slot is not None:
             previous_batch_id = previous_batch_slot.batch_id
@@ -360,12 +360,12 @@ class FeedExporter:
 
     def _batch_deliveries_supported(self, uri):
         """
-        If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time)s or %(batch_id)s
+        If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time_id)s or %(batch_id)s
         to distinguish different files of partial output
         """
-        if self.storage_batch_size is None or '%(time)s' in uri or '%(batch_id)s' in uri:
+        if self.storage_batch_size is None or '%(time_id)s' in uri or '%(batch_id)s' in uri:
             return True
-        logger.warning('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
+        logger.warning('%(time_id)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
         return False
 
     def _storage_supported(self, uri):
@@ -397,8 +397,9 @@ class FeedExporter:
         params = {}
         for k in dir(spider):
             params[k] = getattr(spider, k)
+        params['time'] = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')
+        params['time_id'] = datetime.utcnow().isoformat().replace(':', '-')
         params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
-        params['time'] = datetime.utcnow().isoformat().replace(':', '-')
         uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
         uripar_function(params, spider)
         return params
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 8e03a91c8..da759917a 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -989,7 +989,7 @@ class FeedExportTest(FeedExportTestBase):
 
 class PartialDeliveriesTest(FeedExportTestBase):
     __test__ = True
-    _file_mark = '_%(time)s_#%(batch_id)s_'
+    _file_mark = '_%(time_id)s_#%(batch_id)s_'
 
     @defer.inlineCallbacks
     def run_and_export(self, spider_cls, settings):
@@ -1146,7 +1146,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
         yield self.assertExported(items, header, rows, settings=settings)
 
     def test_wrong_path(self):
-        """ If path is without %(time)s or %(batch_id)s an exception must be raised """
+        """ If path is without %(time_id)s or %(batch_id)s an exception must be raised """
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},
@@ -1236,7 +1236,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
     def test_batch_path_differ(self):
         """
         Test that the name of all batch files differ from each other.
-        So %(time)s replaced with the current date.
+        So %(time_id)s replaced with the current date.
         """
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
@@ -1245,7 +1245,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
         ]
         settings = {
             'FEEDS': {
-                os.path.join(self._random_temp_filename(), '%(time)s'): {
+                os.path.join(self._random_temp_filename(), '%(time_id)s'): {
                     'format': 'json',
                 },
             },

From 2eee6c81017e08bb492da560bc73c03f4f375fcc Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Mon, 27 Apr 2020 09:58:14 +0300
Subject: [PATCH 13/57] Documentation spelling fix

---
 docs/topics/feed-exports.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 2017be78f..6c463fc27 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -438,10 +438,10 @@ FEED_EXPORT_BATCH_SIZE
 Default: ``None``
 
 An integer number which represent number of scraped items stored in each output
-file. Whenever the number of items exceeds this setting, a new file
-creates and output redirects to it.  The name of the new file will be selected
+file. Whenever the number of items exceeds this setting, a new file is
+created and output redirects to it.  The name of the new file will be selected
 based on timestamp when the feed is being created and/or batch sequence number.
-Therefore you must specify %(time_id)s or %(batch_id)s or both in the file path.
+Therefore you must specify %(time_id)s or %(batch_id)s or both in FEED_URI.
 
 * ``%(time_id)s`` - gets replaced by a timestamp when the feed is being created
 * ``%(batch_id)s`` - gets replaced by sequence number of batch

From 204737042ac6672eee73c975d0bd6735893d684c Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Mon, 27 Apr 2020 12:52:18 +0300
Subject: [PATCH 14/57] Extract the slot closing functionality to the function;
 minor changes

---
 scrapy/extensions/feedexport.py | 56 ++++++++++++++++-----------------
 1 file changed, 27 insertions(+), 29 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 72baa6269..fe6061c33 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -255,7 +255,7 @@ class FeedExporter:
         for uri, feed in self.feeds.items():
             uri_params = self._get_uri_params(spider, feed['uri_params'], None)
             self.slots.append(self._start_new_batch(
-                previous_batch_slot=None,
+                batch_id=1,
                 uri=uri % uri_params,
                 feed=feed,
                 spider=spider,
@@ -265,42 +265,38 @@ class FeedExporter:
     def close_spider(self, spider):
         deferred_list = []
         for slot in self.slots:
-            if not slot.itemcount and not slot.store_empty:
-                # We need to call slot.storage.store nonetheless to get the file
-                # properly closed.
-                return defer.maybeDeferred(slot.storage.store, slot.file)
-            slot.finish_exporting()
-            logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
-            log_args = {'format': slot.format,
-                        'itemcount': slot.itemcount,
-                        'uri': slot.uri}
-            d = defer.maybeDeferred(slot.storage.store, slot.file)
-            d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args,
-                                                extra={'spider': spider}))
-            d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args,
-                                                exc_info=failure_to_exc_info(f),
-                                                extra={'spider': spider}))
+            d = self._close_slot(slot, spider)
             deferred_list.append(d)
         return defer.DeferredList(deferred_list) if deferred_list else None
 
-    def _start_new_batch(self, previous_batch_slot, uri, feed, spider, template_uri):
+    def _close_slot(self, slot, spider):
+        if not slot.itemcount and not slot.store_empty:
+            # We need to call slot.storage.store nonetheless to get the file
+            # properly closed.
+            return defer.maybeDeferred(slot.storage.store, slot.file)
+        slot.finish_exporting()
+        logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
+        log_args = {'format': slot.format,
+                    'itemcount': slot.itemcount,
+                    'uri': slot.uri}
+        d = defer.maybeDeferred(slot.storage.store, slot.file)
+        d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args,
+                                            extra={'spider': spider}))
+        d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args,
+                                            exc_info=failure_to_exc_info(f),
+                                            extra={'spider': spider}))
+        return d
+
+    def _start_new_batch(self, batch_id, uri, feed, spider, template_uri):
         """
         Redirect the output data stream to a new file.
         Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified.
-        :param previous_batch_slot: slot of previous batch. We need to call slot.storage.store
-        to get the file properly closed.
+        :param batch_id: sequence number of current batch
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
         :param spider: user spider
         :param template_uri: template uri which contains %(time_id)s or %(batch_id)s to create new uri
         """
-        if previous_batch_slot is not None:
-            previous_batch_id = previous_batch_slot.batch_id
-            previous_batch_slot.exporter.finish_exporting()
-            previous_batch_slot.storage.store(previous_batch_slot.file)
-        else:
-            previous_batch_id = 0
-
         storage = self._get_storage(uri)
         file = storage.open(spider)
         exporter = self._get_exporter(
@@ -317,7 +313,7 @@ class FeedExporter:
             uri=uri,
             format=feed['format'],
             store_empty=feed['store_empty'],
-            batch_id=previous_batch_id + 1,
+            batch_id=batch_id,
             template_uri=template_uri,
         )
         if slot.store_empty:
@@ -330,10 +326,12 @@ class FeedExporter:
             slot.start_exporting()
             slot.exporter.export_item(item)
             slot.itemcount += 1
-            if self.storage_batch_size and slot.itemcount % self.storage_batch_size == 0:
+            # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_SIZE and close the old one
+            if self.storage_batch_size and slot.itemcount == self.storage_batch_size:
                 uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot)
+                self._close_slot(slot, spider)
                 slots.append(self._start_new_batch(
-                    previous_batch_slot=slot,
+                    batch_id=slot.batch_id + 1,
                     uri=slot.template_uri % uri_params,
                     feed=self.feeds[slot.template_uri],
                     spider=spider,

From 3f9874fac9f93c0956afa5975d7b2bbb21816894 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Fri, 1 May 2020 11:52:16 +0300
Subject: [PATCH 15/57] Add test s3 export

---
 tests/test_feedexport.py | 70 ++++++++++++++++++++++++++++++++++++++++
 tox.ini                  |  1 +
 2 files changed, 71 insertions(+)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index da759917a..9fc39c3a6 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1253,3 +1253,73 @@ class PartialDeliveriesTest(FeedExportTestBase):
         }
         data = yield self.exported_data(items, settings)
         self.assertEqual(len(items) + 1, len(data['json']))
+
+    @defer.inlineCallbacks
+    def test_s3_export(self):
+        """
+        Test export of items into s3 bucket.
+        S3_TEST_BUCKET_NAME, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY must be specified in tox.ini
+        to perform this test:
+        [testenv]
+        setenv =
+            AWS_SECRET_ACCESS_KEY = ABCD
+            AWS_ACCESS_KEY_ID = ABCD
+            S3_TEST_BUCKET_NAME = ABCD
+        """
+        try:
+            import boto3
+        except ImportError:
+            raise unittest.SkipTest("S3FeedStorage requires boto3")
+
+        assert_aws_environ()
+        s3_test_bucket_name = os.environ.get('S3_TEST_BUCKET_NAME')
+        access_key = os.environ.get('AWS_ACCESS_KEY_ID')
+        secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
+        if not s3_test_bucket_name:
+            raise unittest.SkipTest("No S3 BUCKET available for testing")
+
+        chars = [random.choice(ascii_letters + digits) for _ in range(15)]
+        filename = ''.join(chars)
+        prefix = 'tmp/{filename}'.format(filename=filename)
+        s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(time_id)s.json'.format(
+            bucket_name=s3_test_bucket_name, prefix=prefix
+        )
+        storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key)
+        settings = {
+            'FEEDS': {
+                s3_test_file_uri: {
+                    'format': 'json',
+                },
+            },
+            'FEED_STORAGE_BATCH_SIZE': 1,
+        }
+        items = [
+            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+            self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
+            self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
+        ]
+        verifyObject(IFeedStorage, storage)
+
+        class TestSpider(scrapy.Spider):
+            name = 'testspider'
+
+            def parse(self, response):
+                for item in items:
+                    yield item
+
+        s3 = boto3.resource('s3')
+        my_bucket = s3.Bucket(s3_test_bucket_name)
+        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+
+        with MockServer() as s:
+            runner = CrawlerRunner(Settings(settings))
+            TestSpider.start_urls = [s.url('/')]
+            yield runner.crawl(TestSpider)
+
+        for file_uri in my_bucket.objects.filter(Prefix=prefix):
+            content = get_s3_content_and_delete(s3_test_bucket_name, file_uri.key)
+            if not content and not items:
+                break
+            content = json.loads(content.decode('utf-8'))
+            expected_batch, items = items[:batch_size], items[batch_size:]
+            self.assertEqual(expected_batch, content)
diff --git a/tox.ini b/tox.ini
index cd118c921..c77fae1f0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -14,6 +14,7 @@ deps =
     # Extras
     botocore>=1.3.23
     Pillow>=3.4.2
+    boto3>=1.13.0
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID

From dad2ea75222d6240c569440d3221f5fc00925682 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Sat, 2 May 2020 01:21:03 +0300
Subject: [PATCH 16/57] Change time_id to batch_time

---
 docs/topics/feed-exports.rst    |  6 +++---
 scrapy/extensions/feedexport.py | 10 +++++-----
 tests/test_feedexport.py        | 10 +++++-----
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 6c463fc27..2106b41f5 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -441,9 +441,9 @@ An integer number which represent number of scraped items stored in each output
 file. Whenever the number of items exceeds this setting, a new file is
 created and output redirects to it.  The name of the new file will be selected
 based on timestamp when the feed is being created and/or batch sequence number.
-Therefore you must specify %(time_id)s or %(batch_id)s or both in FEED_URI.
+Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI.
 
-* ``%(time_id)s`` - gets replaced by a timestamp when the feed is being created
+* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
 * ``%(batch_id)s`` - gets replaced by sequence number of batch
 
 For instance::
@@ -452,7 +452,7 @@ For instance::
 
 Your request can be like::
 
-  scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time_id)s.json
+  scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
 
 The result directory tree of above can be like::
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index fe6061c33..a262f5d18 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -295,7 +295,7 @@ class FeedExporter:
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
         :param spider: user spider
-        :param template_uri: template uri which contains %(time_id)s or %(batch_id)s to create new uri
+        :param template_uri: template uri which contains %(batch_time)s or %(batch_id)s to create new uri
         """
         storage = self._get_storage(uri)
         file = storage.open(spider)
@@ -358,12 +358,12 @@ class FeedExporter:
 
     def _batch_deliveries_supported(self, uri):
         """
-        If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time_id)s or %(batch_id)s
+        If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(batch_time)s or %(batch_id)s
         to distinguish different files of partial output
         """
-        if self.storage_batch_size is None or '%(time_id)s' in uri or '%(batch_id)s' in uri:
+        if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
             return True
-        logger.warning('%(time_id)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
+        logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
         return False
 
     def _storage_supported(self, uri):
@@ -396,7 +396,7 @@ class FeedExporter:
         for k in dir(spider):
             params[k] = getattr(spider, k)
         params['time'] = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')
-        params['time_id'] = datetime.utcnow().isoformat().replace(':', '-')
+        params['batch_time'] = datetime.utcnow().isoformat().replace(':', '-')
         params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
         uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
         uripar_function(params, spider)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 9fc39c3a6..2217bb4ed 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -989,7 +989,7 @@ class FeedExportTest(FeedExportTestBase):
 
 class PartialDeliveriesTest(FeedExportTestBase):
     __test__ = True
-    _file_mark = '_%(time_id)s_#%(batch_id)s_'
+    _file_mark = '_%(batch_time)s_#%(batch_id)s_'
 
     @defer.inlineCallbacks
     def run_and_export(self, spider_cls, settings):
@@ -1146,7 +1146,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
         yield self.assertExported(items, header, rows, settings=settings)
 
     def test_wrong_path(self):
-        """ If path is without %(time_id)s or %(batch_id)s an exception must be raised """
+        """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},
@@ -1236,7 +1236,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
     def test_batch_path_differ(self):
         """
         Test that the name of all batch files differ from each other.
-        So %(time_id)s replaced with the current date.
+        So %(batch_time)s replaced with the current date.
         """
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
@@ -1245,7 +1245,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
         ]
         settings = {
             'FEEDS': {
-                os.path.join(self._random_temp_filename(), '%(time_id)s'): {
+                os.path.join(self._random_temp_filename(), '%(batch_time)s'): {
                     'format': 'json',
                 },
             },
@@ -1281,7 +1281,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
         chars = [random.choice(ascii_letters + digits) for _ in range(15)]
         filename = ''.join(chars)
         prefix = 'tmp/{filename}'.format(filename=filename)
-        s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(time_id)s.json'.format(
+        s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(batch_time)s.json'.format(
             bucket_name=s3_test_bucket_name, prefix=prefix
         )
         storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key)

From b5684909d1cb01ad138a389caa750485b51f79cf Mon Sep 17 00:00:00 2001
From: Jacty <jacty@users.noreply.github.com>
Date: Mon, 11 May 2020 11:18:25 +0800
Subject: [PATCH 17/57] Unnecessary update when value is None

When value is None, it is not necessary to invoke update and run other methods and conditions to make the code complicated there.
---
 scrapy/settings/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index b9a13c018..f28fbfaf9 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -83,7 +83,8 @@ class BaseSettings(MutableMapping):
     def __init__(self, values=None, priority='project'):
         self.frozen = False
         self.attributes = {}
-        self.update(values, priority)
+        if values is not None:
+            self.update(values, priority)
 
     def __getitem__(self, opt_name):
         if opt_name not in self:

From 33ab0a36635fbd45debbc44584002bd7a4ef7fed Mon Sep 17 00:00:00 2001
From: Jacty <jacty@users.noreply.github.com>
Date: Wed, 13 May 2020 06:11:07 +0800
Subject: [PATCH 18/57] Update __init__.py

---
 scrapy/settings/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index f28fbfaf9..0425b48b3 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -83,7 +83,7 @@ class BaseSettings(MutableMapping):
     def __init__(self, values=None, priority='project'):
         self.frozen = False
         self.attributes = {}
-        if values is not None:
+        if values:
             self.update(values, priority)
 
     def __getitem__(self, opt_name):

From 2327ecead085a41d1a71a70a12eb988bbf982268 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Wed, 13 May 2020 22:50:04 +0300
Subject: [PATCH 19/57] Rename FEED_STORAGE_BATCH_SIZE to
 FEED_STORAGE_BATCH_ITEM_COUNT

---
 docs/topics/feed-exports.rst        |  8 ++++----
 scrapy/extensions/feedexport.py     | 10 +++++-----
 scrapy/settings/default_settings.py |  2 +-
 tests/test_feedexport.py            | 28 ++++++++++++++--------------
 4 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 2106b41f5..917240d4d 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -220,7 +220,7 @@ These are the settings used for configuring the feed exports:
  * :setting:`FEED_STORAGE_FTP_ACTIVE`
  * :setting:`FEED_STORAGE_S3_ACL`
  * :setting:`FEED_EXPORTERS`
- * :setting:`FEED_EXPORT_BATCH_SIZE`
+ * :setting:`FEED_STORAGE_BATCH_ITEM_COUNT`
 
 .. currentmodule:: scrapy.extensions.feedexport
 
@@ -431,9 +431,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 .. _botocore: https://github.com/boto/botocore
 .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
-.. setting:: FEED_EXPORT_BATCH_SIZE
+.. setting:: FEED_STORAGE_BATCH_ITEM_COUNT
 
-FEED_EXPORT_BATCH_SIZE
+FEED_STORAGE_BATCH_ITEM_COUNT
 ----------------------
 Default: ``None``
 
@@ -448,7 +448,7 @@ Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI.
 
 For instance::
 
-    FEED_EXPORT_BATCH_SIZE=100
+    FEED_STORAGE_BATCH_ITEM_COUNT=100
 
 Your request can be like::
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index a262f5d18..5bc946634 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -242,7 +242,7 @@ class FeedExporter:
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
-        self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_SIZE', None)
+        self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None)
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
@@ -290,7 +290,7 @@ class FeedExporter:
     def _start_new_batch(self, batch_id, uri, feed, spider, template_uri):
         """
         Redirect the output data stream to a new file.
-        Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified.
+        Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified.
         :param batch_id: sequence number of current batch
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
@@ -326,7 +326,7 @@ class FeedExporter:
             slot.start_exporting()
             slot.exporter.export_item(item)
             slot.itemcount += 1
-            # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_SIZE and close the old one
+            # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
             if self.storage_batch_size and slot.itemcount == self.storage_batch_size:
                 uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot)
                 self._close_slot(slot, spider)
@@ -358,12 +358,12 @@ class FeedExporter:
 
     def _batch_deliveries_supported(self, uri):
         """
-        If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(batch_time)s or %(batch_id)s
+        If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s
         to distinguish different files of partial output
         """
         if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
             return True
-        logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
+        logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified')
         return False
 
     def _storage_supported(self, uri):
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index c3463a505..5a7dc533e 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -146,7 +146,7 @@ FEED_STORAGES_BASE = {
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
 }
-FEED_STORAGE_BATCH_SIZE = None
+FEED_STORAGE_BATCH_ITEM_COUNT = None
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
     'json': 'scrapy.exporters.JsonItemExporter',
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 2217bb4ed..1a21eeba9 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1025,7 +1025,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['jl']:
@@ -1041,7 +1041,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
         data = yield self.exported_data(items, settings)
         for batch in data['csv']:
             got_batch = csv.DictReader(to_unicode(batch).splitlines())
@@ -1057,7 +1057,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['xml']:
@@ -1075,7 +1075,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         # XML
@@ -1100,7 +1100,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import pickle
@@ -1117,7 +1117,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import marshal
@@ -1140,7 +1140,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
             {'foo': 'bar3', 'baz': 'quux3', 'egg': ''}
         ]
         settings = {
-            'FEED_STORAGE_BATCH_SIZE': 2
+            'FEED_STORAGE_BATCH_ITEM_COUNT': 2
         }
         header = self.MyItem.fields.keys()
         yield self.assertExported(items, header, rows, settings=settings)
@@ -1151,7 +1151,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},
             },
-            'FEED_STORAGE_BATCH_SIZE': 1
+            'FEED_STORAGE_BATCH_ITEM_COUNT': 1
         }
         crawler = get_crawler(settings_dict=settings)
         self.assertRaises(NotConfigured, FeedExporter, crawler)
@@ -1163,7 +1163,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 'FEEDS': {
                     os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
                 },
-                'FEED_STORAGE_BATCH_SIZE': 1
+                'FEED_STORAGE_BATCH_ITEM_COUNT': 1
             }
             data = yield self.exported_no_data(settings)
             data = dict(data)
@@ -1185,7 +1185,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 },
                 'FEED_STORE_EMPTY': True,
                 'FEED_EXPORT_INDENT': None,
-                'FEED_STORAGE_BATCH_SIZE': 1,
+                'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
             }
             data = yield self.exported_no_data(settings)
             data = dict(data)
@@ -1225,7 +1225,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                     'encoding': 'utf-8',
                 },
             },
-            'FEED_STORAGE_BATCH_SIZE': 1,
+            'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
         }
         data = yield self.exported_data(items, settings)
         for fmt, expected in formats.items():
@@ -1249,7 +1249,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                     'format': 'json',
                 },
             },
-            'FEED_STORAGE_BATCH_SIZE': 1,
+            'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
         }
         data = yield self.exported_data(items, settings)
         self.assertEqual(len(items) + 1, len(data['json']))
@@ -1291,7 +1291,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                     'format': 'json',
                 },
             },
-            'FEED_STORAGE_BATCH_SIZE': 1,
+            'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
         }
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
@@ -1309,7 +1309,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
 
         s3 = boto3.resource('s3')
         my_bucket = s3.Bucket(s3_test_bucket_name)
-        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
 
         with MockServer() as s:
             runner = CrawlerRunner(Settings(settings))

From 8662d3587df74841d4ea640c0432446569e59262 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Wed, 13 May 2020 23:41:01 +0300
Subject: [PATCH 20/57] Documentation and code refactoring

---
 docs/topics/feed-exports.rst    | 21 ++++++++++++---------
 scrapy/extensions/feedexport.py |  7 ++++---
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 917240d4d..0f15044b3 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -437,24 +437,27 @@ FEED_STORAGE_BATCH_ITEM_COUNT
 ----------------------
 Default: ``None``
 
-An integer number which represent number of scraped items stored in each output
+An integer number that represents the number of scraped items stored in each output
 file. Whenever the number of items exceeds this setting, a new file is
-created and output redirects to it.  The name of the new file will be selected
-based on timestamp when the feed is being created and/or batch sequence number.
-Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI.
+created and the output is redirected to it.  The name of the new file will be selected
+based on the timestamp when the feed is being created and/or on the batch sequence number.
+Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`.
 
 * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
-* ``%(batch_id)s`` - gets replaced by sequence number of batch
+(e.g. `2020-03-28T14-45-08.237134`)
 
-For instance::
+* ``%(batch_id)s`` - gets replaced by the batch  sequence number of batch
+(e.g. `2` for the second file)
+
+For instance, if your settings include::
 
     FEED_STORAGE_BATCH_ITEM_COUNT=100
 
-Your request can be like::
+And your :command:`crawl` command line is::
 
   scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
 
-The result directory tree of above can be like::
+The resulting directory tree of above can be like::
 
 ->projectname
 -->dirname
@@ -462,5 +465,5 @@ The result directory tree of above can be like::
 --->2-filename2020-03-28T14-45-09.148903.json
 --->3-filename2020-03-28T14-45-10.046092.json
 
-Where first and second files contain exactly 100 items. The last one contains
+Where the first and second files contain exactly 100 items. The last one contains
 <= 100 items.
\ No newline at end of file
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 5bc946634..4c9362f3a 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -25,6 +25,7 @@ from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import without_none_values
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -337,9 +338,9 @@ class FeedExporter:
                     spider=spider,
                     template_uri=slot.template_uri,
                 ))
-                self.slots[idx] = None
-        self.slots = [slot for slot in self.slots if slot is not None]
-        self.slots.extend(slots)
+            else:
+                slots.append(slot)
+        self.slots = slots
 
     def _load_components(self, setting_prefix):
         conf = without_none_values(self.settings.getwithbase(setting_prefix))

From 69c005f013eb0dc000611853e66371fad17dea9d Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 14 May 2020 10:35:56 +0300
Subject: [PATCH 21/57] Documentation indent fix

---
 docs/topics/feed-exports.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 42c4e2267..dfeea5b7f 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -444,10 +444,10 @@ based on the timestamp when the feed is being created and/or on the batch sequen
 Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`.
 
 * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
-(e.g. `2020-03-28T14-45-08.237134`)
+  (e.g. `2020-03-28T14-45-08.237134`)
 
 * ``%(batch_id)s`` - gets replaced by the batch  sequence number of batch
-(e.g. `2` for the second file)
+  (e.g. `2` for the second file)
 
 For instance, if your settings include::
 

From 1cdcf8b08b8f1e68c5b107b6ae39b2da1aedd245 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Fri, 15 May 2020 19:46:36 +0300
Subject: [PATCH 22/57] Minor fixes

---
 docs/topics/feed-exports.rst    | 19 ++++++++++---------
 scrapy/extensions/feedexport.py | 20 ++++++++++----------
 2 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index dfeea5b7f..638733b6a 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -437,17 +437,18 @@ FEED_STORAGE_BATCH_ITEM_COUNT
 -----------------------------
 Default: ``None``
 
-An integer number that represents the number of scraped items stored in each output
-file. Whenever the number of items exceeds this setting, a new file is
-created and the output is redirected to it.  The name of the new file will be selected
-based on the timestamp when the feed is being created and/or on the batch sequence number.
-Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`.
+If assigned an integer number higher than ``0``, Scrapy generates multiple output files
+storing up to the specified number of items in each output file.
+
+When generating multiple output files, you must use at least one of the following
+placeholders in :setting:`FEED_URI` to indicate how the different output file names are
+generated:
 
 * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
-  (e.g. `2020-03-28T14-45-08.237134`)
+  (e.g. ``2020-03-28T14-45-08.237134``)
 
 * ``%(batch_id)s`` - gets replaced by the batch  sequence number of batch
-  (e.g. `2` for the second file)
+  (e.g. ``2`` for the second file)
 
 For instance, if your settings include::
 
@@ -457,7 +458,7 @@ And your :command:`crawl` command line is::
 
   scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
 
-The resulting directory tree of above can be like::
+The command line above can generate a directory tree like::
 
 ->projectname
 -->dirname
@@ -466,4 +467,4 @@ The resulting directory tree of above can be like::
 --->3-filename2020-03-28T14-45-10.046092.json
 
 Where the first and second files contain exactly 100 items. The last one contains
-<= 100 items.
\ No newline at end of file
+100 items or fever.
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 4c9362f3a..3d691c580 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -180,7 +180,7 @@ class FTPFeedStorage(BlockingFeedStorage):
 
 
 class _FeedSlot:
-    def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, template_uri):
+    def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template):
         self.file = file
         self.exporter = exporter
         self.storage = storage
@@ -188,7 +188,7 @@ class _FeedSlot:
         self.batch_id = batch_id
         self.format = format
         self.store_empty = store_empty
-        self.template_uri = template_uri
+        self.uri_template = uri_template
         self.uri = uri
         # flags
         self.itemcount = 0
@@ -260,7 +260,7 @@ class FeedExporter:
                 uri=uri % uri_params,
                 feed=feed,
                 spider=spider,
-                template_uri=uri,
+                uri_template=uri,
             ))
 
     def close_spider(self, spider):
@@ -288,7 +288,7 @@ class FeedExporter:
                                             extra={'spider': spider}))
         return d
 
-    def _start_new_batch(self, batch_id, uri, feed, spider, template_uri):
+    def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
         """
         Redirect the output data stream to a new file.
         Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified.
@@ -296,7 +296,7 @@ class FeedExporter:
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
         :param spider: user spider
-        :param template_uri: template uri which contains %(batch_time)s or %(batch_id)s to create new uri
+        :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)s to create new uri
         """
         storage = self._get_storage(uri)
         file = storage.open(spider)
@@ -315,7 +315,7 @@ class FeedExporter:
             format=feed['format'],
             store_empty=feed['store_empty'],
             batch_id=batch_id,
-            template_uri=template_uri,
+            uri_template=uri_template,
         )
         if slot.store_empty:
             slot.start_exporting()
@@ -329,14 +329,14 @@ class FeedExporter:
             slot.itemcount += 1
             # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
             if self.storage_batch_size and slot.itemcount == self.storage_batch_size:
-                uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot)
+                uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
                 self._close_slot(slot, spider)
                 slots.append(self._start_new_batch(
                     batch_id=slot.batch_id + 1,
-                    uri=slot.template_uri % uri_params,
-                    feed=self.feeds[slot.template_uri],
+                    uri=slot.uri_template % uri_params,
+                    feed=self.feeds[slot.uri_template],
                     spider=spider,
-                    template_uri=slot.template_uri,
+                    uri_template=slot.uri_template,
                 ))
             else:
                 slots.append(slot)

From 10ae1a284f759b541d086e3d1a13cda96b6e2040 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Fri, 15 May 2020 22:50:54 +0300
Subject: [PATCH 23/57] Minor fixes

---
 docs/topics/feed-exports.rst    |  2 +-
 scrapy/extensions/feedexport.py | 10 +++++-----
 tests/test_feedexport.py        |  2 +-
 tox.ini                         |  2 +-
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 638733b6a..6f7db20c4 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -467,4 +467,4 @@ The command line above can generate a directory tree like::
 --->3-filename2020-03-28T14-45-10.046092.json
 
 Where the first and second files contain exactly 100 items. The last one contains
-100 items or fever.
+100 items or fewer.
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 3d691c580..cc26ae173 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -243,11 +243,11 @@ class FeedExporter:
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
-        self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None)
+        self.storage_batch_item_count = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None)
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
-            if not self._batch_deliveries_supported(uri):
+            if not self._settings_are_valid(uri):
                 raise NotConfigured
             if not self._exporter_supported(feed['format']):
                 raise NotConfigured
@@ -328,7 +328,7 @@ class FeedExporter:
             slot.exporter.export_item(item)
             slot.itemcount += 1
             # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
-            if self.storage_batch_size and slot.itemcount == self.storage_batch_size:
+            if self.storage_batch_item_count and slot.itemcount == self.storage_batch_item_count:
                 uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
                 self._close_slot(slot, spider)
                 slots.append(self._start_new_batch(
@@ -357,12 +357,12 @@ class FeedExporter:
             return True
         logger.error("Unknown feed format: %(format)s", {'format': format})
 
-    def _batch_deliveries_supported(self, uri):
+    def _settings_are_valid(self, uri):
         """
         If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s
         to distinguish different files of partial output
         """
-        if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
+        if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
             return True
         logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified')
         return False
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index d1374f291..88f9a5933 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -986,7 +986,7 @@ class FeedExportTest(FeedExportTestBase):
         self.assertEqual(data['csv'], b'')
 
 
-class PartialDeliveriesTest(FeedExportTestBase):
+class BatchDeliveriesTest(FeedExportTestBase):
     __test__ = True
     _file_mark = '_%(batch_time)s_#%(batch_id)s_'
 
diff --git a/tox.ini b/tox.ini
index 6dd944dff..7507a14a6 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,9 +12,9 @@ deps =
     -ctests/constraints.txt
     -rtests/requirements-py3.txt
     # Extras
+    boto3>=1.13.0
     botocore>=1.3.23
     Pillow>=3.4.2
-    boto3>=1.13.0
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID

From a7d070f3bb350cbe1f7b580350d5f491f59d47d8 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Mon, 18 May 2020 22:25:29 +0300
Subject: [PATCH 24/57] Change log level to error

---
 scrapy/extensions/feedexport.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index cc26ae173..ce7fc372d 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -364,7 +364,7 @@ class FeedExporter:
         """
         if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
             return True
-        logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified')
+        logger.error('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified')
         return False
 
     def _storage_supported(self, uri):

From 677e619d3761e6669c247786bb95822ce38c8080 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 21 May 2020 14:57:03 +0300
Subject: [PATCH 25/57] Fix too long lines

---
 scrapy/extensions/feedexport.py |  4 +++-
 tests/test_feedexport.py        | 20 ++++++++++++++------
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index ce7fc372d..1f745be98 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -364,7 +364,9 @@ class FeedExporter:
         """
         if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
             return True
-        logger.error('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified')
+        logger.error(
+            '%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified'
+        )
         return False
 
     def _storage_supported(self, uri):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 08ee24768..fecb17e29 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1204,12 +1204,20 @@ class BatchDeliveriesTest(FeedExportTestBase):
         items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]
 
         formats = {
-            'json': [u'[\n{"bar": "BAR"}\n]'.encode('utf-8'),
-                     u'[\n{"bar": "BAR1"}\n]'.encode('utf-8')],
-            'xml': [u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'.encode('latin-1'),
-                    u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO1</foo>\n  </item>\n</items>'.encode('latin-1')],
-            'csv': [u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
-                    u'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')],
+            'json': ['[\n{"bar": "BAR"}\n]'.encode('utf-8'),
+                     '[\n{"bar": "BAR1"}\n]'.encode('utf-8')],
+            'xml': [
+                (
+                    '<?xml version="1.0" encoding="latin-1"?>\n'
+                    '<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'
+                ).encode('latin-1'),
+                (
+                    '<?xml version="1.0" encoding="latin-1"?>\n'
+                    '<items>\n  <item>\n    <foo>FOO1</foo>\n  </item>\n</items>'
+                ).encode('latin-1')
+            ],
+            'csv': ['bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
+                    'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')],
         }
 
         settings = {

From dd96f94e8cc1517b7021e35e46cbdc92580c6333 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Fri, 22 May 2020 23:30:33 +0300
Subject: [PATCH 26/57] Push datetime.utcnow() to its own variable

---
 scrapy/extensions/feedexport.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 1f745be98..45c2971a6 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -398,8 +398,9 @@ class FeedExporter:
         params = {}
         for k in dir(spider):
             params[k] = getattr(spider, k)
-        params['time'] = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')
-        params['batch_time'] = datetime.utcnow().isoformat().replace(':', '-')
+        utc_now = datetime.utcnow()
+        params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-')
+        params['batch_time'] = utc_now.isoformat().replace(':', '-')
         params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
         uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
         uripar_function(params, spider)

From c3cee74fd401e6a6307b5eb1786e532bb2cd5aa8 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Fri, 26 Jun 2020 18:45:21 +0300
Subject: [PATCH 27/57] Change default value of FEED_STORAGE_BATCH_ITEM_COUNT
 to 0

---
 docs/topics/feed-exports.rst        |  2 +-
 scrapy/extensions/feedexport.py     |  2 +-
 scrapy/settings/default_settings.py |  2 +-
 tests/test_feedexport.py            | 20 ++++++++++----------
 4 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 866ce78eb..0b37e9a7d 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -435,7 +435,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 
 FEED_STORAGE_BATCH_ITEM_COUNT
 -----------------------------
-Default: ``None``
+Default: ``0``
 
 If assigned an integer number higher than ``0``, Scrapy generates multiple output files
 storing up to the specified number of items in each output file.
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 1331782e3..e06116acd 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -243,7 +243,7 @@ class FeedExporter:
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
-        self.storage_batch_item_count = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None)
+        self.storage_batch_item_count = self.settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 5a7dc533e..810acd5a3 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -146,7 +146,7 @@ FEED_STORAGES_BASE = {
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
 }
-FEED_STORAGE_BATCH_ITEM_COUNT = None
+FEED_STORAGE_BATCH_ITEM_COUNT = 0
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
     'json': 'scrapy.exporters.JsonItemExporter',
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 1a6a5624b..578cd396b 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1144,7 +1144,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['jl']:
@@ -1160,7 +1160,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         data = yield self.exported_data(items, settings)
         for batch in data['csv']:
             got_batch = csv.DictReader(to_unicode(batch).splitlines())
@@ -1176,7 +1176,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['xml']:
@@ -1194,7 +1194,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         # XML
@@ -1219,7 +1219,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import pickle
@@ -1236,7 +1236,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import marshal
@@ -1262,7 +1262,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
             'FEED_STORAGE_BATCH_ITEM_COUNT': 2
         }
         header = self.MyItem.fields.keys()
-        yield self.assertExported(items, header, rows, settings=settings)
+        yield self.assertExported(items, header, rows, settings=Settings(settings))
 
     def test_wrong_path(self):
         """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """
@@ -1412,14 +1412,14 @@ class BatchDeliveriesTest(FeedExportTestBase):
             bucket_name=s3_test_bucket_name, prefix=prefix
         )
         storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key)
-        settings = {
+        settings = Settings({
             'FEEDS': {
                 s3_test_file_uri: {
                     'format': 'json',
                 },
             },
             'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
-        }
+        })
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
             self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
@@ -1436,7 +1436,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
         s3 = boto3.resource('s3')
         my_bucket = s3.Bucket(s3_test_bucket_name)
-        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
 
         with MockServer() as s:
             runner = CrawlerRunner(Settings(settings))

From 88a52198b90faa0129c8e05072197cdffbb9653b Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Sat, 27 Jun 2020 11:50:26 +0300
Subject: [PATCH 28/57] Add batch_item_count support in FEEDS setting

---
 scrapy/extensions/feedexport.py |  5 +++--
 tests/test_feedexport.py        | 39 ++++++++++++++++++++++++++++++---
 2 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index e06116acd..2312c994e 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -25,7 +25,6 @@ from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import without_none_values
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -337,7 +336,9 @@ class FeedExporter:
             slot.exporter.export_item(item)
             slot.itemcount += 1
             # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
-            if self.storage_batch_item_count and slot.itemcount == self.storage_batch_item_count:
+            if self.feeds[slot.uri_template].get('batch_item_count', self.storage_batch_item_count) \
+                    and slot.itemcount == self.feeds[slot.uri_template].get('batch_item_count',
+                                                                            self.storage_batch_item_count):
                 uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
                 self._close_slot(slot, spider)
                 slots.append(self._start_new_batch(
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 578cd396b..3bc0c083c 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1327,8 +1327,9 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     '<items>\n  <item>\n    <foo>FOO1</foo>\n  </item>\n</items>'
                 ).encode('latin-1')
             ],
-            'csv': ['bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
-                    'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')],
+            'csv': ['foo,bar\r\nFOO,BAR\r\n'.encode('utf-8'),
+                    'foo,bar\r\nFOO1,BAR1\r\n'.encode('utf-8')],
+            'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')],
         }
 
         settings = {
@@ -1348,9 +1349,16 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {
                     'format': 'csv',
                     'indent': None,
-                    'fields': ['bar', 'foo'],
+                    'fields': ['foo', 'bar'],
                     'encoding': 'utf-8',
                 },
+                os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {
+                    'format': 'jsonlines',
+                    'indent': None,
+                    'fields': ['foo', 'bar'],
+                    'encoding': 'utf-8',
+                    'batch_item_count': 0,
+                },
             },
             'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
         }
@@ -1359,6 +1367,31 @@ class BatchDeliveriesTest(FeedExportTestBase):
             for expected_batch, got_batch in zip(expected, data[fmt]):
                 self.assertEqual(expected_batch, got_batch)
 
+    @defer.inlineCallbacks
+    def test_batch_item_count_feeds_setting(self):
+        items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]
+
+        formats = {
+            'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n'.encode('utf-8'),
+                          '{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')],
+        }
+
+        settings = {
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): {
+                    'format': 'jsonlines',
+                    'indent': None,
+                    'fields': ['foo', 'bar'],
+                    'encoding': 'utf-8',
+                    'batch_item_count': 1,
+                },
+            },
+        }
+        data = yield self.exported_data(items, settings)
+        for fmt, expected in formats.items():
+            for expected_batch, got_batch in zip(expected, data[fmt]):
+                self.assertEqual(expected_batch, got_batch)
+
     @defer.inlineCallbacks
     def test_batch_path_differ(self):
         """

From 05c2587c6a32b84a94463f2b1187e49f94957aa2 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Sun, 28 Jun 2020 09:45:45 +0300
Subject: [PATCH 29/57] Docs update and tiny fixes

---
 docs/topics/feed-exports.rst | 1 +
 tests/test_feedexport.py     | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 0b37e9a7d..3da56821e 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -272,6 +272,7 @@ as a fallback value if that key is not provided for a specific feed definition.
 * ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
 * ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
 * ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
+* ``batch_item_count``: falls back to :setting:`FEED_STORAGE_BATCH_ITEM_COUNT`
 
 .. setting:: FEED_EXPORT_ENCODING
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 3bc0c083c..542cce70f 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1352,7 +1352,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     'fields': ['foo', 'bar'],
                     'encoding': 'utf-8',
                 },
-                os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {
+                os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): {
                     'format': 'jsonlines',
                     'indent': None,
                     'fields': ['foo', 'bar'],
@@ -1423,8 +1423,8 @@ class BatchDeliveriesTest(FeedExportTestBase):
         [testenv]
         setenv =
             AWS_SECRET_ACCESS_KEY = ABCD
-            AWS_ACCESS_KEY_ID = ABCD
-            S3_TEST_BUCKET_NAME = ABCD
+            AWS_ACCESS_KEY_ID = EFGH
+            S3_TEST_BUCKET_NAME = IJKL
         """
         try:
             import boto3

From 7b1d3c35ea3bfde2ac7fc69a2a26bbcb94aec1bf Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Wed, 1 Jul 2020 11:54:39 +0300
Subject: [PATCH 30/57] Minor updates

---
 docs/topics/feed-exports.rst    |  4 ++--
 scrapy/extensions/feedexport.py | 34 +++++++++++++++++----------------
 scrapy/utils/conf.py            |  4 ++++
 tests/test_feedexport.py        | 23 ++++++----------------
 tests/test_utils_conf.py        |  4 ++++
 5 files changed, 34 insertions(+), 35 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 3da56821e..0b659f30e 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -442,7 +442,7 @@ If assigned an integer number higher than ``0``, Scrapy generates multiple outpu
 storing up to the specified number of items in each output file.
 
 When generating multiple output files, you must use at least one of the following
-placeholders in :setting:`FEED_URI` to indicate how the different output file names are
+placeholders in the feed URI to indicate how the different output file names are
 generated:
 
 * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
@@ -457,7 +457,7 @@ For instance, if your settings include::
 
 And your :command:`crawl` command line is::
 
-  scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
+    scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
 
 The command line above can generate a directory tree like::
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 2312c994e..5908987a3 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -242,7 +242,6 @@ class FeedExporter:
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
-        self.storage_batch_item_count = self.settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
@@ -253,7 +252,7 @@ class FeedExporter:
 
     def open_spider(self, spider):
         for uri, feed in self.feeds.items():
-            uri_params = self._get_uri_params(spider, feed['uri_params'], None)
+            uri_params = self._get_uri_params(spider, feed['uri_params'])
             self.slots.append(self._start_new_batch(
                 batch_id=1,
                 uri=uri % uri_params,
@@ -299,7 +298,7 @@ class FeedExporter:
     def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
         """
         Redirect the output data stream to a new file.
-        Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified.
+        Execute multiple times if FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
         :param batch_id: sequence number of current batch
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
@@ -331,14 +330,15 @@ class FeedExporter:
 
     def item_scraped(self, item, spider):
         slots = []
-        for idx, slot in enumerate(self.slots):
+        for slot in self.slots:
             slot.start_exporting()
             slot.exporter.export_item(item)
             slot.itemcount += 1
             # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
-            if self.feeds[slot.uri_template].get('batch_item_count', self.storage_batch_item_count) \
-                    and slot.itemcount == self.feeds[slot.uri_template].get('batch_item_count',
-                                                                            self.storage_batch_item_count):
+            if (
+                    self.feeds[slot.uri_template]['batch_item_count']
+                    and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count']
+            ):
                 uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
                 self._close_slot(slot, spider)
                 slots.append(self._start_new_batch(
@@ -369,15 +369,17 @@ class FeedExporter:
 
     def _settings_are_valid(self, uri):
         """
-        If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s
-        to distinguish different files of partial output
+        If FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
+        %(batch_time)s or %(batch_id)s to distinguish different files of partial output
         """
-        if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
-            return True
-        logger.error(
-            '%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified'
-        )
-        return False
+        for uri_template, values in self.feeds.items():
+            if values['batch_item_count'] and not any(s in uri_template for s in ['%(batch_time)s', '%(batch_id)s']):
+                logger.error(
+                    '%(batch_time)s or %(batch_id)s must be in uri({}) if FEED_STORAGE_BATCH_ITEM_COUNT setting '
+                    'or FEEDS.batch_item_count is specified and greater than 0.'.format(uri_template)
+                )
+                return False
+        return True
 
     def _storage_supported(self, uri):
         scheme = urlparse(uri).scheme
@@ -404,7 +406,7 @@ class FeedExporter:
     def _get_storage(self, uri):
         return self._get_instance(self.storages[urlparse(uri).scheme], uri)
 
-    def _get_uri_params(self, spider, uri_params, slot):
+    def _get_uri_params(self, spider, uri_params, slot=None):
         params = {}
         for k in dir(spider):
             params[k] = getattr(spider, k)
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 5921f82bf..0e02f0f28 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -115,6 +115,10 @@ def feed_complete_default_values_from_settings(feed, settings):
     out = feed.copy()
     out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
     out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
+    out.setdefault(
+        "batch_item_count",
+        out.get('batch_item_count', settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT'))
+    )
     out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
     out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
     if settings["FEED_EXPORT_INDENT"] is None:
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 542cce70f..db14b20b9 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1265,7 +1265,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
         yield self.assertExported(items, header, rows, settings=Settings(settings))
 
     def test_wrong_path(self):
-        """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """
+        """ If path is without %(batch_time)s and %(batch_id)s an exception must be raised """
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},
@@ -1329,7 +1329,6 @@ class BatchDeliveriesTest(FeedExportTestBase):
             ],
             'csv': ['foo,bar\r\nFOO,BAR\r\n'.encode('utf-8'),
                     'foo,bar\r\nFOO1,BAR1\r\n'.encode('utf-8')],
-            'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')],
         }
 
         settings = {
@@ -1352,13 +1351,6 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     'fields': ['foo', 'bar'],
                     'encoding': 'utf-8',
                 },
-                os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): {
-                    'format': 'jsonlines',
-                    'indent': None,
-                    'fields': ['foo', 'bar'],
-                    'encoding': 'utf-8',
-                    'batch_item_count': 0,
-                },
             },
             'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
         }
@@ -1369,19 +1361,16 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_batch_item_count_feeds_setting(self):
-        items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]
-
+        items = [dict({'foo': u'FOO'}), dict({'foo': u'FOO1'})]
         formats = {
-            'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n'.encode('utf-8'),
-                          '{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')],
+            'json': ['[{"foo": "FOO"}]'.encode('utf-8'),
+                     '[{"foo": "FOO1"}]'.encode('utf-8')],
         }
-
         settings = {
             'FEEDS': {
-                os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): {
-                    'format': 'jsonlines',
+                os.path.join(self._random_temp_filename(), 'json', self._file_mark): {
+                    'format': 'json',
                     'indent': None,
-                    'fields': ['foo', 'bar'],
                     'encoding': 'utf-8',
                     'batch_item_count': 1,
                 },
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index e5d3ef582..95ec2b64a 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -149,6 +149,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "FEED_EXPORT_INDENT": 42,
             "FEED_STORE_EMPTY": True,
             "FEED_URI_PARAMS": (1, 2, 3, 4),
+            "FEED_STORAGE_BATCH_ITEM_COUNT": 2,
         })
         new_feed = feed_complete_default_values_from_settings(feed, settings)
         self.assertEqual(new_feed, {
@@ -157,6 +158,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "indent": 42,
             "store_empty": True,
             "uri_params": (1, 2, 3, 4),
+            "batch_item_count": 2,
         })
 
     def test_feed_complete_default_values_from_settings_non_empty(self):
@@ -169,6 +171,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],
             "FEED_EXPORT_INDENT": 42,
             "FEED_STORE_EMPTY": True,
+            "FEED_STORAGE_BATCH_ITEM_COUNT": 2,
         })
         new_feed = feed_complete_default_values_from_settings(feed, settings)
         self.assertEqual(new_feed, {
@@ -177,6 +180,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "indent": 42,
             "store_empty": True,
             "uri_params": None,
+            "batch_item_count": 2,
         })
 
 

From 1e245046ed8ac3d9f89860501c2da95b69aaabf6 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 2 Jul 2020 12:38:08 +0300
Subject: [PATCH 31/57] Change setting name. Add leading zeroes to batch_id.
 Minor fixes.

---
 docs/topics/feed-exports.rst        | 19 +++++++++--------
 scrapy/extensions/feedexport.py     | 24 +++++++++++++---------
 scrapy/settings/default_settings.py |  2 +-
 scrapy/utils/conf.py                |  5 +----
 tests/test_feedexport.py            | 32 ++++++++++++++---------------
 tests/test_utils_conf.py            |  4 ++--
 6 files changed, 45 insertions(+), 41 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 0b659f30e..56efa80a7 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -220,7 +220,7 @@ These are the settings used for configuring the feed exports:
  * :setting:`FEED_STORAGE_FTP_ACTIVE`
  * :setting:`FEED_STORAGE_S3_ACL`
  * :setting:`FEED_EXPORTERS`
- * :setting:`FEED_STORAGE_BATCH_ITEM_COUNT`
+ * :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
 
 .. currentmodule:: scrapy.extensions.feedexport
 
@@ -272,7 +272,7 @@ as a fallback value if that key is not provided for a specific feed definition.
 * ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
 * ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
 * ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
-* ``batch_item_count``: falls back to :setting:`FEED_STORAGE_BATCH_ITEM_COUNT`
+* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
 
 .. setting:: FEED_EXPORT_ENCODING
 
@@ -432,9 +432,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 .. _botocore: https://github.com/boto/botocore
 .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
-.. setting:: FEED_STORAGE_BATCH_ITEM_COUNT
+.. setting:: FEED_EXPORT_BATCH_ITEM_COUNT
 
-FEED_STORAGE_BATCH_ITEM_COUNT
+FEED_EXPORT_BATCH_ITEM_COUNT
 -----------------------------
 Default: ``0``
 
@@ -448,16 +448,19 @@ generated:
 * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
   (e.g. ``2020-03-28T14-45-08.237134``)
 
-* ``%(batch_id)s`` - gets replaced by the batch  sequence number of batch
-  (e.g. ``2`` for the second file)
+* ``%(batch_id)0xd`` - gets replaced by the sequence number of the batch.
+By replacing ``x`` with an integer you set the number of leading zeroes to prevent
+inappropriate sorting like this: [``'1'``, ``'10'``, ``'2'``]. Here are some examples:
+    ``%(batch_id)01d`` for the second batch gets replaced by ``2``
+    ``%(batch_id)05d`` for the third batch gets replaced by ``00003``
 
 For instance, if your settings include::
 
-    FEED_STORAGE_BATCH_ITEM_COUNT=100
+    FEED_EXPORT_BATCH_ITEM_COUNT=100
 
 And your :command:`crawl` command line is::
 
-    scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
+    scrapy crawl spidername -o dirname/%(batch_id)d-filename%(batch_time)s.json
 
 The command line above can generate a directory tree like::
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 5908987a3..adb6ea2e4 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -6,6 +6,7 @@ See documentation in docs/topics/feed-exports.rst
 
 import logging
 import os
+import re
 import sys
 import warnings
 from datetime import datetime
@@ -25,6 +26,7 @@ from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import without_none_values
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -245,7 +247,7 @@ class FeedExporter:
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
-            if not self._settings_are_valid(uri):
+            if not self._settings_are_valid():
                 raise NotConfigured
             if not self._exporter_supported(feed['format']):
                 raise NotConfigured
@@ -298,7 +300,7 @@ class FeedExporter:
     def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
         """
         Redirect the output data stream to a new file.
-        Execute multiple times if FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
+        Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
         :param batch_id: sequence number of current batch
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
@@ -334,10 +336,10 @@ class FeedExporter:
             slot.start_exporting()
             slot.exporter.export_item(item)
             slot.itemcount += 1
-            # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
+            # create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one
             if (
-                    self.feeds[slot.uri_template]['batch_item_count']
-                    and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count']
+                self.feeds[slot.uri_template]['batch_item_count']
+                and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count']
             ):
                 uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
                 self._close_slot(slot, spider)
@@ -367,16 +369,18 @@ class FeedExporter:
             return True
         logger.error("Unknown feed format: %(format)s", {'format': format})
 
-    def _settings_are_valid(self, uri):
+    def _settings_are_valid(self):
         """
-        If FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
+        If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
         %(batch_time)s or %(batch_id)s to distinguish different files of partial output
         """
         for uri_template, values in self.feeds.items():
-            if values['batch_item_count'] and not any(s in uri_template for s in ['%(batch_time)s', '%(batch_id)s']):
+            if values['batch_item_count'] and not re.findall(r'(%\(batch_time\)s|(%\(batch_id\)0\d*d))', uri_template):
                 logger.error(
-                    '%(batch_time)s or %(batch_id)s must be in uri({}) if FEED_STORAGE_BATCH_ITEM_COUNT setting '
-                    'or FEEDS.batch_item_count is specified and greater than 0.'.format(uri_template)
+                    '%(batch_time)s or %(batch_id)0xd must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting '
+                    'or FEEDS.batch_item_count is specified and greater than 0. For more info see:'
+                    'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'
+                    ''.format(uri_template)
                 )
                 return False
         return True
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 810acd5a3..0016bbe1b 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -146,7 +146,7 @@ FEED_STORAGES_BASE = {
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
 }
-FEED_STORAGE_BATCH_ITEM_COUNT = 0
+FEED_EXPORT_BATCH_ITEM_COUNT = 0
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
     'json': 'scrapy.exporters.JsonItemExporter',
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 0e02f0f28..64f9c824b 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -113,12 +113,9 @@ def get_sources(use_closest=True):
 
 def feed_complete_default_values_from_settings(feed, settings):
     out = feed.copy()
+    out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT'))
     out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
     out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
-    out.setdefault(
-        "batch_item_count",
-        out.get('batch_item_count', settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT'))
-    )
     out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
     out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
     if settings["FEED_EXPORT_INDENT"] is None:
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index db14b20b9..d20b40e2f 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1108,7 +1108,7 @@ class FeedExportTest(FeedExportTestBase):
 
 class BatchDeliveriesTest(FeedExportTestBase):
     __test__ = True
-    _file_mark = '_%(batch_time)s_#%(batch_id)s_'
+    _file_mark = '_%(batch_time)s_#%(batch_id)02d_'
 
     @defer.inlineCallbacks
     def run_and_export(self, spider_cls, settings):
@@ -1144,7 +1144,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
             },
         })
-        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
+        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['jl']:
@@ -1160,7 +1160,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
             },
         })
-        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
+        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         data = yield self.exported_data(items, settings)
         for batch in data['csv']:
             got_batch = csv.DictReader(to_unicode(batch).splitlines())
@@ -1176,7 +1176,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
             },
         })
-        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
+        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['xml']:
@@ -1194,7 +1194,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
             },
         })
-        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
+        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         # XML
@@ -1219,7 +1219,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
             },
         })
-        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
+        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import pickle
@@ -1236,7 +1236,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
             },
         })
-        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
+        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import marshal
@@ -1259,18 +1259,18 @@ class BatchDeliveriesTest(FeedExportTestBase):
             {'foo': 'bar3', 'baz': 'quux3', 'egg': ''}
         ]
         settings = {
-            'FEED_STORAGE_BATCH_ITEM_COUNT': 2
+            'FEED_EXPORT_BATCH_ITEM_COUNT': 2
         }
         header = self.MyItem.fields.keys()
         yield self.assertExported(items, header, rows, settings=Settings(settings))
 
     def test_wrong_path(self):
-        """ If path is without %(batch_time)s and %(batch_id)s an exception must be raised """
+        """ If path is without %(batch_time)s and %(batch_id)0xd an exception must be raised """
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},
             },
-            'FEED_STORAGE_BATCH_ITEM_COUNT': 1
+            'FEED_EXPORT_BATCH_ITEM_COUNT': 1
         }
         crawler = get_crawler(settings_dict=settings)
         self.assertRaises(NotConfigured, FeedExporter, crawler)
@@ -1282,7 +1282,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 'FEEDS': {
                     os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
                 },
-                'FEED_STORAGE_BATCH_ITEM_COUNT': 1
+                'FEED_EXPORT_BATCH_ITEM_COUNT': 1
             }
             data = yield self.exported_no_data(settings)
             data = dict(data)
@@ -1304,7 +1304,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 },
                 'FEED_STORE_EMPTY': True,
                 'FEED_EXPORT_INDENT': None,
-                'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
+                'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
             }
             data = yield self.exported_no_data(settings)
             data = dict(data)
@@ -1352,7 +1352,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     'encoding': 'utf-8',
                 },
             },
-            'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
+            'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
         }
         data = yield self.exported_data(items, settings)
         for fmt, expected in formats.items():
@@ -1398,7 +1398,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     'format': 'json',
                 },
             },
-            'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
+            'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
         }
         data = yield self.exported_data(items, settings)
         self.assertEqual(len(items) + 1, len(data['json']))
@@ -1440,7 +1440,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     'format': 'json',
                 },
             },
-            'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
+            'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
         })
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
@@ -1458,7 +1458,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
         s3 = boto3.resource('s3')
         my_bucket = s3.Bucket(s3_test_bucket_name)
-        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
+        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
 
         with MockServer() as s:
             runner = CrawlerRunner(Settings(settings))
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index 95ec2b64a..f3ef36127 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -149,7 +149,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "FEED_EXPORT_INDENT": 42,
             "FEED_STORE_EMPTY": True,
             "FEED_URI_PARAMS": (1, 2, 3, 4),
-            "FEED_STORAGE_BATCH_ITEM_COUNT": 2,
+            "FEED_EXPORT_BATCH_ITEM_COUNT": 2,
         })
         new_feed = feed_complete_default_values_from_settings(feed, settings)
         self.assertEqual(new_feed, {
@@ -171,7 +171,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],
             "FEED_EXPORT_INDENT": 42,
             "FEED_STORE_EMPTY": True,
-            "FEED_STORAGE_BATCH_ITEM_COUNT": 2,
+            "FEED_EXPORT_BATCH_ITEM_COUNT": 2,
         })
         new_feed = feed_complete_default_values_from_settings(feed, settings)
         self.assertEqual(new_feed, {

From 6454d456d2bcab0828aba6d81d98f7393ab7e04d Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Fri, 3 Jul 2020 08:29:54 +0300
Subject: [PATCH 32/57] Make check of placeholder less strict

---
 docs/topics/feed-exports.rst    | 11 ++++++-----
 scrapy/extensions/feedexport.py |  4 ++--
 tests/test_feedexport.py        |  2 +-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 56efa80a7..0bb5f1733 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -448,11 +448,12 @@ generated:
 * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
   (e.g. ``2020-03-28T14-45-08.237134``)
 
-* ``%(batch_id)0xd`` - gets replaced by the sequence number of the batch.
-By replacing ``x`` with an integer you set the number of leading zeroes to prevent
-inappropriate sorting like this: [``'1'``, ``'10'``, ``'2'``]. Here are some examples:
-    ``%(batch_id)01d`` for the second batch gets replaced by ``2``
-    ``%(batch_id)05d`` for the third batch gets replaced by ``00003``
+* ``%(batch_id)d`` - gets replaced by the sequence number of the batch.
+
+  Use :ref:`printf-style string formatting <python:old-string-formatting>` to
+  alter the number format. For example, to make the batch ID a 5-digit
+  number by introducing leading zeroes as needed, use ``%(batch_id)05d``
+  (e.g. ``3`` becomes ``00003``, ``123`` becomes ``00123``).
 
 For instance, if your settings include::
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index adb6ea2e4..e15c1a09c 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -375,9 +375,9 @@ class FeedExporter:
         %(batch_time)s or %(batch_id)s to distinguish different files of partial output
         """
         for uri_template, values in self.feeds.items():
-            if values['batch_item_count'] and not re.findall(r'(%\(batch_time\)s|(%\(batch_id\)0\d*d))', uri_template):
+            if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template):
                 logger.error(
-                    '%(batch_time)s or %(batch_id)0xd must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting '
+                    '%(batch_time)s or %(batch_id) must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting '
                     'or FEEDS.batch_item_count is specified and greater than 0. For more info see:'
                     'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'
                     ''.format(uri_template)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index d20b40e2f..4e0b867a4 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1265,7 +1265,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
         yield self.assertExported(items, header, rows, settings=Settings(settings))
 
     def test_wrong_path(self):
-        """ If path is without %(batch_time)s and %(batch_id)0xd an exception must be raised """
+        """ If path is without %(batch_time)s and %(batch_id) an exception must be raised """
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},

From f1020e0e6af064ab31b812c25bda6b0f08827222 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Mon, 6 Jul 2020 15:40:53 +0300
Subject: [PATCH 33/57] Tiny changes

---
 scrapy/extensions/feedexport.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index e15c1a09c..21177b1b0 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -305,7 +305,7 @@ class FeedExporter:
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
         :param spider: user spider
-        :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)s to create new uri
+        :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri
         """
         storage = self._get_storage(uri)
         file = storage.open(spider)
@@ -372,13 +372,13 @@ class FeedExporter:
     def _settings_are_valid(self):
         """
         If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
-        %(batch_time)s or %(batch_id)s to distinguish different files of partial output
+        %(batch_time)s or %(batch_id)d to distinguish different files of partial output
         """
         for uri_template, values in self.feeds.items():
             if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template):
                 logger.error(
-                    '%(batch_time)s or %(batch_id) must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting '
-                    'or FEEDS.batch_item_count is specified and greater than 0. For more info see:'
+                    '%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT '
+                    'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: '
                     'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'
                     ''.format(uri_template)
                 )

From 770a8127e8e76d95243c1d586b4bb6113a38870a Mon Sep 17 00:00:00 2001
From: ajaymittur28 <ajay.cs18@bmsce.ac.in>
Date: Tue, 7 Jul 2020 15:23:29 +0530
Subject: [PATCH 34/57] Added basic `scrapy check` tests

---
 tests/test_command_check.py | 96 +++++++++++++++++++++++++++++++++++++
 1 file changed, 96 insertions(+)
 create mode 100644 tests/test_command_check.py

diff --git a/tests/test_command_check.py b/tests/test_command_check.py
new file mode 100644
index 000000000..52005a4c5
--- /dev/null
+++ b/tests/test_command_check.py
@@ -0,0 +1,96 @@
+from os.path import join, abspath
+
+from tests.test_commands import CommandTest
+
+
+class CheckCommandTest(CommandTest):
+
+    command = 'check'
+
+    def setUp(self):
+        super(CheckCommandTest, self).setUp()
+        self.spider_name = 'check_spider'
+        self.spider = abspath(join(self.proj_mod_path, 'spiders', 'checkspider.py'))
+
+    def _write_contract(self, contracts, parse_def):
+        with open(self.spider, 'w') as file:
+            file.write(f"""
+import scrapy
+
+class CheckSpider(scrapy.Spider):
+    name = '{self.spider_name}'
+    start_urls = ['http://example.com']
+
+    def parse(self, response, **cb_kwargs):
+        \"\"\"
+        @url http://www.amazon.com/s?field-keywords=selfish+gene
+        {contracts}
+        \"\"\"
+        {parse_def}
+            """)
+
+    def _test_contract(self, contracts='', parse_def='pass'):
+        self._write_contract(contracts, parse_def)
+        p, out, err = self.proc('check')
+        self.assertIn('OK', err)
+        self.assertEqual(p.returncode, 0)
+
+    def test_check_returns_requests_contract(self):
+        contracts = """
+        @returns requests 1
+        """
+        parse_def = """
+        yield scrapy.Request(url='http://next-url.com')
+        """
+        self._test_contract(contracts, parse_def)
+
+    def test_check_returns_items_contract(self):
+        contracts = """
+        @returns items 1
+        """
+        parse_def = """
+        yield {'key1': 'val1', 'key2': 'val2'}
+        """
+        self._test_contract(contracts, parse_def)
+
+    def test_check_cb_kwargs_contract(self):
+        contracts = """
+        @cb_kwargs {"arg1": "val1", "arg2": "val2"}
+        """
+        parse_def = """
+        if len(cb_kwargs.items()) == 0:
+            raise Exception("Callback args not set")
+        """
+        self._test_contract(contracts, parse_def)
+
+    def test_check_scrapes_contract(self):
+        contracts = """
+        @scrapes key1 key2
+        """
+        parse_def = """
+        yield {'key1': 'val1', 'key2': 'val2'}
+        """
+        self._test_contract(contracts, parse_def)
+
+    def test_check_all_default_contracts(self):
+        contracts = """
+        @returns items 1
+        @returns requests 1
+        @scrapes key1 key2
+        @cb_kwargs {"arg1": "val1", "arg2": "val2"}
+        """
+        parse_def = """
+        yield {'key1': 'val1', 'key2': 'val2'}
+        yield scrapy.Request(url='http://next-url.com')
+        if len(cb_kwargs.items()) == 0:
+            raise Exception("Callback args not set")
+        """
+        self._test_contract(contracts, parse_def)
+
+    def test_SCRAPY_CHECK_set(self):
+        parse_def = """
+        import os
+        if not os.environ.get('SCRAPY_CHECK'):
+            raise Exception('SCRAPY_CHECK not set')
+        """
+        self._test_contract(parse_def=parse_def)

From d014840672820b3970282f31a51b9ff24cd46bd3 Mon Sep 17 00:00:00 2001
From: ajaymittur28 <ajay.cs18@bmsce.ac.in>
Date: Tue, 7 Jul 2020 15:24:33 +0530
Subject: [PATCH 35/57] Ignore flake8 E501 for `scrapy check` tests`

---
 pytest.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pytest.ini b/pytest.ini
index bae68cd3a..97320a008 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -173,6 +173,7 @@ flake8-ignore =
     tests/pipelines.py F841 E226
     tests/spiders.py E501 E127
     tests/test_closespider.py E501 E127
+    tests/test_command_check.py E501
     tests/test_command_fetch.py E501
     tests/test_command_parse.py E501 E128 E303 E226
     tests/test_command_shell.py E501 E128

From 3e98ed24b6e9189d7fc7b4209d24971068274ddb Mon Sep 17 00:00:00 2001
From: ajaymittur28 <ajay.cs18@bmsce.ac.in>
Date: Wed, 8 Jul 2020 17:13:57 +0530
Subject: [PATCH 36/57] Convert f-string to .format()

---
 tests/test_command_check.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index 52005a4c5..72acd817c 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -14,20 +14,20 @@ class CheckCommandTest(CommandTest):
 
     def _write_contract(self, contracts, parse_def):
         with open(self.spider, 'w') as file:
-            file.write(f"""
+            file.write("""
 import scrapy
 
 class CheckSpider(scrapy.Spider):
-    name = '{self.spider_name}'
+    name = '{0}'
     start_urls = ['http://example.com']
 
     def parse(self, response, **cb_kwargs):
         \"\"\"
         @url http://www.amazon.com/s?field-keywords=selfish+gene
-        {contracts}
+        {1}
         \"\"\"
-        {parse_def}
-            """)
+        {2}
+            """.format(self.spider_name, contracts, parse_def))
 
     def _test_contract(self, contracts='', parse_def='pass'):
         self._write_contract(contracts, parse_def)

From 75bff7b6d33bdc74c1a8eb0e43e4b484473c3062 Mon Sep 17 00:00:00 2001
From: ajaymittur28 <ajay.cs18@bmsce.ac.in>
Date: Wed, 8 Jul 2020 19:48:42 +0530
Subject: [PATCH 37/57] Update url contract value

---
 tests/test_command_check.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index 72acd817c..f27f526a3 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -23,7 +23,7 @@ class CheckSpider(scrapy.Spider):
 
     def parse(self, response, **cb_kwargs):
         \"\"\"
-        @url http://www.amazon.com/s?field-keywords=selfish+gene
+        @url http://example.com
         {1}
         \"\"\"
         {2}
@@ -32,6 +32,7 @@ class CheckSpider(scrapy.Spider):
     def _test_contract(self, contracts='', parse_def='pass'):
         self._write_contract(contracts, parse_def)
         p, out, err = self.proc('check')
+        self.assertNotIn('F', out)
         self.assertIn('OK', err)
         self.assertEqual(p.returncode, 0)
 

From cbe4dc57f3f65ecb851941dcfae0bc18c6c8582a Mon Sep 17 00:00:00 2001
From: Ajay Mittur <ajay.cs18@bmsce.ac.in>
Date: Fri, 10 Jul 2020 18:22:43 +0530
Subject: [PATCH 38/57] Update pytest.ini

---
 pytest.ini | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pytest.ini b/pytest.ini
index 92c5bcb75..ca8191f42 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -39,4 +39,5 @@ flake8-ignore =
     scrapy/utils/markup.py F403
     scrapy/utils/multipart.py F403
     scrapy/utils/url.py F403 F405
-    tests/test_loader.py E741
\ No newline at end of file
+    tests/test_loader.py E741
+    

From 8bdcdb0a76e3780681dcfbbd4a0eee62e2bb05b1 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 16 Jul 2020 09:13:54 +0300
Subject: [PATCH 39/57] Add quotes to example in docs

---
 docs/topics/feed-exports.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 0bb5f1733..7e91b365d 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -461,7 +461,7 @@ For instance, if your settings include::
 
 And your :command:`crawl` command line is::
 
-    scrapy crawl spidername -o dirname/%(batch_id)d-filename%(batch_time)s.json
+    scrapy crawl spidername -o 'dirname/%(batch_id)d-filename%(batch_time)s.json'
 
 The command line above can generate a directory tree like::
 

From 41263f61c6de8048023ba4c80e062f56b21e5a19 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 16 Jul 2020 18:41:45 +0300
Subject: [PATCH 40/57] Change single quotes to double in example in docs

---
 docs/topics/feed-exports.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 7e91b365d..fdc6e7cba 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -461,7 +461,7 @@ For instance, if your settings include::
 
 And your :command:`crawl` command line is::
 
-    scrapy crawl spidername -o 'dirname/%(batch_id)d-filename%(batch_time)s.json'
+    scrapy crawl spidername -o "dirname/%(batch_id)d-filename%(batch_time)s.json"
 
 The command line above can generate a directory tree like::
 

From 86f7ac2f2b5d58e0b2588fa2aa4c777a8decf299 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Fri, 17 Jul 2020 17:48:25 +0300
Subject: [PATCH 41/57] Try to fix error at Windows

---
 tests/test_feedexport.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 129b7fc0b..cc124624d 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1204,6 +1204,8 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
             for path, feed in FEEDS.items():
                 dir_name = os.path.dirname(path)
+                if not os.path.exists(str(dir_name)):
+                    continue
                 for file in sorted(os.listdir(dir_name)):
                     with open(os.path.join(dir_name, file), 'rb') as f:
                         data = f.read()

From 3e0492741d93b05c464457b3b128a2b0d24c994b Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Sun, 19 Jul 2020 00:10:29 +0300
Subject: [PATCH 42/57] Another try to fix test errors on Windows

---
 tests/test_feedexport.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index cc124624d..c49b2e92f 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1190,9 +1190,14 @@ class BatchDeliveriesTest(FeedExportTestBase):
     def run_and_export(self, spider_cls, settings):
         """ Run spider with specified settings; return exported data. """
 
+        def build_url(path):
+            if path[0] != '/':
+                path = '/' + path
+            return urljoin('file:', path)
+
         FEEDS = settings.get('FEEDS') or {}
         settings['FEEDS'] = {
-            urljoin('file:', file_path): feed
+            build_url(file_path): feed
             for file_path, feed in FEEDS.items()
         }
         content = defaultdict(list)
@@ -1204,8 +1209,6 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
             for path, feed in FEEDS.items():
                 dir_name = os.path.dirname(path)
-                if not os.path.exists(str(dir_name)):
-                    continue
                 for file in sorted(os.listdir(dir_name)):
                     with open(os.path.join(dir_name, file), 'rb') as f:
                         data = f.read()

From a6c1d79b7cc3bc2c408eab356bbbf99a0536f110 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Tue, 28 Jul 2020 11:53:05 +0300
Subject: [PATCH 43/57] pep8 tiny changes

---
 docs/topics/feed-exports.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 2c9774b55..dd4eb3c61 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -453,6 +453,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 
 FEED_EXPORT_BATCH_ITEM_COUNT
 -----------------------------
+
 Default: ``0``
 
 If assigned an integer number higher than ``0``, Scrapy generates multiple output files
@@ -474,7 +475,7 @@ generated:
 
 For instance, if your settings include::
 
-    FEED_EXPORT_BATCH_ITEM_COUNT=100
+    FEED_EXPORT_BATCH_ITEM_COUNT = 100
 
 And your :command:`crawl` command line is::
 

From 52658539370c442e63102a3208781335953cdf53 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Tue, 28 Jul 2020 06:15:14 -0300
Subject: [PATCH 44/57] Use ItemAdapter.field_names when writing header in
 CsvItemExporter (#4668)

---
 scrapy/exporters.py     |   8 +-
 tests/test_exporters.py | 203 ++++++++++++++++++++++++++++------------
 2 files changed, 146 insertions(+), 65 deletions(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 712572673..0aba1c904 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -243,12 +243,8 @@ class CsvItemExporter(BaseItemExporter):
     def _write_headers_and_set_fields_to_export(self, item):
         if self.include_headers_line:
             if not self.fields_to_export:
-                if isinstance(item, dict):
-                    # for dicts try using fields of the first item
-                    self.fields_to_export = list(item.keys())
-                else:
-                    # use fields declared in Item
-                    self.fields_to_export = list(item.fields.keys())
+                # use declared field names, or keys if the item is a dict
+                self.fields_to_export = ItemAdapter(item).field_names()
             row = list(self._build_row(self.fields_to_export))
             self.csv_writer.writerow(row)
 
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index b27380309..25da54a65 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -8,6 +8,7 @@ from io import BytesIO
 from datetime import datetime
 
 import lxml.etree
+from itemadapter import ItemAdapter
 
 from scrapy.item import Item, Field
 from scrapy.utils.python import to_unicode
@@ -23,10 +24,37 @@ class TestItem(Item):
     age = Field()
 
 
+def custom_serializer(value):
+    return str(int(value) + 2)
+
+
+class CustomFieldItem(Item):
+    name = Field()
+    age = Field(serializer=custom_serializer)
+
+
+try:
+    from dataclasses import make_dataclass, field
+except ImportError:
+    TestDataClass = None
+    CustomFieldDataclass = None
+else:
+    TestDataClass = make_dataclass("TestDataClass", [("name", str), ("age", int)])
+    CustomFieldDataclass = make_dataclass(
+        "CustomFieldDataclass",
+        [("name", str), ("age", int, field(metadata={"serializer": custom_serializer}))]
+    )
+
+
 class BaseItemExporterTest(unittest.TestCase):
 
+    item_class = TestItem
+    custom_field_item_class = CustomFieldItem
+
     def setUp(self):
-        self.i = TestItem(name=u'John\xa3', age=u'22')
+        if self.item_class is None:
+            raise unittest.SkipTest("item class is None")
+        self.i = self.item_class(name=u'John\xa3', age=u'22')
         self.output = BytesIO()
         self.ie = self._get_exporter()
 
@@ -39,7 +67,7 @@ class BaseItemExporterTest(unittest.TestCase):
     def _assert_expected_item(self, exported_dict):
         for k, v in exported_dict.items():
             exported_dict[k] = to_unicode(v)
-        self.assertEqual(self.i, exported_dict)
+        self.assertEqual(self.i, self.item_class(**exported_dict))
 
     def _get_nonstring_types_item(self):
         return {
@@ -63,13 +91,14 @@ class BaseItemExporterTest(unittest.TestCase):
         self.assertItemExportWorks(self.i)
 
     def test_export_dict_item(self):
-        self.assertItemExportWorks(dict(self.i))
+        self.assertItemExportWorks(ItemAdapter(self.i).asdict())
 
     def test_serialize_field(self):
-        res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name'])
+        a = ItemAdapter(self.i)
+        res = self.ie.serialize_field(a.get_field_meta('name'), 'name', a['name'])
         self.assertEqual(res, u'John\xa3')
 
-        res = self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age'])
+        res = self.ie.serialize_field(a.get_field_meta('age'), 'age', a['age'])
         self.assertEqual(res, u'22')
 
     def test_fields_to_export(self):
@@ -82,18 +111,16 @@ class BaseItemExporterTest(unittest.TestCase):
         self.assertEqual(name, u'John\xa3')
 
     def test_field_custom_serializer(self):
-        def custom_serializer(value):
-            return str(int(value) + 2)
-
-        class CustomFieldItem(Item):
-            name = Field()
-            age = Field(serializer=custom_serializer)
-
-        i = CustomFieldItem(name=u'John\xa3', age=u'22')
-
+        i = self.custom_field_item_class(name=u'John\xa3', age=u'22')
+        a = ItemAdapter(i)
         ie = self._get_exporter()
-        self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), u'John\xa3')
-        self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
+        self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), u'John\xa3')
+        self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24')
+
+
+class BaseItemExporterDataclassTest(BaseItemExporterTest):
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
 
 
 class PythonItemExporterTest(BaseItemExporterTest):
@@ -105,9 +132,9 @@ class PythonItemExporterTest(BaseItemExporterTest):
             PythonItemExporter(invalid_option='something')
 
     def test_nested_item(self):
-        i1 = TestItem(name=u'Joseph', age='22')
+        i1 = self.item_class(name=u'Joseph', age='22')
         i2 = dict(name=u'Maria', age=i1)
-        i3 = TestItem(name=u'Jesus', age=i2)
+        i3 = self.item_class(name=u'Jesus', age=i2)
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(type(exported), dict)
@@ -119,9 +146,9 @@ class PythonItemExporterTest(BaseItemExporterTest):
         self.assertEqual(type(exported['age']['age']), dict)
 
     def test_export_list(self):
-        i1 = TestItem(name=u'Joseph', age='22')
-        i2 = TestItem(name=u'Maria', age=[i1])
-        i3 = TestItem(name=u'Jesus', age=[i2])
+        i1 = self.item_class(name=u'Joseph', age='22')
+        i2 = self.item_class(name=u'Maria', age=[i1])
+        i3 = self.item_class(name=u'Jesus', age=[i2])
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(
@@ -132,9 +159,9 @@ class PythonItemExporterTest(BaseItemExporterTest):
         self.assertEqual(type(exported['age'][0]['age'][0]), dict)
 
     def test_export_item_dict_list(self):
-        i1 = TestItem(name=u'Joseph', age='22')
+        i1 = self.item_class(name=u'Joseph', age='22')
         i2 = dict(name=u'Maria', age=[i1])
-        i3 = TestItem(name=u'Jesus', age=[i2])
+        i3 = self.item_class(name=u'Jesus', age=[i2])
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(
@@ -146,7 +173,7 @@ class PythonItemExporterTest(BaseItemExporterTest):
 
     def test_export_binary(self):
         exporter = PythonItemExporter(binary=True)
-        value = TestItem(name=u'John\xa3', age=u'22')
+        value = self.item_class(name=u'John\xa3', age=u'22')
         expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
         self.assertEqual(expected, exporter.export_item(value))
 
@@ -157,6 +184,11 @@ class PythonItemExporterTest(BaseItemExporterTest):
         self.assertEqual(exported, item)
 
 
+class PythonItemExporterDataclassTest(PythonItemExporterTest):
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
 class PprintItemExporterTest(BaseItemExporterTest):
 
     def _get_exporter(self, **kwargs):
@@ -166,6 +198,11 @@ class PprintItemExporterTest(BaseItemExporterTest):
         self._assert_expected_item(eval(self.output.getvalue()))
 
 
+class PprintItemExporterDataclassTest(PprintItemExporterTest):
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
 class PickleItemExporterTest(BaseItemExporterTest):
 
     def _get_exporter(self, **kwargs):
@@ -175,8 +212,8 @@ class PickleItemExporterTest(BaseItemExporterTest):
         self._assert_expected_item(pickle.loads(self.output.getvalue()))
 
     def test_export_multiple_items(self):
-        i1 = TestItem(name='hello', age='world')
-        i2 = TestItem(name='bye', age='world')
+        i1 = self.item_class(name='hello', age='world')
+        i2 = self.item_class(name='bye', age='world')
         f = BytesIO()
         ie = PickleItemExporter(f)
         ie.start_exporting()
@@ -184,8 +221,8 @@ class PickleItemExporterTest(BaseItemExporterTest):
         ie.export_item(i2)
         ie.finish_exporting()
         f.seek(0)
-        self.assertEqual(pickle.load(f), i1)
-        self.assertEqual(pickle.load(f), i2)
+        self.assertEqual(self.item_class(**pickle.load(f)), i1)
+        self.assertEqual(self.item_class(**pickle.load(f)), i2)
 
     def test_nonstring_types_item(self):
         item = self._get_nonstring_types_item()
@@ -197,6 +234,11 @@ class PickleItemExporterTest(BaseItemExporterTest):
         self.assertEqual(pickle.loads(fp.getvalue()), item)
 
 
+class PickleItemExporterDataclassTest(PickleItemExporterTest):
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
 class MarshalItemExporterTest(BaseItemExporterTest):
 
     def _get_exporter(self, **kwargs):
@@ -219,6 +261,11 @@ class MarshalItemExporterTest(BaseItemExporterTest):
         self.assertEqual(marshal.load(fp), item)
 
 
+class MarshalItemExporterDataclassTest(MarshalItemExporterTest):
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
 class CsvItemExporterTest(BaseItemExporterTest):
     def _get_exporter(self, **kwargs):
         return CsvItemExporter(self.output, **kwargs)
@@ -245,18 +292,18 @@ class CsvItemExporterTest(BaseItemExporterTest):
     def test_header_export_all(self):
         self.assertExportResult(
             item=self.i,
-            fields_to_export=self.i.fields.keys(),
+            fields_to_export=ItemAdapter(self.i).field_names(),
             expected=b'age,name\r\n22,John\xc2\xa3\r\n',
         )
 
     def test_header_export_all_dict(self):
         self.assertExportResult(
-            item=dict(self.i),
+            item=ItemAdapter(self.i).asdict(),
             expected=b'age,name\r\n22,John\xc2\xa3\r\n',
         )
 
     def test_header_export_single_field(self):
-        for item in [self.i, dict(self.i)]:
+        for item in [self.i, ItemAdapter(self.i).asdict()]:
             self.assertExportResult(
                 item=item,
                 fields_to_export=['age'],
@@ -264,7 +311,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
             )
 
     def test_header_export_two_items(self):
-        for item in [self.i, dict(self.i)]:
+        for item in [self.i, ItemAdapter(self.i).asdict()]:
             output = BytesIO()
             ie = CsvItemExporter(output)
             ie.start_exporting()
@@ -275,7 +322,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
                                 b'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n')
 
     def test_header_no_header_line(self):
-        for item in [self.i, dict(self.i)]:
+        for item in [self.i, ItemAdapter(self.i).asdict()]:
             self.assertExportResult(
                 item=item,
                 include_headers_line=False,
@@ -309,6 +356,11 @@ class CsvItemExporterTest(BaseItemExporterTest):
         )
 
 
+class CsvItemExporterDataclassTest(CsvItemExporterTest):
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
 class XmlItemExporterTest(BaseItemExporterTest):
 
     def _get_exporter(self, **kwargs):
@@ -318,8 +370,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
         def xmltuple(elem):
             children = list(elem.iterchildren())
             if children:
-                return [(child.tag, sorted(xmltuple(child)))
-                        for child in children]
+                return [(child.tag, sorted(xmltuple(child))) for child in children]
             else:
                 return [(elem.tag, [(elem.text, ())])]
 
@@ -345,17 +396,21 @@ class XmlItemExporterTest(BaseItemExporterTest):
 
     def test_multivalued_fields(self):
         self.assertExportResult(
-            TestItem(name=[u'John\xa3', u'Doe']),
-            (
-                b'<?xml version="1.0" encoding="utf-8"?>\n'
-                b'<items><item><name><value>John\xc2\xa3</value><value>Doe</value></name></item></items>'
-            )
+            self.item_class(name=[u'John\xa3', u'Doe'], age=[1, 2, 3]),
+            b"""<?xml version="1.0" encoding="utf-8"?>\n
+            <items>
+                <item>
+                    <name><value>John\xc2\xa3</value><value>Doe</value></name>
+                    <age><value>1</value><value>2</value><value>3</value></age>
+                </item>
+            </items>
+            """
         )
 
     def test_nested_item(self):
-        i1 = TestItem(name=u'foo\xa3hoo', age='22')
+        i1 = dict(name=u'foo\xa3hoo', age='22')
         i2 = dict(name=u'bar', age=i1)
-        i3 = TestItem(name=u'buz', age=i2)
+        i3 = self.item_class(name=u'buz', age=i2)
 
         self.assertExportResult(
             i3,
@@ -376,9 +431,9 @@ class XmlItemExporterTest(BaseItemExporterTest):
         )
 
     def test_nested_list_item(self):
-        i1 = TestItem(name=u'foo')
+        i1 = dict(name=u'foo')
         i2 = dict(name=u'bar', v2={"egg": ["spam"]})
-        i3 = TestItem(name=u'buz', age=[i1, i2])
+        i3 = self.item_class(name=u'buz', age=[i1, i2])
 
         self.assertExportResult(
             i3,
@@ -412,6 +467,12 @@ class XmlItemExporterTest(BaseItemExporterTest):
         )
 
 
+class XmlItemExporterDataclassTest(XmlItemExporterTest):
+
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
 class JsonLinesItemExporterTest(BaseItemExporterTest):
 
     _expected_nested = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}}
@@ -421,12 +482,12 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
 
     def _check_output(self):
         exported = json.loads(to_unicode(self.output.getvalue().strip()))
-        self.assertEqual(exported, dict(self.i))
+        self.assertEqual(exported, ItemAdapter(self.i).asdict())
 
     def test_nested_item(self):
-        i1 = TestItem(name=u'Joseph', age='22')
+        i1 = self.item_class(name=u'Joseph', age='22')
         i2 = dict(name=u'Maria', age=i1)
-        i3 = TestItem(name=u'Jesus', age=i2)
+        i3 = self.item_class(name=u'Jesus', age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
@@ -449,6 +510,12 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
         self.assertEqual(exported, item)
 
 
+class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest):
+
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
 class JsonItemExporterTest(JsonLinesItemExporterTest):
 
     _expected_nested = [JsonLinesItemExporterTest._expected_nested]
@@ -458,7 +525,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
 
     def _check_output(self):
         exported = json.loads(to_unicode(self.output.getvalue().strip()))
-        self.assertEqual(exported, [dict(self.i)])
+        self.assertEqual(exported, [ItemAdapter(self.i).asdict()])
 
     def assertTwoItemsExported(self, item):
         self.ie.start_exporting()
@@ -466,28 +533,28 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.ie.export_item(item)
         self.ie.finish_exporting()
         exported = json.loads(to_unicode(self.output.getvalue()))
-        self.assertEqual(exported, [dict(item), dict(item)])
+        self.assertEqual(exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()])
 
     def test_two_items(self):
         self.assertTwoItemsExported(self.i)
 
     def test_two_dict_items(self):
-        self.assertTwoItemsExported(dict(self.i))
+        self.assertTwoItemsExported(ItemAdapter(self.i).asdict())
 
     def test_nested_item(self):
-        i1 = TestItem(name=u'Joseph\xa3', age='22')
-        i2 = TestItem(name=u'Maria', age=i1)
-        i3 = TestItem(name=u'Jesus', age=i2)
+        i1 = self.item_class(name=u'Joseph\xa3', age='22')
+        i2 = self.item_class(name=u'Maria', age=i1)
+        i3 = self.item_class(name=u'Jesus', age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
         exported = json.loads(to_unicode(self.output.getvalue()))
-        expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': dict(i1)}}
+        expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}}
         self.assertEqual(exported, [expected])
 
     def test_nested_dict_item(self):
         i1 = dict(name=u'Joseph\xa3', age='22')
-        i2 = TestItem(name=u'Maria', age=i1)
+        i2 = self.item_class(name=u'Maria', age=i1)
         i3 = dict(name=u'Jesus', age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
@@ -506,7 +573,19 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.assertEqual(exported, [item])
 
 
-class CustomItemExporterTest(unittest.TestCase):
+class JsonItemExporterDataclassTest(JsonItemExporterTest):
+
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
+class CustomExporterItemTest(unittest.TestCase):
+
+    item_class = TestItem
+
+    def setUp(self):
+        if self.item_class is None:
+            raise unittest.SkipTest("item class is None")
 
     def test_exporter_custom_serializer(self):
         class CustomItemExporter(BaseItemExporter):
@@ -516,16 +595,22 @@ class CustomItemExporterTest(unittest.TestCase):
                 else:
                     return super(CustomItemExporter, self).serialize_field(field, name, value)
 
-        i = TestItem(name=u'John', age='22')
+        i = self.item_class(name=u'John', age='22')
+        a = ItemAdapter(i)
         ie = CustomItemExporter()
 
-        self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John')
-        self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '23')
+        self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John')
+        self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '23')
 
         i2 = {'name': u'John', 'age': '22'}
         self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John')
         self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23')
 
 
+class CustomExporterDataclassTest(CustomExporterItemTest):
+
+    item_class = TestDataClass
+
+
 if __name__ == '__main__':
     unittest.main()

From e7a58fe1573176415a9ca054428c53c1ca29931a Mon Sep 17 00:00:00 2001
From: Kshitij Sharma <ksharma@atlassian.com>
Date: Wed, 29 Jul 2020 10:16:18 +0530
Subject: [PATCH 45/57] Code cleanup scrapy.utils.python.WeakKeyCache #4684

---
 scrapy/utils/python.py     | 12 ------------
 tests/test_utils_python.py | 18 +-----------------
 2 files changed, 1 insertion(+), 29 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 9204977cf..7a393925e 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -273,18 +273,6 @@ def equal_attributes(obj1, obj2, attributes):
     return True
 
 
-class WeakKeyCache:
-
-    def __init__(self, default_factory):
-        self.default_factory = default_factory
-        self._weakdict = weakref.WeakKeyDictionary()
-
-    def __getitem__(self, key):
-        if key not in self._weakdict:
-            self._weakdict[key] = self.default_factory(key)
-        return self._weakdict[key]
-
-
 @deprecated
 def retry_on_eintr(function, *args, **kw):
     """Run a function and retry it while getting EINTR errors"""
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index ebce3c079..b23ae2e52 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -9,7 +9,7 @@ from warnings import catch_warnings
 
 from scrapy.utils.python import (
     memoizemethod_noargs, binary_is_text, equal_attributes,
-    WeakKeyCache, get_func_args, to_bytes, to_unicode,
+    get_func_args, to_bytes, to_unicode,
     without_none_values, MutableChain)
 
 
@@ -155,22 +155,6 @@ class UtilsPythonTestCase(unittest.TestCase):
         a.meta['z'] = 2
         self.assertFalse(equal_attributes(a, b, [compare_z, 'x']))
 
-    def test_weakkeycache(self):
-        class _Weakme:
-            pass
-
-        _values = count()
-        wk = WeakKeyCache(lambda k: next(_values))
-        k = _Weakme()
-        v = wk[k]
-        self.assertEqual(v, wk[k])
-        self.assertNotEqual(v, wk[_Weakme()])
-        self.assertEqual(v, wk[k])
-        del k
-        for _ in range(100):
-            if wk._weakdict:
-                gc.collect()
-        self.assertFalse(len(wk._weakdict))
 
     def test_get_func_args(self):
         def f1(a, b, c):

From 403bc7020a5e1ba2b59eced2cc5f4453c7650666 Mon Sep 17 00:00:00 2001
From: Kshitij Sharma <ksharma@atlassian.com>
Date: Wed, 29 Jul 2020 18:05:33 +0530
Subject: [PATCH 46/57] Code cleanup scrapy.utils.python.WeakKeyCache #4684 and
 fixing ci alerts

---
 tests/test_utils_python.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index b23ae2e52..5a53d89e4 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -1,9 +1,7 @@
 import functools
-import gc
 import operator
 import platform
 import unittest
-from itertools import count
 from sys import version_info
 from warnings import catch_warnings
 
@@ -12,7 +10,6 @@ from scrapy.utils.python import (
     get_func_args, to_bytes, to_unicode,
     without_none_values, MutableChain)
 
-
 __doctests__ = ['scrapy.utils.python']
 
 
@@ -155,7 +152,6 @@ class UtilsPythonTestCase(unittest.TestCase):
         a.meta['z'] = 2
         self.assertFalse(equal_attributes(a, b, [compare_z, 'x']))
 
-
     def test_get_func_args(self):
         def f1(a, b, c):
             pass

From 49337bd2ae094d97d364948569f59b8211c8dbbe Mon Sep 17 00:00:00 2001
From: Kshitij Sharma <ksharma@atlassian.com>
Date: Thu, 30 Jul 2020 12:25:21 +0530
Subject: [PATCH 48/57] Code cleanup scrapy.utils.python.WeakKeyCache #4684 and
 fixing ci alerts

---
 scrapy/utils/python.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 7a393925e..c8f921ff3 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -127,6 +127,7 @@ def re_rsearch(pattern, text, chunk_size=1024):
     In case the pattern wasn't found, None is returned, otherwise it returns a tuple containing
     the start position of the match, and the ending (regarding the entire text).
     """
+
     def _chunk_iter():
         offset = len(text)
         while True:
@@ -158,6 +159,7 @@ def memoizemethod_noargs(method):
         if self not in cache:
             cache[self] = method(self, *args, **kwargs)
         return cache[self]
+
     return new_method
 
 
@@ -273,6 +275,19 @@ def equal_attributes(obj1, obj2, attributes):
     return True
 
 
+@deprecated
+class WeakKeyCache:
+
+    def __init__(self, default_factory):
+        self.default_factory = default_factory
+        self._weakdict = weakref.WeakKeyDictionary()
+
+    def __getitem__(self, key):
+        if key not in self._weakdict:
+            self._weakdict[key] = self.default_factory(key)
+        return self._weakdict[key]
+
+
 @deprecated
 def retry_on_eintr(function, *args, **kw):
     """Run a function and retry it while getting EINTR errors"""

From 890b2138a605af2bfbf340a0d48d9d83c4cda53b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 30 Jul 2020 13:39:30 +0200
Subject: [PATCH 49/57] Remove the u prefix from strings

---
 docs/_ext/scrapydocs.py                      |   2 +-
 docs/topics/loaders.rst                      |   6 +-
 docs/topics/selectors.rst                    |   4 +-
 docs/utils/linkfix.py                        |   2 +-
 scrapy/http/request/form.py                  |   6 +-
 scrapy/linkextractors/lxmlhtml.py            |   2 +-
 scrapy/logformatter.py                       |   2 +-
 tests/test_cmdline/__init__.py               |   2 +-
 tests/test_downloader_handlers.py            |   6 +-
 tests/test_downloadermiddleware_cookies.py   |  16 +-
 tests/test_downloadermiddleware_httpproxy.py |   6 +-
 tests/test_downloadermiddleware_redirect.py  |   6 +-
 tests/test_downloadermiddleware_robotstxt.py |   4 +-
 tests/test_exporters.py                      |  84 +++----
 tests/test_feedexport.py                     |   8 +-
 tests/test_http_headers.py                   |   6 +-
 tests/test_http_request.py                   |  90 ++++----
 tests/test_http_response.py                  |  72 +++---
 tests/test_item.py                           |  42 ++--
 tests/test_linkextractors.py                 | 180 +++++++--------
 tests/test_loader.py                         | 124 +++++-----
 tests/test_loader_deprecated.py              | 226 +++++++++----------
 tests/test_logformatter.py                   |  12 +-
 tests/test_mail.py                           |   8 +-
 tests/test_responsetypes.py                  |  10 +-
 tests/test_robotstxt_interface.py            |   8 +-
 tests/test_selector.py                       |  32 +--
 tests/test_spider.py                         |  12 +-
 tests/test_utils_iterators.py                |  90 ++++----
 tests/test_utils_python.py                   |  16 +-
 tests/test_utils_reqser.py                   |   2 +-
 tests/test_utils_template.py                 |   4 +-
 32 files changed, 545 insertions(+), 545 deletions(-)

diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py
index 192123473..640660943 100644
--- a/docs/_ext/scrapydocs.py
+++ b/docs/_ext/scrapydocs.py
@@ -17,7 +17,7 @@ class SettingsListDirective(Directive):
 def is_setting_index(node):
     if node.tagname == 'index':
         # index entries for setting directives look like:
-        # [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')]
+        # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
         entry_type, info, refid = node['entries'][0][:3]
         return entry_type == 'pair' and info.endswith('; setting')
     return False
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index d0eeb4097..29d9c5805 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -237,10 +237,10 @@ metadata. Here is an example::
 
 >>> from scrapy.loader import ItemLoader
 >>> il = ItemLoader(item=Product())
->>> il.add_value('name', [u'Welcome to my', u'<strong>website</strong>'])
->>> il.add_value('price', [u'&euro;', u'<span>1000</span>'])
+>>> il.add_value('name', ['Welcome to my', '<strong>website</strong>'])
+>>> il.add_value('price', ['&euro;', '<span>1000</span>'])
 >>> il.load_item()
-{'name': u'Welcome to my website', 'price': u'1000'}
+{'name': 'Welcome to my website', 'price': '1000'}
 
 The precedence order, for both input and output processors, is as follows:
 
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index bb46ea80f..5014df6ac 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -734,7 +734,7 @@ The ``test()`` function, for example, can prove quite useful when XPath's
 Example selecting links in list item with a "class" attribute ending with a digit:
 
 >>> from scrapy import Selector
->>> doc = u"""
+>>> doc = """
 ... <div>
 ...     <ul>
 ...         <li class="item-0"><a href="link1.html">first item</a></li>
@@ -765,7 +765,7 @@ extracting text elements for example.
 Example extracting microdata (sample content taken from https://schema.org/Product)
 with groups of itemscopes and corresponding itemprops::
 
-    >>> doc = u"""
+    >>> doc = """
     ... <div itemscope itemtype="http://schema.org/Product">
     ...   <span itemprop="name">Kenmore White 17" Microwave</span>
     ...   <img src="kenmore-microwave-17in.jpg" alt='Kenmore 17" Microwave' />
diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py
index 9acfc3b23..95a3f17d5 100755
--- a/docs/utils/linkfix.py
+++ b/docs/utils/linkfix.py
@@ -23,7 +23,7 @@ def main():
     _contents = None
 
     # A regex that matches standard linkcheck output lines
-    line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
+    line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
 
     # Read lines from the linkcheck output file
     try:
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 0e6ceef0b..a260798ac 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -133,7 +133,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
                         '  not(re:test(., "^(?:checkbox|radio)$", "i")))]]',
                         namespaces={
                             "re": "http://exslt.org/regular-expressions"})
-    values = [(k, u'' if v is None else v)
+    values = [(k, '' if v is None else v)
               for k, v in (_value(e) for e in inputs)
               if k and k not in formdata_keys]
 
@@ -168,7 +168,7 @@ def _select_value(ele, n, v):
         # This is a workround to bug in lxml fixed 2.3.1
         # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139
         selected_options = ele.xpath('.//option[@selected]')
-        v = [(o.get('value') or o.text or u'').strip() for o in selected_options]
+        v = [(o.get('value') or o.text or '').strip() for o in selected_options]
     return n, v
 
 
@@ -205,7 +205,7 @@ def _get_clickable(clickdata, form):
 
     # We didn't find it, so now we build an XPath expression out of the other
     # arguments, because they can be used as such
-    xpath = u'.//*' + u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
+    xpath = './/*' + ''.join('[@%s="%s"]' % c for c in clickdata.items())
     el = form.xpath(xpath)
     if len(el) == 1:
         return (el[0].get('name'), el[0].get('value') or '')
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 1615d44d7..8b9f961ee 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -76,7 +76,7 @@ class LxmlParserLinkExtractor:
             url = safe_url_string(url, encoding=response_encoding)
             # to fix relative links after process_value
             url = urljoin(response_url, url)
-            link = Link(url, _collect_string_content(el) or u'',
+            link = Link(url, _collect_string_content(el) or '',
                         nofollow=rel_has_nofollow(el.get('rel')))
             links.append(link)
         return self._deduplicate_if_needed(links)
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 219145f13..0f9e6f1cb 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -44,7 +44,7 @@ class LogFormatter:
                 def dropped(self, item, exception, response, spider):
                     return {
                         'level': logging.INFO, # lowering the level from logging.WARNING
-                        'msg': u"Dropped: %(exception)s" + os.linesep + "%(item)s",
+                        'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s",
                         'args': {
                             'exception': exception,
                             'item': item,
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index da99a6be8..591075a98 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -59,7 +59,7 @@ class CmdlineTest(unittest.TestCase):
                                     'EXTENSIONS=' + json.dumps(EXTENSIONS))
         # XXX: There's gotta be a smarter way to do this...
         self.assertNotIn("...", settingsstr)
-        for char in ("'", "<", ">", 'u"'):
+        for char in ("'", "<", ">"):
             settingsstr = settingsstr.replace(char, '"')
         settingsdict = json.loads(settingsstr)
         self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys())
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 51deb20f4..57d4cdd6b 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1110,7 +1110,7 @@ class DataURITestCase(unittest.TestCase):
 
     def test_default_mediatype(self):
         def _test(response):
-            self.assertEqual(response.text, u'\u038e\u03a3\u038e')
+            self.assertEqual(response.text, '\u038e\u03a3\u038e')
             self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
             self.assertEqual(response.encoding, "iso-8859-7")
 
@@ -1119,7 +1119,7 @@ class DataURITestCase(unittest.TestCase):
 
     def test_text_charset(self):
         def _test(response):
-            self.assertEqual(response.text, u'\u038e\u03a3\u038e')
+            self.assertEqual(response.text, '\u038e\u03a3\u038e')
             self.assertEqual(response.body, b'\xbe\xd3\xbe')
             self.assertEqual(response.encoding, "iso-8859-7")
 
@@ -1128,7 +1128,7 @@ class DataURITestCase(unittest.TestCase):
 
     def test_mediatype_parameters(self):
         def _test(response):
-            self.assertEqual(response.text, u'\u038e\u03a3\u038e')
+            self.assertEqual(response.text, '\u038e\u03a3\u038e')
             self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
             self.assertEqual(response.encoding, "utf-8")
 
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 9ccc2110b..010577415 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -277,33 +277,33 @@ class CookiesMiddlewareTest(TestCase):
 
     def test_request_cookies_encoding(self):
         # 1) UTF8-encoded bytes
-        req1 = Request('http://example.org', cookies={'a': u'á'.encode('utf8')})
+        req1 = Request('http://example.org', cookies={'a': 'á'.encode('utf8')})
         assert self.mw.process_request(req1, self.spider) is None
         self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
 
         # 2) Non UTF8-encoded bytes
-        req2 = Request('http://example.org', cookies={'a': u'á'.encode('latin1')})
+        req2 = Request('http://example.org', cookies={'a': 'á'.encode('latin1')})
         assert self.mw.process_request(req2, self.spider) is None
         self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
 
-        # 3) Unicode string
-        req3 = Request('http://example.org', cookies={'a': u'á'})
+        # 3) String
+        req3 = Request('http://example.org', cookies={'a': 'á'})
         assert self.mw.process_request(req3, self.spider) is None
         self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
 
     def test_request_headers_cookie_encoding(self):
         # 1) UTF8-encoded bytes
-        req1 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('utf8')})
+        req1 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('utf8')})
         assert self.mw.process_request(req1, self.spider) is None
         self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
 
         # 2) Non UTF8-encoded bytes
-        req2 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('latin1')})
+        req2 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('latin1')})
         assert self.mw.process_request(req2, self.spider) is None
         self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
 
-        # 3) Unicode string
-        req3 = Request('http://example.org', headers={'Cookie': u'a=á'})
+        # 3) String
+        req3 = Request('http://example.org', headers={'Cookie': 'a=á'})
         assert self.mw.process_request(req3, self.spider) is None
         self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
 
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 9841d7a76..351631eb8 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -88,7 +88,7 @@ class TestHttpProxyMiddleware(TestCase):
 
     def test_proxy_auth_encoding(self):
         # utf-8 encoding
-        os.environ['http_proxy'] = u'https://m\u00E1n:pass@proxy:3128'
+        os.environ['http_proxy'] = 'https://m\u00E1n:pass@proxy:3128'
         mw = HttpProxyMiddleware(auth_encoding='utf-8')
         req = Request('http://scrapytest.org')
         assert mw.process_request(req, spider) is None
@@ -96,7 +96,7 @@ class TestHttpProxyMiddleware(TestCase):
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz')
 
         # proxy from request.meta
-        req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})
+        req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'})
         assert mw.process_request(req, spider) is None
         self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==')
@@ -109,7 +109,7 @@ class TestHttpProxyMiddleware(TestCase):
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=')
 
         # proxy from request.meta, latin-1 encoding
-        req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})
+        req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'})
         assert mw.process_request(req, spider) is None
         self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz')
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 919dbed23..131332131 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -184,7 +184,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
 
     def test_latin1_location(self):
         req = Request('http://scrapytest.org/first')
-        latin1_location = u'/ação'.encode('latin1')  # HTTP historically supports latin1
+        latin1_location = '/ação'.encode('latin1')  # HTTP historically supports latin1
         resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302)
         req_result = self.mw.process_response(req, resp, self.spider)
         perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o'
@@ -192,7 +192,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
 
     def test_utf8_location(self):
         req = Request('http://scrapytest.org/first')
-        utf8_location = u'/ação'.encode('utf-8')  # header using UTF-8 encoding
+        utf8_location = '/ação'.encode('utf-8')  # header using UTF-8 encoding
         resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302)
         req_result = self.mw.process_response(req, resp, self.spider)
         perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o'
@@ -207,7 +207,7 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
         self.mw = MetaRefreshMiddleware.from_crawler(crawler)
 
     def _body(self, interval=5, url='http://example.org/newpage'):
-        html = u"""<html><head><meta http-equiv="refresh" content="{0};url={1}"/></head></html>"""
+        html = """<html><head><meta http-equiv="refresh" content="{0};url={1}"/></head></html>"""
         return html.format(interval, url).encode('utf-8')
 
     def test_priority_adjust(self):
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index b9452a0e7..f9936baba 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -30,7 +30,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
     def _get_successful_crawler(self):
         crawler = self.crawler
         crawler.settings.set('ROBOTSTXT_OBEY', True)
-        ROBOTS = u"""
+        ROBOTS = """
 User-Agent: *
 Disallow: /admin/
 Disallow: /static/
@@ -56,7 +56,7 @@ Disallow: /some/randome/page.html
             self.assertIgnored(Request('http://site.local/admin/main'), middleware),
             self.assertIgnored(Request('http://site.local/static/'), middleware),
             self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware),
-            self.assertIgnored(Request(u'http://site.local/wiki/Käyttäjä:'), middleware)
+            self.assertIgnored(Request('http://site.local/wiki/Käyttäjä:'), middleware)
         ], fireOnOneErrback=True)
 
     def test_robotstxt_ready_parser(self):
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 25da54a65..660c99ce1 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -54,7 +54,7 @@ class BaseItemExporterTest(unittest.TestCase):
     def setUp(self):
         if self.item_class is None:
             raise unittest.SkipTest("item class is None")
-        self.i = self.item_class(name=u'John\xa3', age=u'22')
+        self.i = self.item_class(name='John\xa3', age='22')
         self.output = BytesIO()
         self.ie = self._get_exporter()
 
@@ -96,25 +96,25 @@ class BaseItemExporterTest(unittest.TestCase):
     def test_serialize_field(self):
         a = ItemAdapter(self.i)
         res = self.ie.serialize_field(a.get_field_meta('name'), 'name', a['name'])
-        self.assertEqual(res, u'John\xa3')
+        self.assertEqual(res, 'John\xa3')
 
         res = self.ie.serialize_field(a.get_field_meta('age'), 'age', a['age'])
-        self.assertEqual(res, u'22')
+        self.assertEqual(res, '22')
 
     def test_fields_to_export(self):
         ie = self._get_exporter(fields_to_export=['name'])
-        self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', u'John\xa3')])
+        self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xa3')])
 
         ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1')
         _, name = list(ie._get_serialized_fields(self.i))[0]
         assert isinstance(name, str)
-        self.assertEqual(name, u'John\xa3')
+        self.assertEqual(name, 'John\xa3')
 
     def test_field_custom_serializer(self):
-        i = self.custom_field_item_class(name=u'John\xa3', age=u'22')
+        i = self.custom_field_item_class(name='John\xa3', age='22')
         a = ItemAdapter(i)
         ie = self._get_exporter()
-        self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), u'John\xa3')
+        self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John\xa3')
         self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24')
 
 
@@ -132,48 +132,48 @@ class PythonItemExporterTest(BaseItemExporterTest):
             PythonItemExporter(invalid_option='something')
 
     def test_nested_item(self):
-        i1 = self.item_class(name=u'Joseph', age='22')
-        i2 = dict(name=u'Maria', age=i1)
-        i3 = self.item_class(name=u'Jesus', age=i2)
+        i1 = self.item_class(name='Joseph', age='22')
+        i2 = dict(name='Maria', age=i1)
+        i3 = self.item_class(name='Jesus', age=i2)
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(type(exported), dict)
         self.assertEqual(
             exported,
-            {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'}
+            {'age': {'age': {'age': '22', 'name': 'Joseph'}, 'name': 'Maria'}, 'name': 'Jesus'}
         )
         self.assertEqual(type(exported['age']), dict)
         self.assertEqual(type(exported['age']['age']), dict)
 
     def test_export_list(self):
-        i1 = self.item_class(name=u'Joseph', age='22')
-        i2 = self.item_class(name=u'Maria', age=[i1])
-        i3 = self.item_class(name=u'Jesus', age=[i2])
+        i1 = self.item_class(name='Joseph', age='22')
+        i2 = self.item_class(name='Maria', age=[i1])
+        i3 = self.item_class(name='Jesus', age=[i2])
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(
             exported,
-            {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}
+            {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'}
         )
         self.assertEqual(type(exported['age'][0]), dict)
         self.assertEqual(type(exported['age'][0]['age'][0]), dict)
 
     def test_export_item_dict_list(self):
-        i1 = self.item_class(name=u'Joseph', age='22')
-        i2 = dict(name=u'Maria', age=[i1])
-        i3 = self.item_class(name=u'Jesus', age=[i2])
+        i1 = self.item_class(name='Joseph', age='22')
+        i2 = dict(name='Maria', age=[i1])
+        i3 = self.item_class(name='Jesus', age=[i2])
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(
             exported,
-            {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}
+            {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'}
         )
         self.assertEqual(type(exported['age'][0]), dict)
         self.assertEqual(type(exported['age'][0]['age'][0]), dict)
 
     def test_export_binary(self):
         exporter = PythonItemExporter(binary=True)
-        value = self.item_class(name=u'John\xa3', age=u'22')
+        value = self.item_class(name='John\xa3', age='22')
         expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
         self.assertEqual(expected, exporter.export_item(value))
 
@@ -279,7 +279,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
         return self.assertEqual(split_csv(first), split_csv(second), msg=msg)
 
     def _check_output(self):
-        self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n')
+        self.assertCsvEqual(to_unicode(self.output.getvalue()), 'age,name\r\n22,John\xa3\r\n')
 
     def assertExportResult(self, item, expected, **kwargs):
         fp = BytesIO()
@@ -396,7 +396,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
 
     def test_multivalued_fields(self):
         self.assertExportResult(
-            self.item_class(name=[u'John\xa3', u'Doe'], age=[1, 2, 3]),
+            self.item_class(name=['John\xa3', 'Doe'], age=[1, 2, 3]),
             b"""<?xml version="1.0" encoding="utf-8"?>\n
             <items>
                 <item>
@@ -408,9 +408,9 @@ class XmlItemExporterTest(BaseItemExporterTest):
         )
 
     def test_nested_item(self):
-        i1 = dict(name=u'foo\xa3hoo', age='22')
-        i2 = dict(name=u'bar', age=i1)
-        i3 = self.item_class(name=u'buz', age=i2)
+        i1 = dict(name='foo\xa3hoo', age='22')
+        i2 = dict(name='bar', age=i1)
+        i3 = self.item_class(name='buz', age=i2)
 
         self.assertExportResult(
             i3,
@@ -431,9 +431,9 @@ class XmlItemExporterTest(BaseItemExporterTest):
         )
 
     def test_nested_list_item(self):
-        i1 = dict(name=u'foo')
-        i2 = dict(name=u'bar', v2={"egg": ["spam"]})
-        i3 = self.item_class(name=u'buz', age=[i1, i2])
+        i1 = dict(name='foo')
+        i2 = dict(name='bar', v2={"egg": ["spam"]})
+        i3 = self.item_class(name='buz', age=[i1, i2])
 
         self.assertExportResult(
             i3,
@@ -475,7 +475,7 @@ class XmlItemExporterDataclassTest(XmlItemExporterTest):
 
 class JsonLinesItemExporterTest(BaseItemExporterTest):
 
-    _expected_nested = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}}
+    _expected_nested = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}}
 
     def _get_exporter(self, **kwargs):
         return JsonLinesItemExporter(self.output, **kwargs)
@@ -485,9 +485,9 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
         self.assertEqual(exported, ItemAdapter(self.i).asdict())
 
     def test_nested_item(self):
-        i1 = self.item_class(name=u'Joseph', age='22')
-        i2 = dict(name=u'Maria', age=i1)
-        i3 = self.item_class(name=u'Jesus', age=i2)
+        i1 = self.item_class(name='Joseph', age='22')
+        i2 = dict(name='Maria', age=i1)
+        i3 = self.item_class(name='Jesus', age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
@@ -542,25 +542,25 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.assertTwoItemsExported(ItemAdapter(self.i).asdict())
 
     def test_nested_item(self):
-        i1 = self.item_class(name=u'Joseph\xa3', age='22')
-        i2 = self.item_class(name=u'Maria', age=i1)
-        i3 = self.item_class(name=u'Jesus', age=i2)
+        i1 = self.item_class(name='Joseph\xa3', age='22')
+        i2 = self.item_class(name='Maria', age=i1)
+        i3 = self.item_class(name='Jesus', age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
         exported = json.loads(to_unicode(self.output.getvalue()))
-        expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}}
+        expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}}
         self.assertEqual(exported, [expected])
 
     def test_nested_dict_item(self):
-        i1 = dict(name=u'Joseph\xa3', age='22')
-        i2 = self.item_class(name=u'Maria', age=i1)
-        i3 = dict(name=u'Jesus', age=i2)
+        i1 = dict(name='Joseph\xa3', age='22')
+        i2 = self.item_class(name='Maria', age=i1)
+        i3 = dict(name='Jesus', age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
         exported = json.loads(to_unicode(self.output.getvalue()))
-        expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}}
+        expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': i1}}
         self.assertEqual(exported, [expected])
 
     def test_nonstring_types_item(self):
@@ -595,14 +595,14 @@ class CustomExporterItemTest(unittest.TestCase):
                 else:
                     return super(CustomItemExporter, self).serialize_field(field, name, value)
 
-        i = self.item_class(name=u'John', age='22')
+        i = self.item_class(name='John', age='22')
         a = ItemAdapter(i)
         ie = CustomItemExporter()
 
         self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John')
         self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '23')
 
-        i2 = {'name': u'John', 'age': '22'}
+        i2 = {'name': 'John', 'age': '22'}
         self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John')
         self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23')
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index c49b2e92f..b57349848 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -874,7 +874,7 @@ class FeedExportTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_export_encoding(self):
-        items = [dict({'foo': u'Test\xd6'})]
+        items = [dict({'foo': 'Test\xd6'})]
 
         formats = {
             'json': '[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
@@ -919,7 +919,7 @@ class FeedExportTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_export_multiple_configs(self):
-        items = [dict({'foo': u'FOO', 'bar': u'BAR'})]
+        items = [dict({'foo': 'FOO', 'bar': 'BAR'})]
 
         formats = {
             'json': '[\n{"bar": "BAR"}\n]'.encode('utf-8'),
@@ -1393,7 +1393,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_export_multiple_configs(self):
-        items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]
+        items = [dict({'foo': 'FOO', 'bar': 'BAR'}), dict({'foo': 'FOO1', 'bar': 'BAR1'})]
 
         formats = {
             'json': ['[\n{"bar": "BAR"}\n]'.encode('utf-8'),
@@ -1442,7 +1442,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_batch_item_count_feeds_setting(self):
-        items = [dict({'foo': u'FOO'}), dict({'foo': u'FOO1'})]
+        items = [dict({'foo': 'FOO'}), dict({'foo': 'FOO1'})]
         formats = {
             'json': ['[{"foo": "FOO"}]'.encode('utf-8'),
                      '[{"foo": "FOO1"}]'.encode('utf-8')],
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index cf3fc8496..64ff7a73d 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -39,19 +39,19 @@ class HeadersTest(unittest.TestCase):
         assert h.getlist('X-Forwarded-For') is not hlist
 
     def test_encode_utf8(self):
-        h = Headers({u'key': u'\xa3'}, encoding='utf-8')
+        h = Headers({'key': '\xa3'}, encoding='utf-8')
         key, val = dict(h).popitem()
         assert isinstance(key, bytes), key
         assert isinstance(val[0], bytes), val[0]
         self.assertEqual(val[0], b'\xc2\xa3')
 
     def test_encode_latin1(self):
-        h = Headers({u'key': u'\xa3'}, encoding='latin1')
+        h = Headers({'key': '\xa3'}, encoding='latin1')
         key, val = dict(h).popitem()
         self.assertEqual(val[0], b'\xa3')
 
     def test_encode_multiple(self):
-        h = Headers({u'key': [u'\xa3']}, encoding='utf-8')
+        h = Headers({'key': ['\xa3']}, encoding='utf-8')
         key, val = dict(h).popitem()
         self.assertEqual(val[0], b'\xc2\xa3')
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 63014b22d..f5cf4e798 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -60,8 +60,8 @@ class RequestTest(unittest.TestCase):
         self.assertFalse(p.headers is r.headers)
 
         # headers must not be unicode
-        h = Headers({'key1': u'val1', u'key2': 'val2'})
-        h[u'newkey'] = u'newval'
+        h = Headers({'key1': 'val1', 'key2': 'val2'})
+        h['newkey'] = 'newval'
         for k, v in h.items():
             self.assertIsInstance(k, bytes)
             for s in v:
@@ -89,30 +89,30 @@ class RequestTest(unittest.TestCase):
         self.assertEqual(r.url, "http://www.scrapy.org/blank%20space")
 
     def test_url_encoding(self):
-        r = self.request_class(url=u"http://www.scrapy.org/price/£")
+        r = self.request_class(url="http://www.scrapy.org/price/£")
         self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
 
     def test_url_encoding_other(self):
         # encoding affects only query part of URI, not path
         # path part should always be UTF-8 encoded before percent-escaping
-        r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="utf-8")
+        r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8")
         self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
 
-        r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="latin1")
+        r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1")
         self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
 
     def test_url_encoding_query(self):
-        r1 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ")
+        r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ")
         self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5")
 
         # should be same as above
-        r2 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ", encoding="utf-8")
+        r2 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8")
         self.assertEqual(r2.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5")
 
     def test_url_encoding_query_latin1(self):
         # encoding is used for encoding query-string before percent-escaping;
         # path is still UTF-8 encoded before percent-escaping
-        r3 = self.request_class(url=u"http://www.scrapy.org/price/µ?currency=£", encoding="latin1")
+        r3 = self.request_class(url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1")
         self.assertEqual(r3.url, "http://www.scrapy.org/price/%C2%B5?currency=%A3")
 
     def test_url_encoding_nonutf8_untouched(self):
@@ -131,16 +131,16 @@ class RequestTest(unittest.TestCase):
         # characters.  Otherwise, in the future the IRI will be mapped to
         # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different
         # URI from "http://www.example.org/r%E9sum%E9.html".
-        r1 = self.request_class(url=u"http://www.scrapy.org/price/%a3")
+        r1 = self.request_class(url="http://www.scrapy.org/price/%a3")
         self.assertEqual(r1.url, "http://www.scrapy.org/price/%a3")
 
-        r2 = self.request_class(url=u"http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
+        r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
         self.assertEqual(r2.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
 
-        r3 = self.request_class(url=u"http://www.scrapy.org/résumé/%a3")
+        r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3")
         self.assertEqual(r3.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
 
-        r4 = self.request_class(url=u"http://www.example.org/r%E9sum%E9.html")
+        r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html")
         self.assertEqual(r4.url, "http://www.example.org/r%E9sum%E9.html")
 
     def test_body(self):
@@ -151,11 +151,11 @@ class RequestTest(unittest.TestCase):
         assert isinstance(r2.body, bytes)
         self.assertEqual(r2.encoding, 'utf-8')  # default encoding
 
-        r3 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8')
+        r3 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='utf-8')
         assert isinstance(r3.body, bytes)
         self.assertEqual(r3.body, b"Price: \xc2\xa3100")
 
-        r4 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='latin1')
+        r4 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='latin1')
         assert isinstance(r4.body, bytes)
         self.assertEqual(r4.body, b"Price: \xa3100")
 
@@ -164,7 +164,7 @@ class RequestTest(unittest.TestCase):
         r = self.request_class(url="http://www.example.com/ajax.html#!key=value")
         self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue")
         # unicode url
-        r = self.request_class(url=u"http://www.example.com/ajax.html#!key=value")
+        r = self.request_class(url="http://www.example.com/ajax.html#!key=value")
         self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue")
 
     def test_copy(self):
@@ -236,7 +236,7 @@ class RequestTest(unittest.TestCase):
         assert r4.dont_filter is False
 
     def test_method_always_str(self):
-        r = self.request_class("http://www.example.com", method=u"POST")
+        r = self.request_class("http://www.example.com", method="POST")
         assert isinstance(r.method, str)
 
     def test_immutable_attributes(self):
@@ -381,7 +381,7 @@ class FormRequestTest(RequestTest):
 
     def test_default_encoding_textual_data(self):
         # using default encoding (utf-8)
-        data = {u'µ one': u'two', u'price': u'£ 100'}
+        data = {'µ one': 'two', 'price': '£ 100'}
         r2 = self.request_class("http://www.example.com", formdata=data)
         self.assertEqual(r2.method, 'POST')
         self.assertEqual(r2.encoding, 'utf-8')
@@ -390,7 +390,7 @@ class FormRequestTest(RequestTest):
 
     def test_default_encoding_mixed_data(self):
         # using default encoding (utf-8)
-        data = {u'\u00b5one': b'two', b'price\xc2\xa3': u'\u00a3 100'}
+        data = {'\u00b5one': b'two', b'price\xc2\xa3': '\u00a3 100'}
         r2 = self.request_class("http://www.example.com", formdata=data)
         self.assertEqual(r2.method, 'POST')
         self.assertEqual(r2.encoding, 'utf-8')
@@ -406,14 +406,14 @@ class FormRequestTest(RequestTest):
         self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded')
 
     def test_custom_encoding_textual_data(self):
-        data = {'price': u'£ 100'}
+        data = {'price': '£ 100'}
         r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
         self.assertEqual(r3.encoding, 'latin1')
         self.assertEqual(r3.body, b'price=%A3+100')
 
     def test_multi_key_values(self):
         # using multiples values for a single key
-        data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']}
+        data = {'price': '\xa3 100', 'colours': ['red', 'blue', 'green']}
         r3 = self.request_class("http://www.example.com", formdata=data)
         self.assertQueryEqual(r3.body, b'colours=red&colours=blue&colours=green&price=%C2%A3+100')
 
@@ -450,10 +450,10 @@ class FormRequestTest(RequestTest):
         self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
         self.assertEqual(req.url, "http://www.example.com/this/post.php")
         fs = _qs(req, to_unicode=True)
-        self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})
-        self.assertEqual(set(fs[u'one']), {u'two', u'three'})
-        self.assertEqual(fs[u'test2'], [u'xxx µ'])
-        self.assertEqual(fs[u'six'], [u'seven'])
+        self.assertEqual(set(fs['test £']), {'val1', 'val2'})
+        self.assertEqual(set(fs['one']), {'two', 'three'})
+        self.assertEqual(fs['test2'], ['xxx µ'])
+        self.assertEqual(fs['six'], ['seven'])
 
     def test_from_response_post_nonascii_bytes_latin1(self):
         response = _buildresponse(
@@ -471,14 +471,14 @@ class FormRequestTest(RequestTest):
         self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
         self.assertEqual(req.url, "http://www.example.com/this/post.php")
         fs = _qs(req, to_unicode=True, encoding='latin1')
-        self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})
-        self.assertEqual(set(fs[u'one']), {u'two', u'three'})
-        self.assertEqual(fs[u'test2'], [u'xxx µ'])
-        self.assertEqual(fs[u'six'], [u'seven'])
+        self.assertEqual(set(fs['test £']), {'val1', 'val2'})
+        self.assertEqual(set(fs['one']), {'two', 'three'})
+        self.assertEqual(fs['test2'], ['xxx µ'])
+        self.assertEqual(fs['six'], ['seven'])
 
     def test_from_response_post_nonascii_unicode(self):
         response = _buildresponse(
-            u"""<form action="post.php" method="POST">
+            """<form action="post.php" method="POST">
             <input type="hidden" name="test £" value="val1">
             <input type="hidden" name="test £" value="val2">
             <input type="hidden" name="test2" value="xxx µ">
@@ -490,10 +490,10 @@ class FormRequestTest(RequestTest):
         self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
         self.assertEqual(req.url, "http://www.example.com/this/post.php")
         fs = _qs(req, to_unicode=True)
-        self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})
-        self.assertEqual(set(fs[u'one']), {u'two', u'three'})
-        self.assertEqual(fs[u'test2'], [u'xxx µ'])
-        self.assertEqual(fs[u'six'], [u'seven'])
+        self.assertEqual(set(fs['test £']), {'val1', 'val2'})
+        self.assertEqual(set(fs['one']), {'two', 'three'})
+        self.assertEqual(fs['test2'], ['xxx µ'])
+        self.assertEqual(fs['six'], ['seven'])
 
     def test_from_response_duplicate_form_key(self):
         response = _buildresponse(
@@ -685,7 +685,7 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="two" value="clicked2">
             </form>""")
         req = self.request_class.from_response(
-            response, clickdata={u'name': u'clickable', u'value': u'clicked2'}
+            response, clickdata={'name': 'clickable', 'value': 'clicked2'}
         )
         fs = _qs(req)
         self.assertEqual(fs[b'clickable'], [b'clicked2'])
@@ -694,21 +694,21 @@ class FormRequestTest(RequestTest):
 
     def test_from_response_unicode_clickdata(self):
         response = _buildresponse(
-            u"""<form action="get.php" method="GET">
+            """<form action="get.php" method="GET">
             <input type="submit" name="price in \u00a3" value="\u00a3 1000">
             <input type="submit" name="price in \u20ac" value="\u20ac 2000">
             <input type="hidden" name="poundsign" value="\u00a3">
             <input type="hidden" name="eurosign" value="\u20ac">
             </form>""")
         req = self.request_class.from_response(
-            response, clickdata={u'name': u'price in \u00a3'}
+            response, clickdata={'name': 'price in \u00a3'}
         )
         fs = _qs(req, to_unicode=True)
-        self.assertTrue(fs[u'price in \u00a3'])
+        self.assertTrue(fs['price in \u00a3'])
 
     def test_from_response_unicode_clickdata_latin1(self):
         response = _buildresponse(
-            u"""<form action="get.php" method="GET">
+            """<form action="get.php" method="GET">
             <input type="submit" name="price in \u00a3" value="\u00a3 1000">
             <input type="submit" name="price in \u00a5" value="\u00a5 2000">
             <input type="hidden" name="poundsign" value="\u00a3">
@@ -716,10 +716,10 @@ class FormRequestTest(RequestTest):
             </form>""",
             encoding='latin1')
         req = self.request_class.from_response(
-            response, clickdata={u'name': u'price in \u00a5'}
+            response, clickdata={'name': 'price in \u00a5'}
         )
         fs = _qs(req, to_unicode=True, encoding='latin1')
-        self.assertTrue(fs[u'price in \u00a5'])
+        self.assertTrue(fs['price in \u00a5'])
 
     def test_from_response_multiple_forms_clickdata(self):
         response = _buildresponse(
@@ -733,7 +733,7 @@ class FormRequestTest(RequestTest):
             </form>
             """)
         req = self.request_class.from_response(
-            response, formname='form2', clickdata={u'name': u'clickable'}
+            response, formname='form2', clickdata={'name': 'clickable'}
         )
         fs = _qs(req)
         self.assertEqual(fs[b'clickable'], [b'clicked2'])
@@ -1072,11 +1072,11 @@ class FormRequestTest(RequestTest):
 
     def test_from_response_unicode_xpath(self):
         response = _buildresponse(b'<form name="\xd1\x8a"></form>')
-        r = self.request_class.from_response(response, formxpath=u"//form[@name='\u044a']")
+        r = self.request_class.from_response(response, formxpath="//form[@name='\u044a']")
         fs = _qs(r)
         self.assertEqual(fs, {})
 
-        xpath = u"//form[@name='\u03b1']"
+        xpath = "//form[@name='\u03b1']"
         self.assertRaisesRegex(ValueError, re.escape(xpath),
                                self.request_class.from_response,
                                response, formxpath=xpath)
@@ -1246,13 +1246,13 @@ class XmlRpcRequestTest(RequestTest):
         self._test_request(params=('value',))
         self._test_request(params=('username', 'password'), methodname='login')
         self._test_request(params=('response', ), methodresponse='login')
-        self._test_request(params=(u'pas£',), encoding='utf-8')
+        self._test_request(params=('pas£',), encoding='utf-8')
         self._test_request(params=(None,), allow_none=1)
         self.assertRaises(TypeError, self._test_request)
         self.assertRaises(TypeError, self._test_request, params=(None,))
 
     def test_latin1(self):
-        self._test_request(params=(u'pas£',), encoding='latin1')
+        self._test_request(params=('pas£',), encoding='latin1')
 
 
 class JsonRequestTest(RequestTest):
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index e0ca3c0e6..56d017de6 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -318,28 +318,28 @@ class TextResponseTest(BaseResponseTest):
 
     def test_unicode_url(self):
         # instantiate with unicode url without encoding (should set default encoding)
-        resp = self.response_class(u"http://www.example.com/")
+        resp = self.response_class("http://www.example.com/")
         self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING)
 
         # make sure urls are converted to str
-        resp = self.response_class(url=u"http://www.example.com/", encoding='utf-8')
+        resp = self.response_class(url="http://www.example.com/", encoding='utf-8')
         assert isinstance(resp.url, str)
 
-        resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8')
+        resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='utf-8')
         self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
-        resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1')
+        resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='latin-1')
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
-        resp = self.response_class(u"http://www.example.com/price/\xa3",
+        resp = self.response_class("http://www.example.com/price/\xa3",
                                    headers={"Content-type": ["text/html; charset=utf-8"]})
         self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
-        resp = self.response_class(u"http://www.example.com/price/\xa3",
+        resp = self.response_class("http://www.example.com/price/\xa3",
                                    headers={"Content-type": ["text/html; charset=iso-8859-1"]})
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
 
     def test_unicode_body(self):
         unicode_string = ('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 '
                           '\u0442\u0435\u043a\u0441\u0442')
-        self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body=u'unicode body')
+        self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body='unicode body')
 
         original_string = unicode_string.encode('cp1251')
         r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251')
@@ -355,7 +355,7 @@ class TextResponseTest(BaseResponseTest):
     def test_encoding(self):
         r1 = self.response_class("http://www.example.com", body=b"\xc2\xa3",
                                  headers={"Content-type": ["text/html; charset=utf-8"]})
-        r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3")
+        r2 = self.response_class("http://www.example.com", encoding='utf-8', body="\xa3")
         r3 = self.response_class("http://www.example.com", body=b"\xa3",
                                  headers={"Content-type": ["text/html; charset=iso-8859-1"]})
         r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3")
@@ -376,14 +376,14 @@ class TextResponseTest(BaseResponseTest):
         self.assertEqual(r5._headers_encoding(), None)
         self._assert_response_encoding(r5, "utf-8")
         assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii'
-        self._assert_response_values(r1, 'utf-8', u"\xa3")
-        self._assert_response_values(r2, 'utf-8', u"\xa3")
-        self._assert_response_values(r3, 'iso-8859-1', u"\xa3")
-        self._assert_response_values(r6, 'gb18030', u"\u2015")
-        self._assert_response_values(r7, 'gb18030', u"\u2015")
+        self._assert_response_values(r1, 'utf-8', "\xa3")
+        self._assert_response_values(r2, 'utf-8', "\xa3")
+        self._assert_response_values(r3, 'iso-8859-1', "\xa3")
+        self._assert_response_values(r6, 'gb18030', "\u2015")
+        self._assert_response_values(r7, 'gb18030', "\u2015")
 
         # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies
-        self.assertRaises(TypeError, self.response_class, "http://www.example.com", body=u"\xa3")
+        self.assertRaises(TypeError, self.response_class, "http://www.example.com", body="\xa3")
 
     def test_declared_encoding_invalid(self):
         """Check that unknown declared encodings are ignored"""
@@ -391,14 +391,14 @@ class TextResponseTest(BaseResponseTest):
                                 headers={"Content-type": ["text/html; charset=UKNOWN"]},
                                 body=b"\xc2\xa3")
         self.assertEqual(r._declared_encoding(), None)
-        self._assert_response_values(r, 'utf-8', u"\xa3")
+        self._assert_response_values(r, 'utf-8', "\xa3")
 
     def test_utf16(self):
         """Test utf-16 because UnicodeDammit is known to have problems with"""
         r = self.response_class("http://www.example.com",
                                 body=b'\xff\xfeh\x00i\x00',
                                 encoding='utf-16')
-        self._assert_response_values(r, 'utf-16', u"hi")
+        self._assert_response_values(r, 'utf-16', "hi")
 
     def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self):
         r6 = self.response_class("http://www.example.com",
@@ -406,8 +406,8 @@ class TextResponseTest(BaseResponseTest):
                                  body=b"\xef\xbb\xbfWORD\xe3\xab")
         self.assertEqual(r6.encoding, 'utf-8')
         self.assertIn(r6.text, {
-            u'WORD\ufffd\ufffd',  # w3lib < 1.19.0
-            u'WORD\ufffd',        # w3lib >= 1.19.0
+            'WORD\ufffd\ufffd',  # w3lib < 1.19.0
+            'WORD\ufffd',        # w3lib >= 1.19.0
         })
 
     def test_bom_is_removed_from_body(self):
@@ -422,9 +422,9 @@ class TextResponseTest(BaseResponseTest):
         # Test response without content-type and BOM encoding
         response = self.response_class(url, body=body)
         self.assertEqual(response.encoding, 'utf-8')
-        self.assertEqual(response.text, u'WORD')
+        self.assertEqual(response.text, 'WORD')
         response = self.response_class(url, body=body)
-        self.assertEqual(response.text, u'WORD')
+        self.assertEqual(response.text, 'WORD')
         self.assertEqual(response.encoding, 'utf-8')
 
         # Body caching sideeffect isn't triggered when encoding is declared in
@@ -432,28 +432,28 @@ class TextResponseTest(BaseResponseTest):
         # body
         response = self.response_class(url, headers=headers, body=body)
         self.assertEqual(response.encoding, 'utf-8')
-        self.assertEqual(response.text, u'WORD')
+        self.assertEqual(response.text, 'WORD')
         response = self.response_class(url, headers=headers, body=body)
-        self.assertEqual(response.text, u'WORD')
+        self.assertEqual(response.text, 'WORD')
         self.assertEqual(response.encoding, 'utf-8')
 
     def test_replace_wrong_encoding(self):
         """Test invalid chars are replaced properly"""
         r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX')
         # XXX: Policy for replacing invalid chars may suffer minor variations
-        # but it should always contain the unicode replacement char (u'\ufffd')
-        assert u'\ufffd' in r.text, repr(r.text)
-        assert u'PREFIX' in r.text, repr(r.text)
-        assert u'SUFFIX' in r.text, repr(r.text)
+        # but it should always contain the unicode replacement char ('\ufffd')
+        assert '\ufffd' in r.text, repr(r.text)
+        assert 'PREFIX' in r.text, repr(r.text)
+        assert 'SUFFIX' in r.text, repr(r.text)
 
         # Do not destroy html tags due to encoding bugs
         r = self.response_class("http://example.com", encoding='utf-8',
                                 body=b'\xf0<span>value</span>')
-        assert u'<span>value</span>' in r.text, repr(r.text)
+        assert '<span>value</span>' in r.text, repr(r.text)
 
         # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse
         # r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX')
-        # assert u'\ufffd' in r.text, repr(r.text)
+        # assert '\ufffd' in r.text, repr(r.text)
 
     def test_selector(self):
         body = b"<html><head><title>Some page</title><body></body></html>"
@@ -466,15 +466,15 @@ class TextResponseTest(BaseResponseTest):
 
         self.assertEqual(
             response.selector.xpath("//title/text()").getall(),
-            [u'Some page']
+            ['Some page']
         )
         self.assertEqual(
             response.selector.css("title::text").getall(),
-            [u'Some page']
+            ['Some page']
         )
         self.assertEqual(
             response.selector.re("Some (.*)</title>"),
-            [u'page']
+            ['page']
         )
 
     def test_selector_shortcuts(self):
@@ -595,7 +595,7 @@ class TextResponseTest(BaseResponseTest):
         resp1 = self.response_class(
             'http://example.com',
             encoding='utf8',
-            body=u'<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8')
+            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8')
         )
         req = self._assert_followed_url(
             resp1.css('a')[0],
@@ -607,7 +607,7 @@ class TextResponseTest(BaseResponseTest):
         resp2 = self.response_class(
             'http://example.com',
             encoding='cp1251',
-            body=u'<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251')
+            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251')
         )
         req = self._assert_followed_url(
             resp2.css('a')[0],
@@ -681,8 +681,8 @@ class TextResponseTest(BaseResponseTest):
 
     def test_body_as_unicode_deprecation_warning(self):
         with catch_warnings(record=True) as warnings:
-            r1 = self.response_class("http://www.example.com", body=u'Hello', encoding='utf-8')
-            self.assertEqual(r1.body_as_unicode(), u'Hello')
+            r1 = self.response_class("http://www.example.com", body='Hello', encoding='utf-8')
+            self.assertEqual(r1.body_as_unicode(), 'Hello')
             self.assertEqual(len(warnings), 1)
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
 
@@ -787,7 +787,7 @@ class XmlResponseTest(TextResponseTest):
 
         self.assertEqual(
             response.selector.xpath("//elem/text()").getall(),
-            [u'value']
+            ['value']
         )
 
     def test_selector_shortcuts(self):
diff --git a/tests/test_item.py b/tests/test_item.py
index 60468971c..0ce78f8c0 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -20,8 +20,8 @@ class ItemTest(unittest.TestCase):
             name = Field()
 
         i = TestItem()
-        i['name'] = u'name'
-        self.assertEqual(i['name'], u'name')
+        i['name'] = 'name'
+        self.assertEqual(i['name'], 'name')
 
     def test_init(self):
         class TestItem(Item):
@@ -30,17 +30,17 @@ class ItemTest(unittest.TestCase):
         i = TestItem()
         self.assertRaises(KeyError, i.__getitem__, 'name')
 
-        i2 = TestItem(name=u'john doe')
-        self.assertEqual(i2['name'], u'john doe')
+        i2 = TestItem(name='john doe')
+        self.assertEqual(i2['name'], 'john doe')
 
-        i3 = TestItem({'name': u'john doe'})
-        self.assertEqual(i3['name'], u'john doe')
+        i3 = TestItem({'name': 'john doe'})
+        self.assertEqual(i3['name'], 'john doe')
 
         i4 = TestItem(i3)
-        self.assertEqual(i4['name'], u'john doe')
+        self.assertEqual(i4['name'], 'john doe')
 
-        self.assertRaises(KeyError, TestItem, {'name': u'john doe',
-                                               'other': u'foo'})
+        self.assertRaises(KeyError, TestItem, {'name': 'john doe',
+                                               'other': 'foo'})
 
     def test_invalid_field(self):
         class TestItem(Item):
@@ -56,7 +56,7 @@ class ItemTest(unittest.TestCase):
             number = Field()
 
         i = TestItem()
-        i['name'] = u'John Doe'
+        i['name'] = 'John Doe'
         i['number'] = 123
         itemrepr = repr(i)
 
@@ -101,9 +101,9 @@ class ItemTest(unittest.TestCase):
 
         i = TestItem()
         self.assertRaises(KeyError, i.get_name)
-        i['name'] = u'lala'
-        self.assertEqual(i.get_name(), u'lala')
-        i.change_name(u'other')
+        i['name'] = 'lala'
+        self.assertEqual(i.get_name(), 'lala')
+        i.change_name('other')
         self.assertEqual(i.get_name(), 'other')
 
     def test_metaclass(self):
@@ -113,22 +113,22 @@ class ItemTest(unittest.TestCase):
             values = Field()
 
         i = TestItem()
-        i['name'] = u'John'
+        i['name'] = 'John'
         self.assertEqual(list(i.keys()), ['name'])
         self.assertEqual(list(i.values()), ['John'])
 
-        i['keys'] = u'Keys'
-        i['values'] = u'Values'
+        i['keys'] = 'Keys'
+        i['values'] = 'Values'
         self.assertSortedEqual(list(i.keys()), ['keys', 'values', 'name'])
-        self.assertSortedEqual(list(i.values()), [u'Keys', u'Values', u'John'])
+        self.assertSortedEqual(list(i.values()), ['Keys', 'Values', 'John'])
 
     def test_metaclass_with_fields_attribute(self):
         class TestItem(Item):
             fields = {'new': Field(default='X')}
 
-        item = TestItem(new=u'New')
+        item = TestItem(new='New')
         self.assertSortedEqual(list(item.keys()), ['new'])
-        self.assertSortedEqual(list(item.values()), [u'New'])
+        self.assertSortedEqual(list(item.values()), ['New'])
 
     def test_metaclass_inheritance(self):
         class ParentItem(Item):
@@ -238,8 +238,8 @@ class ItemTest(unittest.TestCase):
             name = Field()
 
         i = TestItem()
-        i['name'] = u'John'
-        self.assertEqual(dict(i), {'name': u'John'})
+        i['name'] = 'John'
+        self.assertEqual(dict(i), {'name': 'John'})
 
     def test_copy(self):
         class TestItem(Item):
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 8d4538eed..a0bafa5e5 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -31,31 +31,31 @@ class Base:
             page4_url = 'http://example.com/page%204.html'
 
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
                 Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
-                Link(url='http://www.google.com/something', text=u''),
-                Link(url='http://example.com/innertag.html', text=u'inner tag'),
-                Link(url=page4_url, text=u'href with whitespaces'),
+                Link(url='http://www.google.com/something', text=''),
+                Link(url='http://example.com/innertag.html', text='inner tag'),
+                Link(url=page4_url, text='href with whitespaces'),
             ])
 
         def test_extract_filter_allow(self):
             lx = self.extractor_cls(allow=('sample', ))
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
                 Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment')
             ])
 
         def test_extract_filter_allow_with_duplicates(self):
             lx = self.extractor_cls(allow=('sample', ), unique=False)
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
+                Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
                 Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment')
             ])
 
@@ -63,10 +63,10 @@ class Base:
             lx = self.extractor_cls(allow=('sample', ), unique=False,
                                     canonicalize=True)
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
+                Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
                 Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment')
             ])
 
@@ -74,22 +74,22 @@ class Base:
             lx = self.extractor_cls(allow=('sample',), unique=True,
                                     canonicalize=True)
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
             ])
 
         def test_extract_filter_allow_and_deny(self):
             lx = self.extractor_cls(allow=('sample', ), deny=('3', ))
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
             ])
 
         def test_extract_filter_allowed_domains(self):
             lx = self.extractor_cls(allow_domains=('google.com', ))
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://www.google.com/something', text=u''),
+                Link(url='http://www.google.com/something', text=''),
             ])
 
         def test_extraction_using_single_values(self):
@@ -97,27 +97,27 @@ class Base:
 
             lx = self.extractor_cls(allow='sample')
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
                 Link(url='http://example.com/sample3.html#foo',
                      text='sample 3 repetition with fragment')
             ])
 
             lx = self.extractor_cls(allow='sample', deny='3')
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
             ])
 
             lx = self.extractor_cls(allow_domains='google.com')
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://www.google.com/something', text=u''),
+                Link(url='http://www.google.com/something', text=''),
             ])
 
             lx = self.extractor_cls(deny_domains='example.com')
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://www.google.com/something', text=u''),
+                Link(url='http://www.google.com/something', text=''),
             ])
 
         def test_nofollow(self):
@@ -145,11 +145,11 @@ class Base:
 
             lx = self.extractor_cls()
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.org/about.html', text=u'About us'),
-                Link(url='http://example.org/follow.html', text=u'Follow this link'),
-                Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True),
-                Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'),
-                Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+                Link(url='http://example.org/about.html', text='About us'),
+                Link(url='http://example.org/follow.html', text='Follow this link'),
+                Link(url='http://example.org/nofollow.html', text='Dont follow this one', nofollow=True),
+                Link(url='http://example.org/nofollow2.html', text='Choose to follow or not'),
+                Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
             ])
 
         def test_matches(self):
@@ -183,8 +183,8 @@ class Base:
         def test_restrict_xpaths(self):
             lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ))
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
             ])
 
         def test_restrict_xpaths_encoding(self):
@@ -202,14 +202,14 @@ class Base:
 
             lx = self.extractor_cls(restrict_xpaths="//div[@class='links']")
             self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.org/about.html', text=u'About us\xa3')])
+                             [Link(url='http://example.org/about.html', text='About us\xa3')])
 
         def test_restrict_xpaths_with_html_entities(self):
             html = b'<html><body><p><a href="/&hearts;/you?c=&euro;">text</a></p></body></html>'
             response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15')
             links = self.extractor_cls(restrict_xpaths='//p').extract_links(response)
             self.assertEqual(links,
-                             [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text=u'text')])
+                             [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text='text')])
 
         def test_restrict_xpaths_concat_in_handle_data(self):
             """html entities cause SGMLParser to call handle_data hook twice"""
@@ -217,22 +217,22 @@ class Base:
             response = HtmlResponse("http://example.org", body=body, encoding='gb18030')
             lx = self.extractor_cls(restrict_xpaths="//div")
             self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c',
+                             [Link(url='http://example.org/foo', text='>\u4eac<\u4e1c',
                                    fragment='', nofollow=False)])
 
         def test_restrict_css(self):
             lx = self.extractor_cls(restrict_css=('#subwrapper a',))
             self.assertEqual(lx.extract_links(self.response), [
-                Link(url='http://example.com/sample2.html', text=u'sample 2')
+                Link(url='http://example.com/sample2.html', text='sample 2')
             ])
 
         def test_restrict_css_and_restrict_xpaths_together(self):
             lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ),
                                     restrict_css=('#subwrapper + a', ))
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
             ])
 
         def test_area_tag_with_unicode_present(self):
@@ -243,7 +243,7 @@ class Base:
             lx.extract_links(response)
             lx.extract_links(response)
             self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.org/foo', text=u'',
+                             [Link(url='http://example.org/foo', text='',
                                    fragment='', nofollow=False)])
 
         def test_encoded_url(self):
@@ -251,7 +251,7 @@ class Base:
             response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
             lx = self.extractor_cls()
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
+                Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False),
             ])
 
         def test_encoded_url_in_restricted_xpath(self):
@@ -259,7 +259,7 @@ class Base:
             response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
             lx = self.extractor_cls(restrict_xpaths="//div")
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
+                Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False),
             ])
 
         def test_ignored_extensions(self):
@@ -268,7 +268,7 @@ class Base:
             response = HtmlResponse("http://example.org/", body=html)
             lx = self.extractor_cls()
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.org/page.html', text=u'asd'),
+                Link(url='http://example.org/page.html', text='asd'),
             ])
 
             # override denied extensions
@@ -308,25 +308,25 @@ class Base:
             page4_url = 'http://example.com/page%204.html'
 
             self.assertEqual(lx.extract_links(self.response), [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
                 Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
-                Link(url='http://www.google.com/something', text=u''),
-                Link(url='http://example.com/innertag.html', text=u'inner tag'),
-                Link(url=page4_url, text=u'href with whitespaces'),
+                Link(url='http://www.google.com/something', text=''),
+                Link(url='http://example.com/innertag.html', text='inner tag'),
+                Link(url=page4_url, text='href with whitespaces'),
             ])
 
             lx = self.extractor_cls(attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=())
             self.assertEqual(lx.extract_links(self.response), [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample2.jpg', text=u''),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample2.jpg', text=''),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
                 Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
-                Link(url='http://www.google.com/something', text=u''),
-                Link(url='http://example.com/innertag.html', text=u'inner tag'),
-                Link(url=page4_url, text=u'href with whitespaces'),
+                Link(url='http://www.google.com/something', text=''),
+                Link(url='http://example.com/innertag.html', text='inner tag'),
+                Link(url=page4_url, text='href with whitespaces'),
             ])
 
             lx = self.extractor_cls(attrs=None)
@@ -344,24 +344,24 @@ class Base:
 
             lx = self.extractor_cls()
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
             ])
 
             lx = self.extractor_cls(tags="area")
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/sample1.html', text=u''),
+                Link(url='http://example.com/sample1.html', text=''),
             ])
 
             lx = self.extractor_cls(tags="a")
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
             ])
 
             lx = self.extractor_cls(tags=("a", "img"), attrs=("href", "src"), deny_extensions=())
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample2.jpg', text=u''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample2.jpg', text=''),
             ])
 
         def test_tags_attrs(self):
@@ -375,14 +375,14 @@ class Base:
 
             lx = self.extractor_cls(tags='div', attrs='data-url')
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
-                Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
+                Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False),
+                Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False)
             ])
 
             lx = self.extractor_cls(tags=('div',), attrs=('data-url',))
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
-                Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
+                Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False),
+                Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False)
             ])
 
         def test_xhtml(self):
@@ -420,13 +420,13 @@ class Base:
             self.assertEqual(
                 lx.extract_links(response),
                 [
-                    Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
-                    Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
-                    Link(url='http://example.com/nofollow.html', text=u'Dont follow this one',
+                    Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False),
+                    Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False),
+                    Link(url='http://example.com/nofollow.html', text='Dont follow this one',
                          fragment='', nofollow=True),
-                    Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not',
+                    Link(url='http://example.com/nofollow2.html', text='Choose to follow or not',
                          fragment='', nofollow=False),
-                    Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+                    Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
                 ]
             )
 
@@ -436,13 +436,13 @@ class Base:
             self.assertEqual(
                 lx.extract_links(response),
                 [
-                    Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
-                    Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
-                    Link(url='http://example.com/nofollow.html', text=u'Dont follow this one',
+                    Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False),
+                    Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False),
+                    Link(url='http://example.com/nofollow.html', text='Dont follow this one',
                          fragment='', nofollow=True),
-                    Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not',
+                    Link(url='http://example.com/nofollow2.html', text='Choose to follow or not',
                          fragment='', nofollow=False),
-                    Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+                    Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
                 ]
             )
 
@@ -455,8 +455,8 @@ class Base:
             response = HtmlResponse("http://example.org/index.html", body=html)
             lx = self.extractor_cls()
             self.assertEqual([link for link in lx.extract_links(response)], [
-                Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
-                Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
+                Link(url='http://example.org/item1.html', text='Item 1', nofollow=False),
+                Link(url='http://example.org/item3.html', text='Item 3', nofollow=False),
             ])
 
         def test_ftp_links(self):
@@ -467,7 +467,7 @@ class Base:
             response = HtmlResponse("http://www.example.com/index.html", body=body, encoding='utf8')
             lx = self.extractor_cls()
             self.assertEqual(lx.extract_links(response), [
-                Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False),
+                Link(url='ftp://www.external.com/', text='An Item', fragment='', nofollow=False),
             ])
 
         def test_pickle_extractor(self):
@@ -487,8 +487,8 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
         response = HtmlResponse("http://example.org/index.html", body=html)
         lx = self.extractor_cls()
         self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
-            Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
+            Link(url='http://example.org/item1.html', text='Item 1', nofollow=False),
+            Link(url='http://example.org/item3.html', text='Item 3', nofollow=False),
         ])
 
     def test_link_restrict_text(self):
@@ -501,18 +501,18 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
         # Simple text inclusion test
         lx = self.extractor_cls(restrict_text='dog')
         self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
+            Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
         ])
         # Unique regex test
         lx = self.extractor_cls(restrict_text=r'of.*dog')
         self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
+            Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
         ])
         # Multiple regex test
         lx = self.extractor_cls(restrict_text=[r'of.*dog', r'of.*cat'])
         self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item1.html', text=u'Pic of a cat', nofollow=False),
-            Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
+            Link(url='http://example.org/item1.html', text='Pic of a cat', nofollow=False),
+            Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
         ])
 
     def test_restrict_xpaths_with_html_entities(self):
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 581183625..2ed6f365f 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -69,23 +69,23 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_add_value_on_unknown_field(self):
         il = TestItemLoader()
-        self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo'])
+        self.assertRaises(KeyError, il.add_value, 'wrong_field', ['lala', 'lolo'])
 
     def test_load_item_using_default_loader(self):
         i = TestItem()
-        i['summary'] = u'lala'
+        i['summary'] = 'lala'
         il = ItemLoader(item=i)
-        il.add_value('name', u'marta')
+        il.add_value('name', 'marta')
         item = il.load_item()
         assert item is i
-        self.assertEqual(item['summary'], [u'lala'])
-        self.assertEqual(item['name'], [u'marta'])
+        self.assertEqual(item['summary'], ['lala'])
+        self.assertEqual(item['name'], ['marta'])
 
     def test_load_item_using_custom_loader(self):
         il = TestItemLoader()
-        il.add_value('name', u'marta')
+        il.add_value('name', 'marta')
         item = il.load_item()
-        self.assertEqual(item['name'], [u'Marta'])
+        self.assertEqual(item['name'], ['Marta'])
 
 
 class InitializationTestMixin:
@@ -290,137 +290,137 @@ class SelectortemLoaderTest(unittest.TestCase):
         self.assertRaises(RuntimeError, l.get_css, '#name::text')
 
     def test_init_method_with_selector(self):
-        sel = Selector(text=u"<html><body><div>marta</div></body></html>")
+        sel = Selector(text="<html><body><div>marta</div></body></html>")
         l = TestItemLoader(selector=sel)
         self.assertIs(l.selector, sel)
 
         l.add_xpath('name', '//div/text()')
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
 
     def test_init_method_with_selector_css(self):
-        sel = Selector(text=u"<html><body><div>marta</div></body></html>")
+        sel = Selector(text="<html><body><div>marta</div></body></html>")
         l = TestItemLoader(selector=sel)
         self.assertIs(l.selector, sel)
 
         l.add_css('name', 'div::text')
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
 
     def test_init_method_with_response(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
 
         l.add_xpath('name', '//div/text()')
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
 
     def test_init_method_with_response_css(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
 
         l.add_css('name', 'div::text')
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
 
         l.add_css('url', 'a::attr(href)')
-        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
 
         # combining/accumulating CSS selectors and XPath expressions
         l.add_xpath('name', '//div/text()')
-        self.assertEqual(l.get_output_value('name'), [u'Marta', u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta', 'Marta'])
 
         l.add_xpath('url', '//img/@src')
-        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org', u'/images/logo.png'])
+        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org', '/images/logo.png'])
 
     def test_add_xpath_re(self):
         l = TestItemLoader(response=self.response)
         l.add_xpath('name', '//div/text()', re='ma')
-        self.assertEqual(l.get_output_value('name'), [u'Ma'])
+        self.assertEqual(l.get_output_value('name'), ['Ma'])
 
     def test_replace_xpath(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
         l.add_xpath('name', '//div/text()')
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
         l.replace_xpath('name', '//p/text()')
-        self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
+        self.assertEqual(l.get_output_value('name'), ['Paragraph'])
 
         l.replace_xpath('name', ['//p/text()', '//div/text()'])
-        self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta'])
 
     def test_get_xpath(self):
         l = TestItemLoader(response=self.response)
-        self.assertEqual(l.get_xpath('//p/text()'), [u'paragraph'])
-        self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), u'paragraph')
-        self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), u'pa')
+        self.assertEqual(l.get_xpath('//p/text()'), ['paragraph'])
+        self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), 'paragraph')
+        self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), 'pa')
 
-        self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), [u'paragraph', 'marta'])
+        self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), ['paragraph', 'marta'])
 
     def test_replace_xpath_multi_fields(self):
         l = TestItemLoader(response=self.response)
         l.add_xpath(None, '//div/text()', TakeFirst(), lambda x: {'name': x})
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
         l.replace_xpath(None, '//p/text()', TakeFirst(), lambda x: {'name': x})
-        self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
+        self.assertEqual(l.get_output_value('name'), ['Paragraph'])
 
     def test_replace_xpath_re(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
         l.add_xpath('name', '//div/text()')
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
         l.replace_xpath('name', '//div/text()', re='ma')
-        self.assertEqual(l.get_output_value('name'), [u'Ma'])
+        self.assertEqual(l.get_output_value('name'), ['Ma'])
 
     def test_add_css_re(self):
         l = TestItemLoader(response=self.response)
         l.add_css('name', 'div::text', re='ma')
-        self.assertEqual(l.get_output_value('name'), [u'Ma'])
+        self.assertEqual(l.get_output_value('name'), ['Ma'])
 
         l.add_css('url', 'a::attr(href)', re='http://(.+)')
-        self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['www.scrapy.org'])
 
     def test_replace_css(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
         l.add_css('name', 'div::text')
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
         l.replace_css('name', 'p::text')
-        self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
+        self.assertEqual(l.get_output_value('name'), ['Paragraph'])
 
         l.replace_css('name', ['p::text', 'div::text'])
-        self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta'])
 
         l.add_css('url', 'a::attr(href)', re='http://(.+)')
-        self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['www.scrapy.org'])
         l.replace_css('url', 'img::attr(src)')
-        self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])
+        self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
 
     def test_get_css(self):
         l = TestItemLoader(response=self.response)
-        self.assertEqual(l.get_css('p::text'), [u'paragraph'])
-        self.assertEqual(l.get_css('p::text', TakeFirst()), u'paragraph')
-        self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), u'pa')
+        self.assertEqual(l.get_css('p::text'), ['paragraph'])
+        self.assertEqual(l.get_css('p::text', TakeFirst()), 'paragraph')
+        self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), 'pa')
 
-        self.assertEqual(l.get_css(['p::text', 'div::text']), [u'paragraph', 'marta'])
+        self.assertEqual(l.get_css(['p::text', 'div::text']), ['paragraph', 'marta'])
         self.assertEqual(l.get_css(['a::attr(href)', 'img::attr(src)']),
-                         [u'http://www.scrapy.org', u'/images/logo.png'])
+                         ['http://www.scrapy.org', '/images/logo.png'])
 
     def test_replace_css_multi_fields(self):
         l = TestItemLoader(response=self.response)
         l.add_css(None, 'div::text', TakeFirst(), lambda x: {'name': x})
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
         l.replace_css(None, 'p::text', TakeFirst(), lambda x: {'name': x})
-        self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
+        self.assertEqual(l.get_output_value('name'), ['Paragraph'])
 
         l.add_css(None, 'a::attr(href)', TakeFirst(), lambda x: {'url': x})
-        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
         l.replace_css(None, 'img::attr(src)', TakeFirst(), lambda x: {'url': x})
-        self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])
+        self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
 
     def test_replace_css_re(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
         l.add_css('url', 'a::attr(href)')
-        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
         l.replace_css('url', 'a::attr(href)', re=r'http://www\.(.+)')
-        self.assertEqual(l.get_output_value('url'), [u'scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['scrapy.org'])
 
 
 class SubselectorLoaderTest(unittest.TestCase):
@@ -447,9 +447,9 @@ class SubselectorLoaderTest(unittest.TestCase):
         nl.add_css('name_div', '#id')
         nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
 
-        self.assertEqual(l.get_output_value('name'), [u'marta'])
-        self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
-        self.assertEqual(l.get_output_value('name_value'), [u'marta'])
+        self.assertEqual(l.get_output_value('name'), ['marta'])
+        self.assertEqual(l.get_output_value('name_div'), ['<div id="id">marta</div>'])
+        self.assertEqual(l.get_output_value('name_value'), ['marta'])
 
         self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
         self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
@@ -462,9 +462,9 @@ class SubselectorLoaderTest(unittest.TestCase):
         nl.add_css('name_div', '#id')
         nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
 
-        self.assertEqual(l.get_output_value('name'), [u'marta'])
-        self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
-        self.assertEqual(l.get_output_value('name_value'), [u'marta'])
+        self.assertEqual(l.get_output_value('name'), ['marta'])
+        self.assertEqual(l.get_output_value('name_div'), ['<div id="id">marta</div>'])
+        self.assertEqual(l.get_output_value('name_value'), ['marta'])
 
         self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
         self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
@@ -476,11 +476,11 @@ class SubselectorLoaderTest(unittest.TestCase):
         nl2 = nl1.nested_xpath('a')
 
         l.add_xpath('url', '//footer/a/@href')
-        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
         nl1.replace_xpath('url', 'img/@src')
-        self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])
+        self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
         nl2.replace_xpath('url', '@href')
-        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
 
     def test_nested_ordering(self):
         l = NestedItemLoader(response=self.response)
@@ -493,10 +493,10 @@ class SubselectorLoaderTest(unittest.TestCase):
         l.add_xpath('url', '//footer/a/@href')
 
         self.assertEqual(l.get_output_value('url'), [
-            u'/images/logo.png',
-            u'http://www.scrapy.org',
-            u'homepage',
-            u'http://www.scrapy.org',
+            '/images/logo.png',
+            'http://www.scrapy.org',
+            'homepage',
+            'http://www.scrapy.org',
         ])
 
     def test_nested_load_item(self):
@@ -514,9 +514,9 @@ class SubselectorLoaderTest(unittest.TestCase):
         assert item is nl1.item
         assert item is nl2.item
 
-        self.assertEqual(item['name'], [u'marta'])
-        self.assertEqual(item['url'], [u'http://www.scrapy.org'])
-        self.assertEqual(item['image'], [u'/images/logo.png'])
+        self.assertEqual(item['name'], ['marta'])
+        self.assertEqual(item['url'], ['http://www.scrapy.org'])
+        self.assertEqual(item['image'], ['/images/logo.png'])
 
 
 # Functions as processors
diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py
index d0a59e8cd..eb14de14f 100644
--- a/tests/test_loader_deprecated.py
+++ b/tests/test_loader_deprecated.py
@@ -51,19 +51,19 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_load_item_using_default_loader(self):
         i = TestItem()
-        i['summary'] = u'lala'
+        i['summary'] = 'lala'
         il = ItemLoader(item=i)
-        il.add_value('name', u'marta')
+        il.add_value('name', 'marta')
         item = il.load_item()
         assert item is i
-        self.assertEqual(item['summary'], [u'lala'])
-        self.assertEqual(item['name'], [u'marta'])
+        self.assertEqual(item['summary'], ['lala'])
+        self.assertEqual(item['name'], ['marta'])
 
     def test_load_item_using_custom_loader(self):
         il = TestItemLoader()
-        il.add_value('name', u'marta')
+        il.add_value('name', 'marta')
         item = il.load_item()
-        self.assertEqual(item['name'], [u'Marta'])
+        self.assertEqual(item['name'], ['Marta'])
 
     def test_load_item_ignore_none_field_values(self):
         def validate_sku(value):
@@ -76,23 +76,23 @@ class BasicItemLoaderTest(unittest.TestCase):
             price_out = Compose(TakeFirst(), float)
             sku_out = Compose(TakeFirst(), validate_sku)
 
-        valid_fragment = u'SKU: 1234'
-        invalid_fragment = u'SKU: not available'
+        valid_fragment = 'SKU: 1234'
+        invalid_fragment = 'SKU: not available'
         sku_re = 'SKU: (.+)'
 
         il = MyLoader(item={})
         # Should not return "sku: None".
         il.add_value('sku', [invalid_fragment], re=sku_re)
         # Should not ignore empty values.
-        il.add_value('name', u'')
-        il.add_value('price', [u'0'])
+        il.add_value('name', '')
+        il.add_value('price', ['0'])
         self.assertEqual(il.load_item(), {
-            'name': u'',
+            'name': '',
             'price': 0.0,
         })
 
         il.replace_value('sku', [valid_fragment], re=sku_re)
-        self.assertEqual(il.load_item()['sku'], u'1234')
+        self.assertEqual(il.load_item()['sku'], '1234')
 
     def test_self_referencing_loader(self):
         class MyLoader(ItemLoader):
@@ -117,19 +117,19 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_add_value(self):
         il = TestItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'Marta'])
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
-        il.add_value('name', u'pepe')
-        self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe'])
-        self.assertEqual(il.get_output_value('name'), [u'Marta', u'Pepe'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_collected_values('name'), ['Marta'])
+        self.assertEqual(il.get_output_value('name'), ['Marta'])
+        il.add_value('name', 'pepe')
+        self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe'])
+        self.assertEqual(il.get_output_value('name'), ['Marta', 'Pepe'])
 
         # test add object value
         il.add_value('summary', {'key': 1})
         self.assertEqual(il.get_collected_values('summary'), [{'key': 1}])
 
-        il.add_value(None, u'Jim', lambda x: {'name': x})
-        self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe', u'Jim'])
+        il.add_value(None, 'Jim', lambda x: {'name': x})
+        self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe', 'Jim'])
 
     def test_add_zero(self):
         il = NameItemLoader()
@@ -138,49 +138,49 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_replace_value(self):
         il = TestItemLoader()
-        il.replace_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'Marta'])
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
-        il.replace_value('name', u'pepe')
-        self.assertEqual(il.get_collected_values('name'), [u'Pepe'])
-        self.assertEqual(il.get_output_value('name'), [u'Pepe'])
+        il.replace_value('name', 'marta')
+        self.assertEqual(il.get_collected_values('name'), ['Marta'])
+        self.assertEqual(il.get_output_value('name'), ['Marta'])
+        il.replace_value('name', 'pepe')
+        self.assertEqual(il.get_collected_values('name'), ['Pepe'])
+        self.assertEqual(il.get_output_value('name'), ['Pepe'])
 
-        il.replace_value(None, u'Jim', lambda x: {'name': x})
-        self.assertEqual(il.get_collected_values('name'), [u'Jim'])
+        il.replace_value(None, 'Jim', lambda x: {'name': x})
+        self.assertEqual(il.get_collected_values('name'), ['Jim'])
 
     def test_get_value(self):
         il = NameItemLoader()
-        self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper))
-        self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$'))
-        self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$'))
+        self.assertEqual('FOO', il.get_value(['foo', 'bar'], TakeFirst(), str.upper))
+        self.assertEqual(['foo', 'bar'], il.get_value(['name:foo', 'name:bar'], re='name:(.*)$'))
+        self.assertEqual('foo', il.get_value(['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$'))
 
-        il.add_value('name', [u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')
-        self.assertEqual([u'foo'], il.get_collected_values('name'))
-        il.replace_value('name', u'name:bar', re=u'name:(.*)$')
-        self.assertEqual([u'bar'], il.get_collected_values('name'))
+        il.add_value('name', ['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$')
+        self.assertEqual(['foo'], il.get_collected_values('name'))
+        il.replace_value('name', 'name:bar', re='name:(.*)$')
+        self.assertEqual(['bar'], il.get_collected_values('name'))
 
     def test_iter_on_input_processor_input(self):
         class NameFirstItemLoader(NameItemLoader):
             name_in = TakeFirst()
 
         il = NameFirstItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_collected_values('name'), ['marta'])
         il = NameFirstItemLoader()
-        il.add_value('name', [u'marta', u'jose'])
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+        il.add_value('name', ['marta', 'jose'])
+        self.assertEqual(il.get_collected_values('name'), ['marta'])
 
         il = NameFirstItemLoader()
-        il.replace_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+        il.replace_value('name', 'marta')
+        self.assertEqual(il.get_collected_values('name'), ['marta'])
         il = NameFirstItemLoader()
-        il.replace_value('name', [u'marta', u'jose'])
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+        il.replace_value('name', ['marta', 'jose'])
+        self.assertEqual(il.get_collected_values('name'), ['marta'])
 
         il = NameFirstItemLoader()
-        il.add_value('name', u'marta')
-        il.add_value('name', [u'jose', u'pedro'])
-        self.assertEqual(il.get_collected_values('name'), [u'marta', u'jose'])
+        il.add_value('name', 'marta')
+        il.add_value('name', ['jose', 'pedro'])
+        self.assertEqual(il.get_collected_values('name'), ['marta', 'jose'])
 
     def test_map_compose_filter(self):
         def filter_world(x):
@@ -195,87 +195,87 @@ class BasicItemLoaderTest(unittest.TestCase):
             name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1])
 
         il = TestItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'Mart'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['Mart'])
         item = il.load_item()
-        self.assertEqual(item['name'], [u'Mart'])
+        self.assertEqual(item['name'], ['Mart'])
 
     def test_default_input_processor(self):
         il = DefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'mart'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['mart'])
 
     def test_inherited_default_input_processor(self):
         class InheritDefaultedItemLoader(DefaultedItemLoader):
             pass
 
         il = InheritDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'mart'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['mart'])
 
     def test_input_processor_inheritance(self):
         class ChildItemLoader(TestItemLoader):
             url_in = MapCompose(lambda v: v.lower())
 
         il = ChildItemLoader()
-        il.add_value('url', u'HTTP://scrapy.ORG')
-        self.assertEqual(il.get_output_value('url'), [u'http://scrapy.org'])
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
+        il.add_value('url', 'HTTP://scrapy.ORG')
+        self.assertEqual(il.get_output_value('url'), ['http://scrapy.org'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['Marta'])
 
         class ChildChildItemLoader(ChildItemLoader):
             url_in = MapCompose(lambda v: v.upper())
             summary_in = MapCompose(lambda v: v)
 
         il = ChildChildItemLoader()
-        il.add_value('url', u'http://scrapy.org')
-        self.assertEqual(il.get_output_value('url'), [u'HTTP://SCRAPY.ORG'])
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
+        il.add_value('url', 'http://scrapy.org')
+        self.assertEqual(il.get_output_value('url'), ['HTTP://SCRAPY.ORG'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['Marta'])
 
     def test_empty_map_compose(self):
         class IdentityDefaultedItemLoader(DefaultedItemLoader):
             name_in = MapCompose()
 
         il = IdentityDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'marta'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['marta'])
 
     def test_identity_input_processor(self):
         class IdentityDefaultedItemLoader(DefaultedItemLoader):
             name_in = Identity()
 
         il = IdentityDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'marta'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['marta'])
 
     def test_extend_custom_input_processors(self):
         class ChildItemLoader(TestItemLoader):
             name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
 
         il = ChildItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'mARTA'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['mARTA'])
 
     def test_extend_default_input_processors(self):
         class ChildDefaultedItemLoader(DefaultedItemLoader):
             name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase)
 
         il = ChildDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'MART'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['MART'])
 
     def test_output_processor_using_function(self):
         il = TestItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+        il.add_value('name', ['mar', 'ta'])
+        self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
 
         class TakeFirstItemLoader(TestItemLoader):
-            name_out = u" ".join
+            name_out = " ".join
 
         il = TakeFirstItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), u'Mar Ta')
+        il.add_value('name', ['mar', 'ta'])
+        self.assertEqual(il.get_output_value('name'), 'Mar Ta')
 
     def test_output_processor_error(self):
         class TestItemLoader(ItemLoader):
@@ -283,9 +283,9 @@ class BasicItemLoaderTest(unittest.TestCase):
             name_out = MapCompose(float)
 
         il = TestItemLoader()
-        il.add_value('name', [u'$10'])
+        il.add_value('name', ['$10'])
         try:
-            float(u'$10')
+            float('$10')
         except Exception as e:
             expected_exc_str = str(e)
 
@@ -303,53 +303,53 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_output_processor_using_classes(self):
         il = TestItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+        il.add_value('name', ['mar', 'ta'])
+        self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
 
         class TakeFirstItemLoader(TestItemLoader):
             name_out = Join()
 
         il = TakeFirstItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), u'Mar Ta')
+        il.add_value('name', ['mar', 'ta'])
+        self.assertEqual(il.get_output_value('name'), 'Mar Ta')
 
         class TakeFirstItemLoader(TestItemLoader):
             name_out = Join("<br>")
 
         il = TakeFirstItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), u'Mar<br>Ta')
+        il.add_value('name', ['mar', 'ta'])
+        self.assertEqual(il.get_output_value('name'), 'Mar<br>Ta')
 
     def test_default_output_processor(self):
         il = TestItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+        il.add_value('name', ['mar', 'ta'])
+        self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
 
         class LalaItemLoader(TestItemLoader):
             default_output_processor = Identity()
 
         il = LalaItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+        il.add_value('name', ['mar', 'ta'])
+        self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
 
     def test_loader_context_on_declaration(self):
         class ChildItemLoader(TestItemLoader):
-            url_in = MapCompose(processor_with_args, key=u'val')
+            url_in = MapCompose(processor_with_args, key='val')
 
         il = ChildItemLoader()
-        il.add_value('url', u'text')
+        il.add_value('url', 'text')
         self.assertEqual(il.get_output_value('url'), ['val'])
-        il.replace_value('url', u'text2')
+        il.replace_value('url', 'text2')
         self.assertEqual(il.get_output_value('url'), ['val'])
 
     def test_loader_context_on_instantiation(self):
         class ChildItemLoader(TestItemLoader):
             url_in = MapCompose(processor_with_args)
 
-        il = ChildItemLoader(key=u'val')
-        il.add_value('url', u'text')
+        il = ChildItemLoader(key='val')
+        il.add_value('url', 'text')
         self.assertEqual(il.get_output_value('url'), ['val'])
-        il.replace_value('url', u'text2')
+        il.replace_value('url', 'text2')
         self.assertEqual(il.get_output_value('url'), ['val'])
 
     def test_loader_context_on_assign(self):
@@ -357,10 +357,10 @@ class BasicItemLoaderTest(unittest.TestCase):
             url_in = MapCompose(processor_with_args)
 
         il = ChildItemLoader()
-        il.context['key'] = u'val'
-        il.add_value('url', u'text')
+        il.context['key'] = 'val'
+        il.add_value('url', 'text')
         self.assertEqual(il.get_output_value('url'), ['val'])
-        il.replace_value('url', u'text2')
+        il.replace_value('url', 'text2')
         self.assertEqual(il.get_output_value('url'), ['val'])
 
     def test_item_passed_to_input_processor_functions(self):
@@ -372,9 +372,9 @@ class BasicItemLoaderTest(unittest.TestCase):
 
         it = TestItem(name='marta')
         il = ChildItemLoader(item=it)
-        il.add_value('url', u'text')
+        il.add_value('url', 'text')
         self.assertEqual(il.get_output_value('url'), ['marta'])
-        il.replace_value('url', u'text2')
+        il.replace_value('url', 'text2')
         self.assertEqual(il.get_output_value('url'), ['marta'])
 
     def test_compose_processor(self):
@@ -382,10 +382,10 @@ class BasicItemLoaderTest(unittest.TestCase):
             name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1])
 
         il = TestItemLoader()
-        il.add_value('name', [u'marta', u'other'])
-        self.assertEqual(il.get_output_value('name'), u'Mart')
+        il.add_value('name', ['marta', 'other'])
+        self.assertEqual(il.get_output_value('name'), 'Mart')
         item = il.load_item()
-        self.assertEqual(item['name'], u'Mart')
+        self.assertEqual(item['name'], 'Mart')
 
     def test_partial_processor(self):
         def join(values, sep=None, loader_context=None, ignored=None):
@@ -402,13 +402,13 @@ class BasicItemLoaderTest(unittest.TestCase):
             summary_out = Compose(partial(join, ignored='foo'))
 
         il = TestItemLoader()
-        il.add_value('name', [u'rabbit', u'hole'])
-        il.add_value('url', [u'rabbit', u'hole'])
-        il.add_value('summary', [u'rabbit', u'hole'])
+        il.add_value('name', ['rabbit', 'hole'])
+        il.add_value('url', ['rabbit', 'hole'])
+        il.add_value('summary', ['rabbit', 'hole'])
         item = il.load_item()
-        self.assertEqual(item['name'], u'rabbit+hole')
-        self.assertEqual(item['url'], u'rabbit.hole')
-        self.assertEqual(item['summary'], u'rabbithole')
+        self.assertEqual(item['name'], 'rabbit+hole')
+        self.assertEqual(item['url'], 'rabbit.hole')
+        self.assertEqual(item['summary'], 'rabbithole')
 
     def test_error_input_processor(self):
         class TestItem(Item):
@@ -420,7 +420,7 @@ class BasicItemLoaderTest(unittest.TestCase):
 
         il = TestItemLoader()
         self.assertRaises(ValueError, il.add_value, 'name',
-                          [u'marta', u'other'])
+                          ['marta', 'other'])
 
     def test_error_output_processor(self):
         class TestItem(Item):
@@ -431,7 +431,7 @@ class BasicItemLoaderTest(unittest.TestCase):
             name_out = Compose(Join(), float)
 
         il = TestItemLoader()
-        il.add_value('name', u'marta')
+        il.add_value('name', 'marta')
         with self.assertRaises(ValueError):
             il.load_item()
 
@@ -444,7 +444,7 @@ class BasicItemLoaderTest(unittest.TestCase):
 
         il = TestItemLoader()
         self.assertRaises(ValueError, il.add_value, 'name',
-                          [u'marta', u'other'], Compose(float))
+                          ['marta', 'other'], Compose(float))
 
 
 class InitializationFromDictTest(unittest.TestCase):
@@ -608,8 +608,8 @@ class ProcessorsTest(unittest.TestCase):
     def test_join(self):
         proc = Join()
         self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
-        self.assertEqual(proc(['', 'hello', 'world']), u' hello world')
-        self.assertEqual(proc(['hello', 'world']), u'hello world')
+        self.assertEqual(proc(['', 'hello', 'world']), ' hello world')
+        self.assertEqual(proc(['hello', 'world']), 'hello world')
         self.assertIsInstance(proc(['hello', 'world']), str)
 
     def test_compose(self):
@@ -626,8 +626,8 @@ class ProcessorsTest(unittest.TestCase):
         def filter_world(x):
             return None if x == 'world' else x
         proc = MapCompose(filter_world, str.upper)
-        self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
-                         [u'HELLO', u'THIS', u'IS', u'SCRAPY'])
+        self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']),
+                         ['HELLO', 'THIS', 'IS', 'SCRAPY'])
         proc = MapCompose(filter_world, str.upper)
         self.assertEqual(proc(None), [])
         proc = MapCompose(filter_world, str.upper)
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 7064337ad..b771e7d79 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -56,13 +56,13 @@ class LogFormatterTestCase(unittest.TestCase):
 
     def test_dropped(self):
         item = {}
-        exception = Exception(u"\u2018")
+        exception = Exception("\u2018")
         response = Response("http://www.example.com")
         logkws = self.formatter.dropped(item, exception, response, self.spider)
         logline = logkws['msg'] % logkws['args']
         lines = logline.splitlines()
         assert all(isinstance(x, str) for x in lines)
-        self.assertEqual(lines, [u"Dropped: \u2018", '{}'])
+        self.assertEqual(lines, ["Dropped: \u2018", '{}'])
 
     def test_item_error(self):
         # In practice, the complete traceback is shown by passing the
@@ -72,7 +72,7 @@ class LogFormatterTestCase(unittest.TestCase):
         response = Response("http://www.example.com")
         logkws = self.formatter.item_error(item, exception, response, self.spider)
         logline = logkws['msg'] % logkws['args']
-        self.assertEqual(logline, u"Error processing {'key': 'value'}")
+        self.assertEqual(logline, "Error processing {'key': 'value'}")
 
     def test_spider_error(self):
         # In practice, the complete traceback is shown by passing the
@@ -107,20 +107,20 @@ class LogFormatterTestCase(unittest.TestCase):
 
     def test_scraped(self):
         item = CustomItem()
-        item['name'] = u'\xa3'
+        item['name'] = '\xa3'
         response = Response("http://www.example.com")
         logkws = self.formatter.scraped(item, response, self.spider)
         logline = logkws['msg'] % logkws['args']
         lines = logline.splitlines()
         assert all(isinstance(x, str) for x in lines)
-        self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3'])
+        self.assertEqual(lines, ["Scraped from <200 http://www.example.com>", 'name: \xa3'])
 
 
 class LogFormatterSubclass(LogFormatter):
     def crawled(self, request, response, spider):
         kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider)
         CRAWLEDMSG = (
-            u"Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
+            "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
         )
         log_args = kwargs['args']
         log_args['flags'] = str(request.flags)
diff --git a/tests/test_mail.py b/tests/test_mail.py
index 53dbc0686..9b248fbfa 100644
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@ -73,8 +73,8 @@ class MailSenderTest(unittest.TestCase):
         self.catched_msg = dict(**kwargs)
 
     def test_send_utf8(self):
-        subject = u'sübjèçt'
-        body = u'bödÿ-àéïöñß'
+        subject = 'sübjèçt'
+        body = 'bödÿ-àéïöñß'
         mailsender = MailSender(debug=True)
         mailsender.send(to=['test@scrapy.org'], subject=subject, body=body,
                         charset='utf-8', _callback=self._catch_mail_sent)
@@ -90,8 +90,8 @@ class MailSenderTest(unittest.TestCase):
         self.assertEqual(msg.get('Content-Type'), 'text/plain; charset="utf-8"')
 
     def test_send_attach_utf8(self):
-        subject = u'sübjèçt'
-        body = u'bödÿ-àéïöñß'
+        subject = 'sübjèçt'
+        body = 'bödÿ-àéïöñß'
         attach = BytesIO()
         attach.write(body.encode('utf-8'))
         attach.seek(0)
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index dd19a69d5..a175f88ca 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -23,11 +23,11 @@ class ResponseTypesTest(unittest.TestCase):
         mappings = [
             (b'attachment; filename="data.xml"', XmlResponse),
             (b'attachment; filename=data.xml', XmlResponse),
-            (u'attachment;filename=data£.tar.gz'.encode('utf-8'), Response),
-            (u'attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response),
-            (u'attachment;filename=data高.doc'.encode('gbk'), Response),
-            (u'attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse),
-            (u'attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse),
+            ('attachment;filename=data£.tar.gz'.encode('utf-8'), Response),
+            ('attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response),
+            ('attachment;filename=data高.doc'.encode('gbk'), Response),
+            ('attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse),
+            ('attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse),
 
         ]
         for source, cls in mappings:
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 24aaaf7ec..9d8c201dd 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -93,7 +93,7 @@ class BaseRobotParserTest:
         self.assertTrue(rp.allowed("https://site.local/disallowed", "*"))
 
     def test_unicode_url_and_useragent(self):
-        robotstxt_robotstxt_body = u"""
+        robotstxt_robotstxt_body = """
         User-Agent: *
         Disallow: /admin/
         Disallow: /static/
@@ -107,11 +107,11 @@ class BaseRobotParserTest:
         self.assertTrue(rp.allowed("https://site.local/", "*"))
         self.assertFalse(rp.allowed("https://site.local/admin/", "*"))
         self.assertFalse(rp.allowed("https://site.local/static/", "*"))
-        self.assertTrue(rp.allowed("https://site.local/admin/", u"UnicödeBöt"))
+        self.assertTrue(rp.allowed("https://site.local/admin/", "UnicödeBöt"))
         self.assertFalse(rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*"))
-        self.assertFalse(rp.allowed(u"https://site.local/wiki/Käyttäjä:", "*"))
+        self.assertFalse(rp.allowed("https://site.local/wiki/Käyttäjä:", "*"))
         self.assertTrue(rp.allowed("https://site.local/some/randome/page.html", "*"))
-        self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", u"UnicödeBöt"))
+        self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt"))
 
 
 class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 00e663c11..62036ad8c 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -25,19 +25,19 @@ class SelectorTestCase(unittest.TestCase):
         )
         self.assertEqual(
             [x.get() for x in sel.xpath("//input[@name='a']/@name")],
-            [u'a']
+            ['a']
         )
         self.assertEqual(
             [x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
-            [u'12.0']
+            ['12.0']
         )
         self.assertEqual(
             sel.xpath("concat('xpath', 'rules')").getall(),
-            [u'xpathrules']
+            ['xpathrules']
         )
         self.assertEqual(
             [x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
-            [u'12']
+            ['12']
         )
 
     def test_root_base_url(self):
@@ -52,30 +52,30 @@ class SelectorTestCase(unittest.TestCase):
         sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))
         self.assertEqual(sel.type, 'xml')
         self.assertEqual(sel.xpath("//div").getall(),
-                         [u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
+                         ['<div><img src="a.jpg"><p>Hello</p></img></div>'])
 
         sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8'))
         self.assertEqual(sel.type, 'html')
         self.assertEqual(sel.xpath("//div").getall(),
-                         [u'<div><img src="a.jpg"><p>Hello</p></div>'])
+                         ['<div><img src="a.jpg"><p>Hello</p></div>'])
 
     def test_http_header_encoding_precedence(self):
-        # u'\xa3'     = pound symbol in unicode
-        # u'\xc2\xa3' = pound symbol in utf-8
-        # u'\xa3'     = pound symbol in latin-1 (iso-8859-1)
+        # '\xa3'     = pound symbol in unicode
+        # '\xc2\xa3' = pound symbol in utf-8
+        # '\xa3'     = pound symbol in latin-1 (iso-8859-1)
 
-        meta = u'<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">'
-        head = u'<head>' + meta + u'</head>'
-        body_content = u'<span id="blank">\xa3</span>'
-        body = u'<body>' + body_content + u'</body>'
-        html = u'<html>' + head + body + u'</html>'
+        meta = '<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">'
+        head = '<head>' + meta + '</head>'
+        body_content = '<span id="blank">\xa3</span>'
+        body = '<body>' + body_content + '</body>'
+        html = '<html>' + head + body + '</html>'
         encoding = 'utf-8'
         html_utf8 = html.encode(encoding)
 
         headers = {'Content-Type': ['text/html; charset=utf-8']}
         response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
         x = Selector(response)
-        self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3'])
+        self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), ['\xa3'])
 
     def test_badly_encoded_body(self):
         # \xe9 alone isn't valid utf8 sequence
@@ -92,4 +92,4 @@ class SelectorTestCase(unittest.TestCase):
 
     def test_selector_bad_args(self):
         with self.assertRaisesRegex(ValueError, 'received both response and text'):
-            Selector(TextResponse(url='http://example.com', body=b''), text=u'')
+            Selector(TextResponse(url='http://example.com', body=b''), text='')
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 83c10a3c3..78157a9b9 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -153,13 +153,13 @@ class XMLFeedSpiderTest(SpiderTest):
             output = list(spider._parse(response))
             self.assertEqual(len(output), 2, iterator)
             self.assertEqual(output, [
-                {'loc': [u'http://www.example.com/Special-Offers.html'],
-                 'updated': [u'2009-08-16'],
-                 'custom': [u'fuu'],
-                 'other': [u'bar']},
+                {'loc': ['http://www.example.com/Special-Offers.html'],
+                 'updated': ['2009-08-16'],
+                 'custom': ['fuu'],
+                 'other': ['bar']},
                 {'loc': [],
-                 'updated': [u'2009-08-16'],
-                 'other': [u'foo'],
+                 'updated': ['2009-08-16'],
+                 'other': ['foo'],
                  'custom': []},
             ], iterator)
 
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index d17bb2cbc..298178f08 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -54,7 +54,7 @@ class XmliterTestCase(unittest.TestCase):
 
     def test_xmliter_unicode(self):
         # example taken from https://github.com/scrapy/scrapy/issues/1665
-        body = u"""<?xml version="1.0" encoding="UTF-8"?>
+        body = """<?xml version="1.0" encoding="UTF-8"?>
             <þingflokkar>
                <þingflokkur id="26">
                   <heiti />
@@ -97,15 +97,15 @@ class XmliterTestCase(unittest.TestCase):
             XmlResponse(url="http://example.com", body=body, encoding='utf-8'),
         ):
             attrs = []
-            for x in self.xmliter(r, u'þingflokkur'):
+            for x in self.xmliter(r, 'þingflokkur'):
                 attrs.append((x.attrib['id'],
-                              x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(),
-                              x.xpath(u'./tímabil/fyrstaþing/text()').getall()))
+                              x.xpath('./skammstafanir/stuttskammstöfun/text()').getall(),
+                              x.xpath('./tímabil/fyrstaþing/text()').getall()))
 
             self.assertEqual(attrs,
-                             [(u'26', [u'-'], [u'80']),
-                              (u'21', [u'Ab'], [u'76']),
-                              (u'27', [u'A'], [u'27'])])
+                             [('26', ['-'], ['80']),
+                              ('21', ['Ab'], ['76']),
+                              ('27', ['A'], ['27'])])
 
     def test_xmliter_text(self):
         body = (
@@ -114,7 +114,7 @@ class XmliterTestCase(unittest.TestCase):
         )
 
         self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')],
-                         [[u'one'], [u'two']])
+                         [['one'], ['two']])
 
     def test_xmliter_namespaces(self):
         body = b"""
@@ -179,7 +179,7 @@ class XmliterTestCase(unittest.TestCase):
         response = XmlResponse('http://www.example.com', body=body)
         self.assertEqual(
             next(self.xmliter(response, 'item')).get(),
-            u'<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'
+            '<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'
         )
 
 
@@ -265,10 +265,10 @@ class UtilsCsvTestCase(unittest.TestCase):
 
         result = [row for row in csv]
         self.assertEqual(result,
-                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
-                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty', u'value': u''}])
+                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '4', 'name': 'empty', 'value': ''}])
 
         # explicit type check cuz' we no like stinkin' autocasting! yarrr
         for result_row in result:
@@ -281,10 +281,10 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv = csviter(response, delimiter='\t')
 
         self.assertEqual([row for row in csv],
-                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
-                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty', u'value': u''}])
+                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '4', 'name': 'empty', 'value': ''}])
 
     def test_csviter_quotechar(self):
         body1 = get_testdata('feeds', 'feed-sample6.csv')
@@ -294,19 +294,19 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv1 = csviter(response1, quotechar="'")
 
         self.assertEqual([row for row in csv1],
-                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
-                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty', u'value': u''}])
+                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '4', 'name': 'empty', 'value': ''}])
 
         response2 = TextResponse(url="http://example.com/", body=body2)
         csv2 = csviter(response2, delimiter="|", quotechar="'")
 
         self.assertEqual([row for row in csv2],
-                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
-                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty', u'value': u''}])
+                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '4', 'name': 'empty', 'value': ''}])
 
     def test_csviter_wrong_quotechar(self):
         body = get_testdata('feeds', 'feed-sample6.csv')
@@ -314,10 +314,10 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv = csviter(response)
 
         self.assertEqual([row for row in csv],
-                         [{u"'id'": u"1", u"'name'": u"'alpha'", u"'value'": u"'foobar'"},
-                          {u"'id'": u"2", u"'name'": u"'unicode'", u"'value'": u"'\xfan\xedc\xf3d\xe9\u203d'"},
-                          {u"'id'": u"'3'", u"'name'": u"'multi'", u"'value'": u"'foo"},
-                          {u"'id'": u"4", u"'name'": u"'empty'", u"'value'": u""}])
+                         [{"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"},
+                          {"'id'": "2", "'name'": "'unicode'", "'value'": "'\xfan\xedc\xf3d\xe9\u203d'"},
+                          {"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"},
+                          {"'id'": "4", "'name'": "'empty'", "'value'": ""}])
 
     def test_csviter_delimiter_binary_response_assume_utf8_encoding(self):
         body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
@@ -325,10 +325,10 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv = csviter(response, delimiter='\t')
 
         self.assertEqual([row for row in csv],
-                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
-                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty', u'value': u''}])
+                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '4', 'name': 'empty', 'value': ''}])
 
     def test_csviter_headers(self):
         sample = get_testdata('feeds', 'feed-sample3.csv').splitlines()
@@ -338,10 +338,10 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv = csviter(response, headers=[h.decode('utf-8') for h in headers])
 
         self.assertEqual([row for row in csv],
-                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
-                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'},
-                          {u'id': u'4', u'name': u'empty', u'value': u''}])
+                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+                          {'id': '3', 'name': 'multi', 'value': 'foo\nbar'},
+                          {'id': '4', 'name': 'empty', 'value': ''}])
 
     def test_csviter_falserow(self):
         body = get_testdata('feeds', 'feed-sample3.csv')
@@ -351,10 +351,10 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv = csviter(response)
 
         self.assertEqual([row for row in csv],
-                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
-                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty', u'value': u''}])
+                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '4', 'name': 'empty', 'value': ''}])
 
     def test_csviter_exception(self):
         body = get_testdata('feeds', 'feed-sample3.csv')
@@ -377,8 +377,8 @@ class UtilsCsvTestCase(unittest.TestCase):
         self.assertEqual(
             list(csv),
             [
-                {u'id': u'1', u'name': u'latin1', u'value': u'test'},
-                {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'},
+                {'id': '1', 'name': 'latin1', 'value': 'test'},
+                {'id': '2', 'name': 'something', 'value': '\xf1\xe1\xe9\xf3'},
             ]
         )
 
@@ -387,8 +387,8 @@ class UtilsCsvTestCase(unittest.TestCase):
         self.assertEqual(
             list(csv),
             [
-                {u'id': u'1', u'name': u'cp852', u'value': u'test'},
-                {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'},
+                {'id': '1', 'name': 'cp852', 'value': 'test'},
+                {'id': '2', 'name': 'something', 'value': '\u255a\u2569\u2569\u2569\u2550\u2550\u2557'},
             ]
         )
 
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index ebce3c079..3f93f509e 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -34,13 +34,13 @@ class MutableChainTest(unittest.TestCase):
 
 class ToUnicodeTest(unittest.TestCase):
     def test_converting_an_utf8_encoded_string_to_unicode(self):
-        self.assertEqual(to_unicode(b'lel\xc3\xb1e'), u'lel\xf1e')
+        self.assertEqual(to_unicode(b'lel\xc3\xb1e'), 'lel\xf1e')
 
     def test_converting_a_latin_1_encoded_string_to_unicode(self):
-        self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), u'lel\xf1e')
+        self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), 'lel\xf1e')
 
     def test_converting_a_unicode_to_unicode_should_return_the_same_object(self):
-        self.assertEqual(to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e')
+        self.assertEqual(to_unicode('\xf1e\xf1e\xf1e'), '\xf1e\xf1e\xf1e')
 
     def test_converting_a_strange_object_should_raise_TypeError(self):
         self.assertRaises(TypeError, to_unicode, 423)
@@ -48,16 +48,16 @@ class ToUnicodeTest(unittest.TestCase):
     def test_errors_argument(self):
         self.assertEqual(
             to_unicode(b'a\xedb', 'utf-8', errors='replace'),
-            u'a\ufffdb'
+            'a\ufffdb'
         )
 
 
 class ToBytesTest(unittest.TestCase):
     def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self):
-        self.assertEqual(to_bytes(u'\xa3 49'), b'\xc2\xa3 49')
+        self.assertEqual(to_bytes('\xa3 49'), b'\xc2\xa3 49')
 
     def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self):
-        self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), b'\xa3 49')
+        self.assertEqual(to_bytes('\xa3 49', 'latin-1'), b'\xa3 49')
 
     def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self):
         self.assertEqual(to_bytes(b'lel\xf1e'), b'lel\xf1e')
@@ -67,7 +67,7 @@ class ToBytesTest(unittest.TestCase):
 
     def test_errors_argument(self):
         self.assertEqual(
-            to_bytes(u'a\ufffdb', 'latin-1', errors='replace'),
+            to_bytes('a\ufffdb', 'latin-1', errors='replace'),
             b'a?b'
         )
 
@@ -96,7 +96,7 @@ class BinaryIsTextTest(unittest.TestCase):
         assert binary_is_text(b"hello")
 
     def test_utf_16_strings_contain_null_bytes(self):
-        assert binary_is_text(u"hello".encode('utf-16'))
+        assert binary_is_text("hello".encode('utf-16'))
 
     def test_one_with_encoding(self):
         assert binary_is_text(b"<div>Price \xa3</div>")
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 450e4bdca..de94ec960 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -22,7 +22,7 @@ class RequestSerializationTest(unittest.TestCase):
             method="POST",
             body=b"some body",
             headers={'content-encoding': 'text/html; charset=latin-1'},
-            cookies={'currency': u'руб'},
+            cookies={'currency': 'руб'},
             encoding='latin-1',
             priority=20,
             meta={'a': 'b'},
diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py
index 5a52dd695..5ff2e41ef 100644
--- a/tests/test_utils_template.py
+++ b/tests/test_utils_template.py
@@ -19,8 +19,8 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase):
     def test_simple_render(self):
 
         context = dict(project_name='proj', name='spi', classname='TheSpider')
-        template = u'from ${project_name}.spiders.${name} import ${classname}'
-        rendered = u'from proj.spiders.spi import TheSpider'
+        template = 'from ${project_name}.spiders.${name} import ${classname}'
+        rendered = 'from proj.spiders.spi import TheSpider'
 
         template_path = os.path.join(self.tmp_path, 'templ.py.tmpl')
         render_path = os.path.join(self.tmp_path, 'templ.py')

From 3600582f56071dd9f40f31ed44e09294a78dc13f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 4 Aug 2020 20:05:56 +0200
Subject: [PATCH 50/57] Cover Scrapy 2.2.1 and 2.3 in the release notes (#4708)

---
 docs/news.rst                   | 133 ++++++++++++++++++++++++++++++++
 docs/topics/commands.rst        |   2 +
 docs/topics/developer-tools.rst |   6 +-
 docs/topics/feed-exports.rst    |  36 ++++++++-
 scrapy/utils/curl.py            |   3 +-
 5 files changed, 176 insertions(+), 4 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 80d130e4a..850b323ef 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,139 @@
 Release notes
 =============
 
+.. _release-2.3.0:
+
+Scrapy 2.3.0 (2020-08-04)
+-------------------------
+
+Highlights:
+
+*   :ref:`Feed exports <topics-feed-exports>` now support :ref:`Google Cloud
+    Storage <topics-feed-storage-gcs>` as a storage backend
+
+*   The new :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting allows to deliver
+    output items in batches of up to the specified number of items.
+
+    It also serves as a workaround for :ref:`delayed file delivery
+    <delayed-file-delivery>`, which causes Scrapy to only start item delivery
+    after the crawl has finished when using certain storage backends
+    (:ref:`S3 <topics-feed-storage-s3>`, :ref:`FTP <topics-feed-storage-ftp>`,
+    and now :ref:`GCS <topics-feed-storage-gcs>`).
+
+*   The base implementation of :ref:`item loaders <topics-loaders>` has been
+    moved into a separate library, :doc:`itemloaders <itemloaders:index>`,
+    allowing usage from outside Scrapy and a separate release schedule
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+*   Removed the following classes and their parent modules from
+    ``scrapy.linkextractors``:
+
+    *   ``htmlparser.HtmlParserLinkExtractor``
+    *   ``regex.RegexLinkExtractor``
+    *   ``sgml.BaseSgmlLinkExtractor``
+    *   ``sgml.SgmlLinkExtractor``
+
+    Use
+    :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
+    instead (:issue:`4356`, :issue:`4679`)
+
+
+Deprecations
+~~~~~~~~~~~~
+
+*   The ``scrapy.utils.python.retry_on_eintr`` function is now deprecated
+    (:issue:`4683`)
+
+
+New features
+~~~~~~~~~~~~
+
+*   :ref:`Feed exports <topics-feed-exports>` support :ref:`Google Cloud
+    Storage <topics-feed-storage-gcs>` (:issue:`685`, :issue:`3608`)
+
+*   New :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting for batch deliveries
+    (:issue:`4250`, :issue:`4434`)
+
+*   The :command:`parse` command now allows specifying an output file
+    (:issue:`4317`, :issue:`4377`)
+
+*   :meth:`Request.from_curl <scrapy.http.Request.from_curl>` and
+    :func:`~scrapy.utils.curl.curl_to_request_kwargs` now also support
+    ``--data-raw`` (:issue:`4612`)
+
+*   A ``parse`` callback may now be used in built-in spider subclasses, such
+    as :class:`~scrapy.spiders.CrawlSpider` (:issue:`712`, :issue:`732`,
+    :issue:`781`, :issue:`4254` )
+
+
+Bug fixes
+~~~~~~~~~
+
+*   Fixed the :ref:`CSV exporting <topics-feed-format-csv>` of
+    :ref:`dataclass items <dataclass-items>` and :ref:`attr.s items
+    <attrs-items>` (:issue:`4667`, :issue:`4668`)
+
+*   :meth:`Request.from_curl <scrapy.http.Request.from_curl>` and
+    :func:`~scrapy.utils.curl.curl_to_request_kwargs` now set the request
+    method to ``POST`` when a request body is specified and no request method
+    is specified (:issue:`4612`)
+
+*   The processing of ANSI escape sequences in enabled in Windows 10.0.14393
+    and later, where it is required for colored output (:issue:`4393`,
+    :issue:`4403`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+*   Updated the `OpenSSL cipher list format`_ link in the documentation about
+    the :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` setting (:issue:`4653`)
+
+*   Simplified the code example in :ref:`topics-loaders-dataclass`
+    (:issue:`4652`)
+
+.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT
+
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+*   The base implementation of :ref:`item loaders <topics-loaders>` has been
+    moved into :doc:`itemloaders <itemloaders:index>` (:issue:`4005`,
+    :issue:`4516`)
+
+*   Fixed a silenced error in some scheduler tests (:issue:`4644`,
+    :issue:`4645`)
+
+*   Renewed the localhost certificate used for SSL tests (:issue:`4650`)
+
+*   Removed cookie-handling code specific to Python 2 (:issue:`4682`)
+
+*   Stopped using Python 2 unicode literal syntax (:issue:`4704`)
+
+*   Stopped using a backlash for line continuation (:issue:`4673`)
+
+*   Removed unneeded entries from the MyPy exception list (:issue:`4690`)
+
+*   Automated tests now pass on Windows as part of our continuous integration
+    system (:issue:`4458`)
+
+*   Automated tests now pass on the latest PyPy version for supported Python
+    versions in our continuous integration system (:issue:`4504`)
+
+
+.. _release-2.2.1:
+
+Scrapy 2.2.1 (2020-07-17)
+-------------------------
+
+*   The :command:`startproject` command no longer makes unintended changes to
+    the permissions of files in the destination folder, such as removing
+    execution permissions (:issue:`4662`, :issue:`4666`)
+
+
 .. _release-2.2.0:
 
 Scrapy 2.2.0 (2020-06-24)
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 4fce51abc..9638a2322 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -493,6 +493,8 @@ Supported options:
 
 * ``--output`` or ``-o``: dump scraped items to a file
 
+  .. versionadded:: 2.3
+
 .. skip: start
 
 Usage example::
diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index 4e87a00f2..101aa159c 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -289,8 +289,10 @@ request::
         "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'")
 
 Alternatively, if you want to know the arguments needed to recreate that
-request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs`
-function to get a dictionary with the equivalent arguments.
+request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs`
+function to get a dictionary with the equivalent arguments:
+
+.. autofunction:: scrapy.utils.curl.curl_to_request_kwargs
 
 Note that to translate a cURL command into a Scrapy request,
 you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index dd4eb3c61..37b7096f6 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -100,6 +100,7 @@ The storages backends supported out of the box are:
  * :ref:`topics-feed-storage-fs`
  * :ref:`topics-feed-storage-ftp`
  * :ref:`topics-feed-storage-s3` (requires botocore_)
+ * :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
  * :ref:`topics-feed-storage-stdout`
 
 Some storage backends may be unavailable if the required external libraries are
@@ -169,6 +170,9 @@ FTP supports two different connection modes: `active or passive
 mode by default. To use the active connection mode instead, set the
 :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
 
+This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
+
+
 .. _topics-feed-storage-s3:
 
 S3
@@ -194,11 +198,16 @@ You can also define a custom ACL for exported feeds using this setting:
 
  * :setting:`FEED_STORAGE_S3_ACL`
 
+This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
+
+
 .. _topics-feed-storage-gcs:
 
 Google Cloud Storage (GCS)
 --------------------------
 
+.. versionadded:: 2.3
+
 The feeds are stored on `Google Cloud Storage`_.
 
  * URI scheme: ``gs``
@@ -206,7 +215,7 @@ The feeds are stored on `Google Cloud Storage`_.
 
    * ``gs://mybucket/path/to/export.csv``
 
- * Required external libraries: `google-cloud-storage <https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python>`_.
+ * Required external libraries: `google-cloud-storage`_.
 
 For more information about authentication, please refer to `Google Cloud documentation <https://cloud.google.com/docs/authentication/production>`_.
 
@@ -215,6 +224,11 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following
  * :setting:`FEED_STORAGE_GCS_ACL`
  * :setting:`GCS_PROJECT_ID`
 
+This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
+
+.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
+
+
 .. _topics-feed-storage-stdout:
 
 Standard output
@@ -227,6 +241,26 @@ The feeds are written to the standard output of the Scrapy process.
  * Required external libraries: none
 
 
+.. _delayed-file-delivery:
+
+Delayed file delivery
+---------------------
+
+As indicated above, some of the described storage backends use delayed file
+delivery.
+
+These storage backends do not upload items to the feed URI as those items are
+scraped. Instead, Scrapy writes items into a temporary local file, and only
+once all the file contents have been written (i.e. at the end of the crawl) is
+that file uploaded to the feed URI.
+
+If you want item delivery to start earlier when using one of these storage
+backends, use :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` to split the output items
+in multiple files, with the specified maximum item count per file. That way, as
+soon as a file reaches the maximum item count, that file is delivered to the
+feed URI, allowing item delivery to start way before the end of the crawl.
+
+
 Settings
 ========
 
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index aa681522f..9c0efcec4 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -39,7 +39,8 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
 
     :param str curl_command: string containing the curl command
     :param bool ignore_unknown_options: If true, only a warning is emitted when
-    cURL options are unknown. Otherwise raises an error. (default: True)
+                                        cURL options are unknown. Otherwise
+                                        raises an error. (default: True)
     :return: dictionary of Request kwargs
     """
 

From 1278e76d9093b1c5c9ec810768d1066772a8a134 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 4 Aug 2020 20:07:02 +0200
Subject: [PATCH 51/57] =?UTF-8?q?Bump=20version:=202.2.0=20=E2=86=92=202.3?=
 =?UTF-8?q?.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 8d4d74bc5..3c1c8f891 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.2.0
+current_version = 2.3.0
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index ccbccc3dc..276cbf9e2 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-2.2.0
+2.3.0

From 4ee538e44b2650c49054b1f7f4c87ac70350471a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 4 Aug 2020 20:34:11 +0200
Subject: [PATCH 52/57] Update unicode references from Python 2 times in the
 documentation (#4703)

---
 docs/topics/exporters.rst        |  8 ++------
 docs/topics/loaders.rst          |  4 ++--
 docs/topics/request-response.rst | 30 ++++++++++++++++--------------
 docs/topics/selectors.rst        |  7 ++++---
 4 files changed, 24 insertions(+), 25 deletions(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index e5c99e5b1..8c84b85fc 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -166,8 +166,7 @@ BaseItemExporter
       By default, this method looks for a serializer :ref:`declared in the item
       field <topics-exporters-serializers>` and returns the result of applying
       that serializer to the value. If no serializer is found, it returns the
-      value unchanged except for ``unicode`` values which are encoded to
-      ``str`` using the encoding declared in the :attr:`encoding` attribute.
+      value unchanged.
 
       :param field: the field being serialized. If the source :ref:`item object
           <item-types>` does not define field metadata, *field* is an empty
@@ -217,10 +216,7 @@ BaseItemExporter
 
    .. attribute:: encoding
 
-      The encoding that will be used to encode unicode values. This only
-      affects unicode values (which are always serialized to str using this
-      encoding). Other value types are passed unchanged to the specific
-      serialization library.
+      The output character encoding.
 
    .. attribute:: indent
 
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 29d9c5805..c0f534493 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -193,10 +193,10 @@ Item Loaders are declared using a class definition syntax. Here is an example::
 
         default_output_processor = TakeFirst()
 
-        name_in = MapCompose(unicode.title)
+        name_in = MapCompose(str.title)
         name_out = Join()
 
-        price_in = MapCompose(unicode.strip)
+        price_in = MapCompose(str.strip)
 
         # ...
 
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index fbd8e4b73..1dffd1d55 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -51,12 +51,12 @@ Request objects
        given, the dict passed in this parameter will be shallow copied.
     :type meta: dict
 
-    :param body: the request body. If a ``unicode`` is passed, then it's encoded to
-      ``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If
-      ``body`` is not given, an empty string is stored. Regardless of the
-      type of this argument, the final value stored will be a ``str`` (never
-      ``unicode`` or ``None``).
-    :type body: str or unicode
+    :param body: the request body. If a string is passed, then it's encoded as
+      bytes using the ``encoding`` passed (which defaults to ``utf-8``). If
+      ``body`` is not given, an empty bytes object is stored. Regardless of the
+      type of this argument, the final value stored will be a bytes object
+      (never a string or ``None``).
+    :type body: bytes or str
 
     :param headers: the headers of this request. The dict values can be strings
        (for single valued headers) or lists (for multi-valued headers). If
@@ -106,7 +106,7 @@ Request objects
 
     :param encoding: the encoding of this request (defaults to ``'utf-8'``).
        This encoding will be used to percent-encode the URL and to convert the
-       body to ``str`` (if given as ``unicode``).
+       body to bytes (if given as a string).
     :type encoding: string
 
     :param priority: the priority of this request (defaults to ``0``).
@@ -721,7 +721,7 @@ Response objects
     .. attribute:: Response.body
 
         The body of this Response. Keep in mind that Response.body
-        is always a bytes object. If you want the unicode version use
+        is always a bytes object. If you want the string version use
         :attr:`TextResponse.text` (only available in :class:`TextResponse`
         and subclasses).
 
@@ -842,9 +842,9 @@ TextResponse objects
     is the same as for the :class:`Response` class and is not documented here.
 
     :param encoding: is a string which contains the encoding to use for this
-       response. If you create a :class:`TextResponse` object with a unicode
+       response. If you create a :class:`TextResponse` object with a string as
        body, it will be encoded using this encoding (remember the body attribute
-       is always a string). If ``encoding`` is ``None`` (default value), the
+       is always a bytes object). If ``encoding`` is ``None`` (default value), the
        encoding will be looked up in the response headers and body instead.
     :type encoding: string
 
@@ -853,7 +853,7 @@ TextResponse objects
 
     .. attribute:: TextResponse.text
 
-       Response body, as unicode.
+       Response body, as a string.
 
        The same as ``response.body.decode(response.encoding)``, but the
        result is cached after the first call, so you can access
@@ -861,9 +861,11 @@ TextResponse objects
 
        .. note::
 
-            ``unicode(response.body)`` is not a correct way to convert response
-            body to unicode: you would be using the system default encoding
-            (typically ``ascii``) instead of the response encoding.
+            ``str(response.body)`` is not a correct way to convert the response
+            body into a string:
+
+            >>> str(b'body')
+            "b'body'"
 
 
     .. attribute:: TextResponse.encoding
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 5014df6ac..9e2c6ba42 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -64,7 +64,8 @@ more shortcuts: ``response.xpath()`` and ``response.css()``:
 
 Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
 constructed by passing either :class:`~scrapy.http.TextResponse` object or
-markup as an unicode string (in ``text`` argument).
+markup as a string (in ``text`` argument).
+
 Usually there is no need to construct Scrapy selectors manually:
 ``response`` object is available in Spider callbacks, so in most cases
 it is more convenient to use ``response.css()`` and ``response.xpath()``
@@ -383,7 +384,7 @@ Using selectors with regular expressions
 
 :class:`~scrapy.selector.Selector` also has a ``.re()`` method for extracting
 data using regular expressions. However, unlike using ``.xpath()`` or
-``.css()`` methods, ``.re()`` returns a list of unicode strings. So you
+``.css()`` methods, ``.re()`` returns a list of strings. So you
 can't construct nested ``.re()`` calls.
 
 Here's an example used to extract image names from the :ref:`HTML code
@@ -989,7 +990,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this::
       sel.xpath("//h1")
 
 2. Extract the text of all ``<h1>`` elements from an HTML response body,
-   returning a list of unicode strings::
+   returning a list of strings::
 
       sel.xpath("//h1").getall()         # this includes the h1 tag
       sel.xpath("//h1/text()").getall()  # this excludes the h1 tag

From 336f19f5cc6edd0392c77f38b857d3e40bf565da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com>
Date: Tue, 4 Aug 2020 20:42:01 +0200
Subject: [PATCH 53/57] Change super syntax (#4707)

---
 extras/qpsclient.py                          |  2 +-
 scrapy/commands/view.py                      |  2 +-
 scrapy/contracts/default.py                  |  2 +-
 scrapy/core/downloader/contextfactory.py     |  4 ++--
 scrapy/core/downloader/handlers/http11.py    | 10 +++++-----
 scrapy/core/downloader/tls.py                |  2 +-
 scrapy/core/spidermw.py                      |  2 +-
 scrapy/crawler.py                            |  2 +-
 scrapy/downloadermiddlewares/redirect.py     |  2 +-
 scrapy/exceptions.py                         |  4 ++--
 scrapy/exporters.py                          |  2 +-
 scrapy/http/headers.py                       |  8 ++++----
 scrapy/http/request/form.py                  |  2 +-
 scrapy/http/request/json_request.py          |  4 ++--
 scrapy/http/request/rpc.py                   |  2 +-
 scrapy/http/response/text.py                 | 10 +++++-----
 scrapy/item.py                               | 10 +++++-----
 scrapy/linkextractors/__init__.py            |  2 +-
 scrapy/linkextractors/lxmlhtml.py            |  2 +-
 scrapy/pipelines/files.py                    |  2 +-
 scrapy/pipelines/images.py                   |  3 +--
 scrapy/resolver.py                           | 10 +++++-----
 scrapy/selector/unified.py                   |  2 +-
 scrapy/settings/__init__.py                  |  2 +-
 scrapy/spidermiddlewares/httperror.py        |  2 +-
 scrapy/spiders/crawl.py                      |  4 ++--
 scrapy/spiders/init.py                       |  2 +-
 scrapy/spiders/sitemap.py                    |  2 +-
 scrapy/squeues.py                            | 12 ++++++------
 scrapy/statscollectors.py                    |  2 +-
 scrapy/utils/datatypes.py                    | 14 +++++++-------
 scrapy/utils/deprecate.py                    |  8 ++++----
 scrapy/utils/log.py                          |  2 +-
 scrapy/utils/serialize.py                    |  2 +-
 scrapy/utils/testsite.py                     |  4 ++--
 tests/spiders.py                             | 20 ++++++++++----------
 tests/test_command_parse.py                  |  2 +-
 tests/test_commands.py                       |  4 ++--
 tests/test_contracts.py                      |  2 +-
 tests/test_downloader_handlers.py            |  4 ++--
 tests/test_downloadermiddleware_httpcache.py |  4 ++--
 tests/test_downloadermiddleware_robotstxt.py |  4 ++--
 tests/test_exporters.py                      |  2 +-
 tests/test_http_request.py                   |  4 ++--
 tests/test_http_response.py                  |  2 +-
 tests/test_item.py                           |  2 +-
 tests/test_linkextractors.py                 |  2 +-
 tests/test_loader.py                         |  2 +-
 tests/test_loader_deprecated.py              |  2 +-
 tests/test_logformatter.py                   |  2 +-
 tests/test_middleware.py                     |  2 +-
 tests/test_pipeline_media.py                 | 12 ++++++------
 tests/test_request_left.py                   |  2 +-
 tests/test_robotstxt_interface.py            |  8 ++++----
 tests/test_scheduler.py                      |  4 ++--
 tests/test_spidermiddleware_httperror.py     |  2 +-
 56 files changed, 117 insertions(+), 118 deletions(-)

diff --git a/extras/qpsclient.py b/extras/qpsclient.py
index 7554f7eec..fe1f96cbb 100644
--- a/extras/qpsclient.py
+++ b/extras/qpsclient.py
@@ -27,7 +27,7 @@ class QPSSpider(Spider):
     slots = 1
 
     def __init__(self, *a, **kw):
-        super(QPSSpider, self).__init__(*a, **kw)
+        super().__init__(*a, **kw)
         if self.qps is not None:
             self.qps = float(self.qps)
             self.download_delay = 1 / self.qps
diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py
index 908bee966..c8f873334 100644
--- a/scrapy/commands/view.py
+++ b/scrapy/commands/view.py
@@ -11,7 +11,7 @@ class Command(fetch.Command):
         return "Fetch a URL using the Scrapy downloader and show its contents in a browser"
 
     def add_options(self, parser):
-        super(Command, self).add_options(parser)
+        super().add_options(parser)
         parser.remove_option("--headers")
 
     def _print_response(self, response, opts):
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 34f0d36d4..cfdcc7c25 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -56,7 +56,7 @@ class ReturnsContract(Contract):
     }
 
     def __init__(self, *args, **kwargs):
-        super(ReturnsContract, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
         if len(self.args) not in [1, 2, 3]:
             raise ValueError(
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 452242d47..8a7d656a1 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -20,7 +20,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
     """
 
     def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
-        super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self._ssl_method = method
         self.tls_verbose_logging = tls_verbose_logging
         if tls_ciphers:
@@ -45,7 +45,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
         #   (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
         #
         # * getattr() for `_ssl_method` attribute for context factories
-        #   not calling super(..., self).__init__
+        #   not calling super().__init__
         return CertificateOptions(
             verify=False,
             method=getattr(self, 'method', getattr(self, '_ssl_method', None)),
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 22c9ac520..fb04d1fb7 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -126,7 +126,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
 
     def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None):
         proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
-        super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
+        super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
         self._tunnelReadyDeferred = defer.Deferred()
         self._tunneledHost = host
         self._tunneledPort = port
@@ -178,7 +178,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
 
     def connect(self, protocolFactory):
         self._protocolFactory = protocolFactory
-        connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory)
+        connectDeferred = super().connect(protocolFactory)
         connectDeferred.addCallback(self.requestTunnel)
         connectDeferred.addErrback(self.connectFailed)
         return self._tunnelReadyDeferred
@@ -215,7 +215,7 @@ class TunnelingAgent(Agent):
 
     def __init__(self, reactor, proxyConf, contextFactory=None,
                  connectTimeout=None, bindAddress=None, pool=None):
-        super(TunnelingAgent, self).__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
+        super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
         self._proxyConf = proxyConf
         self._contextFactory = contextFactory
 
@@ -235,7 +235,7 @@ class TunnelingAgent(Agent):
         # otherwise, same remote host connection request could reuse
         # a cached tunneled connection to a different proxy
         key = key + self._proxyConf
-        return super(TunnelingAgent, self)._requestWithEndpoint(
+        return super()._requestWithEndpoint(
             key=key,
             endpoint=endpoint,
             method=method,
@@ -249,7 +249,7 @@ class TunnelingAgent(Agent):
 class ScrapyProxyAgent(Agent):
 
     def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None):
-        super(ScrapyProxyAgent, self).__init__(
+        super().__init__(
             reactor=reactor,
             connectTimeout=connectTimeout,
             bindAddress=bindAddress,
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index e43a3c83e..d9f3750d5 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -47,7 +47,7 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
     """
 
     def __init__(self, hostname, ctx, verbose_logging=False):
-        super(ScrapyClientTLSOptions, self).__init__(hostname, ctx)
+        super().__init__(hostname, ctx)
         self.verbose_logging = verbose_logging
 
     def _identityVerifyingInfoCallback(self, connection, where, ret):
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 35264a92b..5a99b96be 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -34,7 +34,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
 
     def _add_middleware(self, mw):
-        super(SpiderMiddlewareManager, self)._add_middleware(mw)
+        super()._add_middleware(mw)
         if hasattr(mw, 'process_spider_input'):
             self.methods['process_spider_input'].append(mw.process_spider_input)
         if hasattr(mw, 'process_start_requests'):
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 6f43771e2..48f19424c 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -277,7 +277,7 @@ class CrawlerProcess(CrawlerRunner):
     """
 
     def __init__(self, settings=None, install_root_handler=True):
-        super(CrawlerProcess, self).__init__(settings)
+        super().__init__(settings)
         install_shutdown_handlers(self._signal_shutdown)
         configure_logging(self.settings, install_root_handler)
         log_scrapy_info(self.settings)
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 366d60dcb..4053fecc5 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -92,7 +92,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
     enabled_setting = 'METAREFRESH_ENABLED'
 
     def __init__(self, settings):
-        super(MetaRefreshMiddleware, self).__init__(settings)
+        super().__init__(settings)
         self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS')
         self._maxdelay = settings.getint('METAREFRESH_MAXDELAY')
 
diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py
index 45f152321..0c410f035 100644
--- a/scrapy/exceptions.py
+++ b/scrapy/exceptions.py
@@ -37,7 +37,7 @@ class CloseSpider(Exception):
     """Raise this from callbacks to request the spider to be closed"""
 
     def __init__(self, reason='cancelled'):
-        super(CloseSpider, self).__init__()
+        super().__init__()
         self.reason = reason
 
 
@@ -74,7 +74,7 @@ class UsageError(Exception):
 
     def __init__(self, *a, **kw):
         self.print_help = kw.pop('print_help', True)
-        super(UsageError, self).__init__(*a, **kw)
+        super().__init__(*a, **kw)
 
 
 class ScrapyDeprecationWarning(Warning):
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 0aba1c904..95518b3ac 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -301,7 +301,7 @@ class PythonItemExporter(BaseItemExporter):
 
     def _configure(self, options, dont_fail=False):
         self.binary = options.pop('binary', True)
-        super(PythonItemExporter, self)._configure(options, dont_fail)
+        super()._configure(options, dont_fail)
         if self.binary:
             warnings.warn(
                 "PythonItemExporter will drop support for binary export in the future",
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index dcaaeddfa..6bf9e5346 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -8,7 +8,7 @@ class Headers(CaselessDict):
 
     def __init__(self, seq=None, encoding='utf-8'):
         self.encoding = encoding
-        super(Headers, self).__init__(seq)
+        super().__init__(seq)
 
     def normkey(self, key):
         """Normalize key to bytes"""
@@ -37,19 +37,19 @@ class Headers(CaselessDict):
 
     def __getitem__(self, key):
         try:
-            return super(Headers, self).__getitem__(key)[-1]
+            return super().__getitem__(key)[-1]
         except IndexError:
             return None
 
     def get(self, key, def_val=None):
         try:
-            return super(Headers, self).get(key, def_val)[-1]
+            return super().get(key, def_val)[-1]
         except IndexError:
             return None
 
     def getlist(self, key, def_val=None):
         try:
-            return super(Headers, self).__getitem__(key)
+            return super().__getitem__(key)
         except KeyError:
             if def_val is not None:
                 return self.normvalue(def_val)
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index a260798ac..59af81321 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -24,7 +24,7 @@ class FormRequest(Request):
         if formdata and kwargs.get('method') is None:
             kwargs['method'] = 'POST'
 
-        super(FormRequest, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
         if formdata:
             items = formdata.items() if isinstance(formdata, dict) else formdata
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index f08b25280..eae3f9f6b 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -32,7 +32,7 @@ class JsonRequest(Request):
             if 'method' not in kwargs:
                 kwargs['method'] = 'POST'
 
-        super(JsonRequest, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.headers.setdefault('Content-Type', 'application/json')
         self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
 
@@ -47,7 +47,7 @@ class JsonRequest(Request):
         elif not body_passed and data_passed:
             kwargs['body'] = self._dumps(data)
 
-        return super(JsonRequest, self).replace(*args, **kwargs)
+        return super().replace(*args, **kwargs)
 
     def _dumps(self, data):
         """Convert to JSON """
diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py
index 811d3ad6b..c70912e49 100644
--- a/scrapy/http/request/rpc.py
+++ b/scrapy/http/request/rpc.py
@@ -31,5 +31,5 @@ class XmlRpcRequest(Request):
         if encoding is not None:
             kwargs['encoding'] = encoding
 
-        super(XmlRpcRequest, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.headers.setdefault('Content-Type', 'text/xml')
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 0f300c8da..a7bb34d48 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -35,13 +35,13 @@ class TextResponse(Response):
         self._cached_benc = None
         self._cached_ubody = None
         self._cached_selector = None
-        super(TextResponse, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
     def _set_url(self, url):
         if isinstance(url, str):
             self._url = to_unicode(url, self.encoding)
         else:
-            super(TextResponse, self)._set_url(url)
+            super()._set_url(url)
 
     def _set_body(self, body):
         self._body = b''  # used by encoding detection
@@ -51,7 +51,7 @@ class TextResponse(Response):
                                 type(self).__name__)
             self._body = body.encode(self._encoding)
         else:
-            super(TextResponse, self)._set_body(body)
+            super()._set_body(body)
 
     def replace(self, *args, **kwargs):
         kwargs.setdefault('encoding', self.encoding)
@@ -166,7 +166,7 @@ class TextResponse(Response):
         elif isinstance(url, parsel.SelectorList):
             raise ValueError("SelectorList is not supported")
         encoding = self.encoding if encoding is None else encoding
-        return super(TextResponse, self).follow(
+        return super().follow(
             url=url,
             callback=callback,
             method=method,
@@ -226,7 +226,7 @@ class TextResponse(Response):
             for sel in selectors:
                 with suppress(_InvalidSelector):
                     urls.append(_url_from_selector(sel))
-        return super(TextResponse, self).follow_all(
+        return super().follow_all(
             urls=urls,
             callback=callback,
             method=method,
diff --git a/scrapy/item.py b/scrapy/item.py
index 4ab83d1a0..c262a153c 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -39,7 +39,7 @@ class BaseItem(_BaseItem, metaclass=_BaseItemMeta):
         if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)):
             warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
                  ScrapyDeprecationWarning, stacklevel=2)
-        return super(BaseItem, cls).__new__(cls, *args, **kwargs)
+        return super().__new__(cls, *args, **kwargs)
 
 
 class Field(dict):
@@ -55,7 +55,7 @@ class ItemMeta(_BaseItemMeta):
     def __new__(mcs, class_name, bases, attrs):
         classcell = attrs.pop('__classcell__', None)
         new_bases = tuple(base._class for base in bases if hasattr(base, '_class'))
-        _class = super(ItemMeta, mcs).__new__(mcs, 'x_' + class_name, new_bases, attrs)
+        _class = super().__new__(mcs, 'x_' + class_name, new_bases, attrs)
 
         fields = getattr(_class, 'fields', {})
         new_attrs = {}
@@ -70,7 +70,7 @@ class ItemMeta(_BaseItemMeta):
         new_attrs['_class'] = _class
         if classcell is not None:
             new_attrs['__classcell__'] = classcell
-        return super(ItemMeta, mcs).__new__(mcs, class_name, bases, new_attrs)
+        return super().__new__(mcs, class_name, bases, new_attrs)
 
 
 class DictItem(MutableMapping, BaseItem):
@@ -81,7 +81,7 @@ class DictItem(MutableMapping, BaseItem):
         if issubclass(cls, DictItem) and not issubclass(cls, Item):
             warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead',
                  ScrapyDeprecationWarning, stacklevel=2)
-        return super(DictItem, cls).__new__(cls, *args, **kwargs)
+        return super().__new__(cls, *args, **kwargs)
 
     def __init__(self, *args, **kwargs):
         self._values = {}
@@ -109,7 +109,7 @@ class DictItem(MutableMapping, BaseItem):
     def __setattr__(self, name, value):
         if not name.startswith('_'):
             raise AttributeError("Use item[%r] = %r to set field value" % (name, value))
-        super(DictItem, self).__setattr__(name, value)
+        super().__setattr__(name, value)
 
     def __len__(self):
         return len(self._values)
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 984a5c4e1..08a6ca1e8 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -65,7 +65,7 @@ class FilteringLinkExtractor:
             warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
                  'please use scrapy.linkextractors.LinkExtractor instead',
                  ScrapyDeprecationWarning, stacklevel=2)
-        return super(FilteringLinkExtractor, cls).__new__(cls)
+        return super().__new__(cls)
 
     def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,
                  restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text):
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 8b9f961ee..e941c4321 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -126,7 +126,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
             strip=strip,
             canonicalized=canonicalize
         )
-        super(LxmlLinkExtractor, self).__init__(
+        super().__init__(
             link_extractor=lx,
             allow=allow,
             deny=deny,
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 487382a38..6bc5d46eb 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -376,7 +376,7 @@ class FilesPipeline(MediaPipeline):
             resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD
         )
 
-        super(FilesPipeline, self).__init__(download_func=download_func, settings=settings)
+        super().__init__(download_func=download_func, settings=settings)
 
     @classmethod
     def from_settings(cls, settings):
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 46f2bfb58..e2dd70215 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -45,8 +45,7 @@ class ImagesPipeline(FilesPipeline):
     DEFAULT_IMAGES_RESULT_FIELD = 'images'
 
     def __init__(self, store_uri, download_func=None, settings=None):
-        super(ImagesPipeline, self).__init__(store_uri, settings=settings,
-                                             download_func=download_func)
+        super().__init__(store_uri, settings=settings, download_func=download_func)
 
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index f69894b1e..f191deac6 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -17,7 +17,7 @@ class CachingThreadedResolver(ThreadedResolver):
     """
 
     def __init__(self, reactor, cache_size, timeout):
-        super(CachingThreadedResolver, self).__init__(reactor)
+        super().__init__(reactor)
         dnscache.limit = cache_size
         self.timeout = timeout
 
@@ -40,7 +40,7 @@ class CachingThreadedResolver(ThreadedResolver):
         # so the input argument above is simply overridden
         # to enforce Scrapy's DNS_TIMEOUT setting's value
         timeout = (self.timeout,)
-        d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
+        d = super().getHostByName(name, timeout)
         if dnscache.limit:
             d.addCallback(self._cache_result, name)
         return d
@@ -80,16 +80,16 @@ class CachingHostnameResolver:
         class CachingResolutionReceiver(resolutionReceiver):
 
             def resolutionBegan(self, resolution):
-                super(CachingResolutionReceiver, self).resolutionBegan(resolution)
+                super().resolutionBegan(resolution)
                 self.resolution = resolution
                 self.resolved = False
 
             def addressResolved(self, address):
-                super(CachingResolutionReceiver, self).addressResolved(address)
+                super().addressResolved(address)
                 self.resolved = True
 
             def resolutionComplete(self):
-                super(CachingResolutionReceiver, self).resolutionComplete()
+                super().resolutionComplete()
                 if self.resolved:
                     dnscache[hostName] = self.resolution
 
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 85a9bb526..f12c61081 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -79,4 +79,4 @@ class Selector(_ParselSelector, object_ref):
             kwargs.setdefault('base_url', response.url)
 
         self.response = response
-        super(Selector, self).__init__(text=text, type=st, root=root, **kwargs)
+        super().__init__(text=text, type=st, root=root, **kwargs)
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index ff8317cd1..b8ae32d7c 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -439,7 +439,7 @@ class Settings(BaseSettings):
         # Do not pass kwarg values here. We don't want to promote user-defined
         # dicts, and we want to update, not replace, default dicts with the
         # values given by the user
-        super(Settings, self).__init__()
+        super().__init__()
         self.setmodule(default_settings, 'default')
         # Promote default dictionaries to BaseSettings instances for per-key
         # priorities
diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py
index 375042340..db9d0f2ae 100644
--- a/scrapy/spidermiddlewares/httperror.py
+++ b/scrapy/spidermiddlewares/httperror.py
@@ -15,7 +15,7 @@ class HttpError(IgnoreRequest):
 
     def __init__(self, response, *args, **kwargs):
         self.response = response
-        super(HttpError, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
 
 class HttpErrorMiddleware:
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index cb7260892..c9fbce08d 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -75,7 +75,7 @@ class CrawlSpider(Spider):
     rules = ()
 
     def __init__(self, *a, **kw):
-        super(CrawlSpider, self).__init__(*a, **kw)
+        super().__init__(*a, **kw)
         self._compile_rules()
 
     def _parse(self, response, **kwargs):
@@ -145,6 +145,6 @@ class CrawlSpider(Spider):
 
     @classmethod
     def from_crawler(cls, crawler, *args, **kwargs):
-        spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs)
+        spider = super().from_crawler(crawler, *args, **kwargs)
         spider._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)
         return spider
diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py
index fd41133ea..fe8c94e78 100644
--- a/scrapy/spiders/init.py
+++ b/scrapy/spiders/init.py
@@ -6,7 +6,7 @@ class InitSpider(Spider):
     """Base Spider with initialization facilities"""
 
     def start_requests(self):
-        self._postinit_reqs = super(InitSpider, self).start_requests()
+        self._postinit_reqs = super().start_requests()
         return iterate_spider_output(self.init_request())
 
     def initialized(self, response=None):
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index c5360bfa7..1f72e76b7 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -18,7 +18,7 @@ class SitemapSpider(Spider):
     sitemap_alternate_links = False
 
     def __init__(self, *a, **kw):
-        super(SitemapSpider, self).__init__(*a, **kw)
+        super().__init__(*a, **kw)
         self._cbs = []
         for r, c in self.sitemap_rules:
             if isinstance(c, str):
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index c7ad4d53d..77ffda6f7 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -20,7 +20,7 @@ def _with_mkdir(queue_class):
             if not os.path.exists(dirname):
                 os.makedirs(dirname, exist_ok=True)
 
-            super(DirectoriesCreated, self).__init__(path, *args, **kwargs)
+            super().__init__(path, *args, **kwargs)
 
     return DirectoriesCreated
 
@@ -31,10 +31,10 @@ def _serializable_queue(queue_class, serialize, deserialize):
 
         def push(self, obj):
             s = serialize(obj)
-            super(SerializableQueue, self).push(s)
+            super().push(s)
 
         def pop(self):
-            s = super(SerializableQueue, self).pop()
+            s = super().pop()
             if s:
                 return deserialize(s)
 
@@ -47,7 +47,7 @@ def _scrapy_serialization_queue(queue_class):
 
         def __init__(self, crawler, key):
             self.spider = crawler.spider
-            super(ScrapyRequestQueue, self).__init__(key)
+            super().__init__(key)
 
         @classmethod
         def from_crawler(cls, crawler, key, *args, **kwargs):
@@ -55,10 +55,10 @@ def _scrapy_serialization_queue(queue_class):
 
         def push(self, request):
             request = request_to_dict(request, self.spider)
-            return super(ScrapyRequestQueue, self).push(request)
+            return super().push(request)
 
         def pop(self):
-            request = super(ScrapyRequestQueue, self).pop()
+            request = super().pop()
 
             if not request:
                 return None
diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py
index 579c60180..ba7d1a6bf 100644
--- a/scrapy/statscollectors.py
+++ b/scrapy/statscollectors.py
@@ -54,7 +54,7 @@ class StatsCollector:
 class MemoryStatsCollector(StatsCollector):
 
     def __init__(self, crawler):
-        super(MemoryStatsCollector, self).__init__(crawler)
+        super().__init__(crawler)
         self.spider_stats = {}
 
     def _persist_stats(self, stats, spider):
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 2a92d0588..e31284a7f 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -15,7 +15,7 @@ class CaselessDict(dict):
     __slots__ = ()
 
     def __init__(self, seq=None):
-        super(CaselessDict, self).__init__()
+        super().__init__()
         if seq:
             self.update(seq)
 
@@ -53,7 +53,7 @@ class CaselessDict(dict):
     def update(self, seq):
         seq = seq.items() if isinstance(seq, Mapping) else seq
         iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
-        super(CaselessDict, self).update(iseq)
+        super().update(iseq)
 
     @classmethod
     def fromkeys(cls, keys, value=None):
@@ -70,14 +70,14 @@ class LocalCache(collections.OrderedDict):
     """
 
     def __init__(self, limit=None):
-        super(LocalCache, self).__init__()
+        super().__init__()
         self.limit = limit
 
     def __setitem__(self, key, value):
         if self.limit:
             while len(self) >= self.limit:
                 self.popitem(last=False)
-        super(LocalCache, self).__setitem__(key, value)
+        super().__setitem__(key, value)
 
 
 class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
@@ -93,18 +93,18 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
     """
 
     def __init__(self, limit=None):
-        super(LocalWeakReferencedCache, self).__init__()
+        super().__init__()
         self.data = LocalCache(limit=limit)
 
     def __setitem__(self, key, value):
         try:
-            super(LocalWeakReferencedCache, self).__setitem__(key, value)
+            super().__setitem__(key, value)
         except TypeError:
             pass  # key is not weak-referenceable, skip caching
 
     def __getitem__(self, key):
         try:
-            return super(LocalWeakReferencedCache, self).__getitem__(key)
+            return super().__getitem__(key)
         except (TypeError, KeyError):
             return None  # key is either not weak-referenceable or not cached
 
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index 3dbea5fee..3c8e3c8b5 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -57,7 +57,7 @@ def create_deprecated_class(
         warned_on_subclass = False
 
         def __new__(metacls, name, bases, clsdict_):
-            cls = super(DeprecatedClass, metacls).__new__(metacls, name, bases, clsdict_)
+            cls = super().__new__(metacls, name, bases, clsdict_)
             if metacls.deprecated_class is None:
                 metacls.deprecated_class = cls
             return cls
@@ -73,7 +73,7 @@ def create_deprecated_class(
                 if warn_once:
                     msg += ' (warning only on first subclass, there may be others)'
                 warnings.warn(msg, warn_category, stacklevel=2)
-            super(DeprecatedClass, cls).__init__(name, bases, clsdict_)
+            super().__init__(name, bases, clsdict_)
 
         # see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass
         # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks
@@ -88,7 +88,7 @@ def create_deprecated_class(
                 # is the deprecated class itself - subclasses of the
                 # deprecated class should not use custom `__subclasscheck__`
                 # method.
-                return super(DeprecatedClass, cls).__subclasscheck__(sub)
+                return super().__subclasscheck__(sub)
 
             if not inspect.isclass(sub):
                 raise TypeError("issubclass() arg 1 must be a class")
@@ -102,7 +102,7 @@ def create_deprecated_class(
                 msg = instance_warn_message.format(cls=_clspath(cls, old_class_path),
                                                    new=_clspath(new_class, new_class_path))
                 warnings.warn(msg, warn_category, stacklevel=2)
-            return super(DeprecatedClass, cls).__call__(*args, **kwargs)
+            return super().__call__(*args, **kwargs)
 
     deprecated_cls = DeprecatedClass(name, (new_class,), clsdict or {})
 
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 51d276097..1d6a2c39d 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -176,7 +176,7 @@ class LogCounterHandler(logging.Handler):
     """Record log levels count into a crawler stats"""
 
     def __init__(self, crawler, *args, **kwargs):
-        super(LogCounterHandler, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.crawler = crawler
 
     def emit(self, record):
diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py
index dc9604578..cc3263602 100644
--- a/scrapy/utils/serialize.py
+++ b/scrapy/utils/serialize.py
@@ -33,7 +33,7 @@ class ScrapyJSONEncoder(json.JSONEncoder):
         elif isinstance(o, Response):
             return "<%s %s %s>" % (type(o).__name__, o.status, o.url)
         else:
-            return super(ScrapyJSONEncoder, self).default(o)
+            return super().default(o)
 
 
 class ScrapyJSONDecoder(json.JSONDecoder):
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index 66930ad2c..397e54703 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -7,12 +7,12 @@ class SiteTest:
 
     def setUp(self):
         from twisted.internet import reactor
-        super(SiteTest, self).setUp()
+        super().setUp()
         self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
         self.baseurl = "http://localhost:%d/" % self.site.getHost().port
 
     def tearDown(self):
-        super(SiteTest, self).tearDown()
+        super().tearDown()
         self.site.stopListening()
 
     def url(self, path):
diff --git a/tests/spiders.py b/tests/spiders.py
index 3eb681819..63bd726fb 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -19,7 +19,7 @@ from scrapy.utils.test import get_from_asyncio_queue
 
 class MockServerSpider(Spider):
     def __init__(self, mockserver=None, *args, **kwargs):
-        super(MockServerSpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.mockserver = mockserver
 
 
@@ -28,7 +28,7 @@ class MetaSpider(MockServerSpider):
     name = 'meta'
 
     def __init__(self, *args, **kwargs):
-        super(MetaSpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.meta = {}
 
     def closed(self, reason):
@@ -41,7 +41,7 @@ class FollowAllSpider(MetaSpider):
     link_extractor = LinkExtractor()
 
     def __init__(self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs):
-        super(FollowAllSpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.urls_visited = []
         self.times = []
         qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency}
@@ -60,7 +60,7 @@ class DelaySpider(MetaSpider):
     name = 'delay'
 
     def __init__(self, n=1, b=0, *args, **kwargs):
-        super(DelaySpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.n = n
         self.b = b
         self.t1 = self.t2 = self.t2_err = 0
@@ -82,7 +82,7 @@ class SimpleSpider(MetaSpider):
     name = 'simple'
 
     def __init__(self, url="http://localhost:8998", *args, **kwargs):
-        super(SimpleSpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.start_urls = [url]
 
     def parse(self, response):
@@ -153,7 +153,7 @@ class ItemSpider(FollowAllSpider):
     name = 'item'
 
     def parse(self, response):
-        for request in super(ItemSpider, self).parse(response):
+        for request in super().parse(response):
             yield request
             yield Item()
             yield {}
@@ -172,7 +172,7 @@ class ErrorSpider(FollowAllSpider):
         raise self.exception_cls('Expected exception')
 
     def parse(self, response):
-        for request in super(ErrorSpider, self).parse(response):
+        for request in super().parse(response):
             yield request
             self.raise_exception()
 
@@ -183,7 +183,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
     fail_yielding = False
 
     def __init__(self, *a, **kw):
-        super(BrokenStartRequestsSpider, self).__init__(*a, **kw)
+        super().__init__(*a, **kw)
         self.seedsseen = []
 
     def start_requests(self):
@@ -201,7 +201,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
 
     def parse(self, response):
         self.seedsseen.append(response.meta.get('seed'))
-        for req in super(BrokenStartRequestsSpider, self).parse(response):
+        for req in super().parse(response):
             yield req
 
 
@@ -243,7 +243,7 @@ class DuplicateStartRequestsSpider(MockServerSpider):
                 yield Request(url, dont_filter=self.dont_filter)
 
     def __init__(self, url="http://localhost:8998", *args, **kwargs):
-        super(DuplicateStartRequestsSpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.visited = 0
 
     def parse(self, response):
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 5754a5478..e115f420f 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -17,7 +17,7 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
     command = 'parse'
 
     def setUp(self):
-        super(ParseCommandTest, self).setUp()
+        super().setUp()
         self.spider_name = 'parse_spider'
         fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py'))
         with open(fname, 'w') as f:
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 42091ab00..8938156fc 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -151,7 +151,7 @@ def get_permissions_dict(path, renamings=None, ignore=None):
 class StartprojectTemplatesTest(ProjectTest):
 
     def setUp(self):
-        super(StartprojectTemplatesTest, self).setUp()
+        super().setUp()
         self.tmpl = join(self.temp_path, 'templates')
         self.tmpl_proj = join(self.tmpl, 'project')
 
@@ -315,7 +315,7 @@ class StartprojectTemplatesTest(ProjectTest):
 class CommandTest(ProjectTest):
 
     def setUp(self):
-        super(CommandTest, self).setUp()
+        super().setUp()
         self.call('startproject', self.project_name)
         self.cwd = join(self.temp_path, self.project_name)
         self.env['SCRAPY_SETTINGS_MODULE'] = '%s.settings' % self.project_name
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 99120b128..2e7e3ccc4 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -378,7 +378,7 @@ class ContractsManagerTest(unittest.TestCase):
             name = 'test_same_url'
 
             def __init__(self, *args, **kwargs):
-                super(TestSameUrlSpider, self).__init__(*args, **kwargs)
+                super().__init__(*args, **kwargs)
                 self.visited = 0
 
             def start_requests(s):
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 57d4cdd6b..13063d106 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -530,7 +530,7 @@ class Https11InvalidDNSId(Https11TestCase):
     """Connect to HTTPS hosts with IP while certificate uses domain names IDs."""
 
     def setUp(self):
-        super(Https11InvalidDNSId, self).setUp()
+        super().setUp()
         self.host = '127.0.0.1'
 
 
@@ -549,7 +549,7 @@ class Https11InvalidDNSPattern(Https11TestCase):
             'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", '
             'subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
         )
-        super(Https11InvalidDNSPattern, self).setUp()
+        super().setUp()
 
 
 class Https11CustomCiphers(unittest.TestCase):
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 9b77c97a8..299fb0eb8 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -134,7 +134,7 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest):
 
     def _get_settings(self, **new_settings):
         new_settings.setdefault('HTTPCACHE_DBM_MODULE', self.dbm_module)
-        return super(DbmStorageWithCustomDbmModuleTest, self)._get_settings(**new_settings)
+        return super()._get_settings(**new_settings)
 
     def test_custom_dbm_module_loaded(self):
         # make sure our dbm module has been loaded
@@ -151,7 +151,7 @@ class FilesystemStorageGzipTest(FilesystemStorageTest):
 
     def _get_settings(self, **new_settings):
         new_settings.setdefault('HTTPCACHE_GZIP', True)
-        return super(FilesystemStorageTest, self)._get_settings(**new_settings)
+        return super()._get_settings(**new_settings)
 
 
 class DummyPolicyTest(_BaseTest):
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index f9936baba..858138f81 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -189,7 +189,7 @@ class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest):
         skip = "Rerp parser is not installed"
 
     def setUp(self):
-        super(RobotsTxtMiddlewareWithRerpTest, self).setUp()
+        super().setUp()
         self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.RerpRobotParser')
 
 
@@ -198,5 +198,5 @@ class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest):
         skip = "Reppy parser is not installed"
 
     def setUp(self):
-        super(RobotsTxtMiddlewareWithReppyTest, self).setUp()
+        super().setUp()
         self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.ReppyRobotParser')
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 660c99ce1..6c25a0064 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -593,7 +593,7 @@ class CustomExporterItemTest(unittest.TestCase):
                 if name == 'age':
                     return str(int(value) + 1)
                 else:
-                    return super(CustomItemExporter, self).serialize_field(field, name, value)
+                    return super().serialize_field(field, name, value)
 
         i = self.item_class(name='John', age='22')
         a = ItemAdapter(i)
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index f5cf4e798..0a303dbe2 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -1265,7 +1265,7 @@ class JsonRequestTest(RequestTest):
 
     def setUp(self):
         warnings.simplefilter("always")
-        super(JsonRequestTest, self).setUp()
+        super().setUp()
 
     def test_data(self):
         r1 = self.request_class(url="http://www.example.com/")
@@ -1419,7 +1419,7 @@ class JsonRequestTest(RequestTest):
 
     def tearDown(self):
         warnings.resetwarnings()
-        super(JsonRequestTest, self).tearDown()
+        super().tearDown()
 
 
 if __name__ == "__main__":
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 56d017de6..f831ef5dc 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -305,7 +305,7 @@ class TextResponseTest(BaseResponseTest):
     response_class = TextResponse
 
     def test_replace(self):
-        super(TextResponseTest, self).test_replace()
+        super().test_replace()
         r1 = self.response_class("http://www.example.com", body="hello", encoding="cp852")
         r2 = r1.replace(url="http://www.example.com/other")
         r3 = r1.replace(url="http://www.example.com/other", encoding="latin1")
diff --git a/tests/test_item.py b/tests/test_item.py
index 0ce78f8c0..66fa761f0 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -312,7 +312,7 @@ class ItemMetaClassCellRegression(unittest.TestCase):
                 # requirement. When not done properly raises an error:
                 # TypeError: __class__ set to <class '__main__.MyItem'>
                 # defining 'MyItem' as <class '__main__.MyItem'>
-                super(MyItem, self).__init__(*args, **kwargs)
+                super().__init__(*args, **kwargs)
 
 
 class DictItemTest(unittest.TestCase):
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index a0bafa5e5..6f133d77a 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -516,7 +516,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
         ])
 
     def test_restrict_xpaths_with_html_entities(self):
-        super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities()
+        super().test_restrict_xpaths_with_html_entities()
 
     def test_filteringlinkextractor_deprecation_warning(self):
         """Make sure the FilteringLinkExtractor deprecation warning is not
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 2ed6f365f..b0bc82f4e 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -250,7 +250,7 @@ class TestOutputProcessorItem(unittest.TestCase):
             temp = Field()
 
             def __init__(self, *args, **kwargs):
-                super(TempItem, self).__init__(self, *args, **kwargs)
+                super().__init__(self, *args, **kwargs)
                 self.setdefault('temp', 0.3)
 
         class TempLoader(ItemLoader):
diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py
index eb14de14f..624dd9ab8 100644
--- a/tests/test_loader_deprecated.py
+++ b/tests/test_loader_deprecated.py
@@ -579,7 +579,7 @@ class TestOutputProcessorDict(unittest.TestCase):
 
         class TempDict(dict):
             def __init__(self, *args, **kwargs):
-                super(TempDict, self).__init__(self, *args, **kwargs)
+                super().__init__(self, *args, **kwargs)
                 self.setdefault('temp', 0.3)
 
         class TempLoader(ItemLoader):
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index b771e7d79..41ff3651d 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -118,7 +118,7 @@ class LogFormatterTestCase(unittest.TestCase):
 
 class LogFormatterSubclass(LogFormatter):
     def crawled(self, request, response, spider):
-        kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider)
+        kwargs = super().crawled(request, response, spider)
         CRAWLEDMSG = (
             "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
         )
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index 3364d2258..b2b75ef20 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -53,7 +53,7 @@ class TestMiddlewareManager(MiddlewareManager):
         return ['tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']]
 
     def _add_middleware(self, mw):
-        super(TestMiddlewareManager, self)._add_middleware(mw)
+        super()._add_middleware(mw)
         if hasattr(mw, 'process'):
             self.methods['process'].append(mw.process)
 
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 19ff00350..4f130c0c9 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -162,18 +162,18 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
 class MockedMediaPipeline(MediaPipeline):
 
     def __init__(self, *args, **kwargs):
-        super(MockedMediaPipeline, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self._mockcalled = []
 
     def download(self, request, info):
         self._mockcalled.append('download')
-        return super(MockedMediaPipeline, self).download(request, info)
+        return super().download(request, info)
 
     def media_to_download(self, request, info):
         self._mockcalled.append('media_to_download')
         if 'result' in request.meta:
             return request.meta.get('result')
-        return super(MockedMediaPipeline, self).media_to_download(request, info)
+        return super().media_to_download(request, info)
 
     def get_media_requests(self, item, info):
         self._mockcalled.append('get_media_requests')
@@ -181,15 +181,15 @@ class MockedMediaPipeline(MediaPipeline):
 
     def media_downloaded(self, response, request, info):
         self._mockcalled.append('media_downloaded')
-        return super(MockedMediaPipeline, self).media_downloaded(response, request, info)
+        return super().media_downloaded(response, request, info)
 
     def media_failed(self, failure, request, info):
         self._mockcalled.append('media_failed')
-        return super(MockedMediaPipeline, self).media_failed(failure, request, info)
+        return super().media_failed(failure, request, info)
 
     def item_completed(self, results, item, info):
         self._mockcalled.append('item_completed')
-        item = super(MockedMediaPipeline, self).item_completed(results, item, info)
+        item = super().item_completed(results, item, info)
         item['results'] = results
         return item
 
diff --git a/tests/test_request_left.py b/tests/test_request_left.py
index 5cfef8e7d..373b2e49c 100644
--- a/tests/test_request_left.py
+++ b/tests/test_request_left.py
@@ -10,7 +10,7 @@ class SignalCatcherSpider(Spider):
     name = 'signal_catcher'
 
     def __init__(self, crawler, url, *args, **kwargs):
-        super(SignalCatcherSpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         crawler.signals.connect(self.on_request_left,
                                 signal=request_left_downloader)
         self.caught_times = 0
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 9d8c201dd..4b15d0fab 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -117,7 +117,7 @@ class BaseRobotParserTest:
 class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
     def setUp(self):
         from scrapy.robotstxt import PythonRobotParser
-        super(PythonRobotParserTest, self)._setUp(PythonRobotParser)
+        super()._setUp(PythonRobotParser)
 
     def test_length_based_precedence(self):
         raise unittest.SkipTest("RobotFileParser does not support length based directives precedence.")
@@ -132,7 +132,7 @@ class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase):
 
     def setUp(self):
         from scrapy.robotstxt import ReppyRobotParser
-        super(ReppyRobotParserTest, self)._setUp(ReppyRobotParser)
+        super()._setUp(ReppyRobotParser)
 
     def test_order_based_precedence(self):
         raise unittest.SkipTest("Reppy does not support order based directives precedence.")
@@ -144,7 +144,7 @@ class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):
 
     def setUp(self):
         from scrapy.robotstxt import RerpRobotParser
-        super(RerpRobotParserTest, self)._setUp(RerpRobotParser)
+        super()._setUp(RerpRobotParser)
 
     def test_length_based_precedence(self):
         raise unittest.SkipTest("Rerp does not support length based directives precedence.")
@@ -156,7 +156,7 @@ class ProtegoRobotParserTest(BaseRobotParserTest, unittest.TestCase):
 
     def setUp(self):
         from scrapy.robotstxt import ProtegoRobotParser
-        super(ProtegoRobotParserTest, self)._setUp(ProtegoRobotParser)
+        super()._setUp(ProtegoRobotParser)
 
     def test_order_based_precedence(self):
         raise unittest.SkipTest("Protego does not support order based directives precedence.")
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 2b6cb0902..512a7460e 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -53,7 +53,7 @@ class MockCrawler(Crawler):
             JOBDIR=jobdir,
             DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter',
         )
-        super(MockCrawler, self).__init__(Spider, settings)
+        super().__init__(Spider, settings)
         self.engine = MockEngine(downloader=MockDownloader())
 
 
@@ -296,7 +296,7 @@ class StartUrlsSpider(Spider):
 
     def __init__(self, start_urls):
         self.start_urls = start_urls
-        super(StartUrlsSpider, self).__init__(name='StartUrlsSpider')
+        super().__init__(name='StartUrlsSpider')
 
     def parse(self, response):
         pass
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index e032b247c..e449cd706 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -19,7 +19,7 @@ class _HttpErrorSpider(MockServerSpider):
     bypass_status_codes = set()
 
     def __init__(self, *args, **kwargs):
-        super(_HttpErrorSpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.start_urls = [
             self.mockserver.url("/status?n=200"),
             self.mockserver.url("/status?n=404"),

From 9d84289109b2368d5929d8b60ce583529c19fe4c Mon Sep 17 00:00:00 2001
From: Kshitij Sharma <ikshitijsharma@gmail.com>
Date: Wed, 5 Aug 2020 09:11:59 +0530
Subject: [PATCH 54/57] deprecated weakkeycache by specifying in __init__

---
 scrapy/utils/python.py     |  4 +++-
 scrapy/utils/tester.py     |  3 +++
 tests/test_utils_python.py | 22 +++++++++++++++++++++-
 3 files changed, 27 insertions(+), 2 deletions(-)
 create mode 100644 scrapy/utils/tester.py

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index c8f921ff3..4756b07b6 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -7,10 +7,12 @@ import inspect
 import re
 import sys
 import weakref
+import warnings
 from functools import partial, wraps
 from itertools import chain
 
 from scrapy.utils.decorators import deprecated
+from scrapy.exceptions import ScrapyDeprecationWarning
 
 
 def flatten(x):
@@ -275,10 +277,10 @@ def equal_attributes(obj1, obj2, attributes):
     return True
 
 
-@deprecated
 class WeakKeyCache:
 
     def __init__(self, default_factory):
+        warnings.warn("Call to deprecated Class WeakKeyCache", category=ScrapyDeprecationWarning, stacklevel=2)
         self.default_factory = default_factory
         self._weakdict = weakref.WeakKeyDictionary()
 
diff --git a/scrapy/utils/tester.py b/scrapy/utils/tester.py
new file mode 100644
index 000000000..691e9bc1a
--- /dev/null
+++ b/scrapy/utils/tester.py
@@ -0,0 +1,3 @@
+from scrapy.utils.decorators import deprecated
+
+
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 5a53d89e4..ebce3c079 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -1,15 +1,18 @@
 import functools
+import gc
 import operator
 import platform
 import unittest
+from itertools import count
 from sys import version_info
 from warnings import catch_warnings
 
 from scrapy.utils.python import (
     memoizemethod_noargs, binary_is_text, equal_attributes,
-    get_func_args, to_bytes, to_unicode,
+    WeakKeyCache, get_func_args, to_bytes, to_unicode,
     without_none_values, MutableChain)
 
+
 __doctests__ = ['scrapy.utils.python']
 
 
@@ -152,6 +155,23 @@ class UtilsPythonTestCase(unittest.TestCase):
         a.meta['z'] = 2
         self.assertFalse(equal_attributes(a, b, [compare_z, 'x']))
 
+    def test_weakkeycache(self):
+        class _Weakme:
+            pass
+
+        _values = count()
+        wk = WeakKeyCache(lambda k: next(_values))
+        k = _Weakme()
+        v = wk[k]
+        self.assertEqual(v, wk[k])
+        self.assertNotEqual(v, wk[_Weakme()])
+        self.assertEqual(v, wk[k])
+        del k
+        for _ in range(100):
+            if wk._weakdict:
+                gc.collect()
+        self.assertFalse(len(wk._weakdict))
+
     def test_get_func_args(self):
         def f1(a, b, c):
             pass

From b35d1f2b2c430f4d12cb8f9d408dfa0c0051746d Mon Sep 17 00:00:00 2001
From: Kshitij Sharma <ikshitijsharma@gmail.com>
Date: Wed, 5 Aug 2020 09:14:04 +0530
Subject: [PATCH 55/57] deleted tester.py

---
 scrapy/utils/tester.py | 3 ---
 1 file changed, 3 deletions(-)
 delete mode 100644 scrapy/utils/tester.py

diff --git a/scrapy/utils/tester.py b/scrapy/utils/tester.py
deleted file mode 100644
index 691e9bc1a..000000000
--- a/scrapy/utils/tester.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from scrapy.utils.decorators import deprecated
-
-

From 983b7ddf2e39c480efc6d104054f92f570714ac8 Mon Sep 17 00:00:00 2001
From: Kshitij Sharma <ikshitijsharma@gmail.com>
Date: Wed, 5 Aug 2020 16:13:52 +0530
Subject: [PATCH 56/57] aesthetic fixes

---
 scrapy/utils/python.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 4756b07b6..59f1b8371 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -6,13 +6,13 @@ import gc
 import inspect
 import re
 import sys
-import weakref
 import warnings
+import weakref
 from functools import partial, wraps
 from itertools import chain
 
-from scrapy.utils.decorators import deprecated
 from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.decorators import deprecated
 
 
 def flatten(x):
@@ -280,7 +280,7 @@ def equal_attributes(obj1, obj2, attributes):
 class WeakKeyCache:
 
     def __init__(self, default_factory):
-        warnings.warn("Call to deprecated Class WeakKeyCache", category=ScrapyDeprecationWarning, stacklevel=2)
+        warnings.warn("The WeakKeyCache class is deprecated", category=ScrapyDeprecationWarning, stacklevel=2)
         self.default_factory = default_factory
         self._weakdict = weakref.WeakKeyDictionary()
 

From 4dc09f09aa9698b02f2cbf2e3001202388eba043 Mon Sep 17 00:00:00 2001
From: linchiwei123 <40888469+linchiwei123@users.noreply.github.com>
Date: Wed, 5 Aug 2020 22:23:19 +0800
Subject: [PATCH 57/57] Update setup.py

---
 setup.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/setup.py b/setup.py
index 58090f7a2..d0880051f 100644
--- a/setup.py
+++ b/setup.py
@@ -23,7 +23,6 @@ install_requires = [
     'cryptography>=2.0',
     'cssselect>=0.9.1',
     'itemloaders>=1.0.1',
-    'lxml>=3.5.0',
     'parsel>=1.5.0',
     'PyDispatcher>=2.0.5',
     'pyOpenSSL>=16.2.0',