diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 2106b41f5..917240d4d 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -220,7 +220,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_STORAGE_FTP_ACTIVE` * :setting:`FEED_STORAGE_S3_ACL` * :setting:`FEED_EXPORTERS` - * :setting:`FEED_EXPORT_BATCH_SIZE` + * :setting:`FEED_STORAGE_BATCH_ITEM_COUNT` .. currentmodule:: scrapy.extensions.feedexport @@ -431,9 +431,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. _botocore: https://github.com/boto/botocore .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl -.. setting:: FEED_EXPORT_BATCH_SIZE +.. setting:: FEED_STORAGE_BATCH_ITEM_COUNT -FEED_EXPORT_BATCH_SIZE +FEED_STORAGE_BATCH_ITEM_COUNT ---------------------- Default: ``None`` @@ -448,7 +448,7 @@ Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI. For instance:: - FEED_EXPORT_BATCH_SIZE=100 + FEED_STORAGE_BATCH_ITEM_COUNT=100 Your request can be like:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index a262f5d18..5bc946634 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -242,7 +242,7 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_SIZE', None) + self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None) for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured @@ -290,7 +290,7 @@ class FeedExporter: def _start_new_batch(self, batch_id, uri, feed, spider, template_uri): """ Redirect the output data stream to a new file. - Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified. + Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified. :param batch_id: sequence number of current batch :param uri: uri of the new batch to start :param feed: dict with parameters of feed @@ -326,7 +326,7 @@ class FeedExporter: slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 - # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_SIZE and close the old one + # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one if self.storage_batch_size and slot.itemcount == self.storage_batch_size: uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot) self._close_slot(slot, spider) @@ -358,12 +358,12 @@ class FeedExporter: def _batch_deliveries_supported(self, uri): """ - If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(batch_time)s or %(batch_id)s + If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri: return True - logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') + logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified') return False def _storage_supported(self, uri): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index c3463a505..5a7dc533e 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -146,7 +146,7 @@ FEED_STORAGES_BASE = { 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } -FEED_STORAGE_BATCH_SIZE = None +FEED_STORAGE_BATCH_ITEM_COUNT = None FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2217bb4ed..1a21eeba9 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1025,7 +1025,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['jl']: @@ -1041,7 +1041,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] data = yield self.exported_data(items, settings) for batch in data['csv']: got_batch = csv.DictReader(to_unicode(batch).splitlines()) @@ -1057,7 +1057,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['xml']: @@ -1075,7 +1075,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) # XML @@ -1100,7 +1100,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import pickle @@ -1117,7 +1117,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import marshal @@ -1140,7 +1140,7 @@ class PartialDeliveriesTest(FeedExportTestBase): {'foo': 'bar3', 'baz': 'quux3', 'egg': ''} ] settings = { - 'FEED_STORAGE_BATCH_SIZE': 2 + 'FEED_STORAGE_BATCH_ITEM_COUNT': 2 } header = self.MyItem.fields.keys() yield self.assertExported(items, header, rows, settings=settings) @@ -1151,7 +1151,7 @@ class PartialDeliveriesTest(FeedExportTestBase): 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, }, - 'FEED_STORAGE_BATCH_SIZE': 1 + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1 } crawler = get_crawler(settings_dict=settings) self.assertRaises(NotConfigured, FeedExporter, crawler) @@ -1163,7 +1163,7 @@ class PartialDeliveriesTest(FeedExportTestBase): 'FEEDS': { os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, }, - 'FEED_STORAGE_BATCH_SIZE': 1 + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1 } data = yield self.exported_no_data(settings) data = dict(data) @@ -1185,7 +1185,7 @@ class PartialDeliveriesTest(FeedExportTestBase): }, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None, - 'FEED_STORAGE_BATCH_SIZE': 1, + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } data = yield self.exported_no_data(settings) data = dict(data) @@ -1225,7 +1225,7 @@ class PartialDeliveriesTest(FeedExportTestBase): 'encoding': 'utf-8', }, }, - 'FEED_STORAGE_BATCH_SIZE': 1, + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } data = yield self.exported_data(items, settings) for fmt, expected in formats.items(): @@ -1249,7 +1249,7 @@ class PartialDeliveriesTest(FeedExportTestBase): 'format': 'json', }, }, - 'FEED_STORAGE_BATCH_SIZE': 1, + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } data = yield self.exported_data(items, settings) self.assertEqual(len(items) + 1, len(data['json'])) @@ -1291,7 +1291,7 @@ class PartialDeliveriesTest(FeedExportTestBase): 'format': 'json', }, }, - 'FEED_STORAGE_BATCH_SIZE': 1, + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), @@ -1309,7 +1309,7 @@ class PartialDeliveriesTest(FeedExportTestBase): s3 = boto3.resource('s3') my_bucket = s3.Bucket(s3_test_bucket_name) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] with MockServer() as s: runner = CrawlerRunner(Settings(settings))