diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 638733b6a..6f7db20c4 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -467,4 +467,4 @@ The command line above can generate a directory tree like:: --->3-filename2020-03-28T14-45-10.046092.json Where the first and second files contain exactly 100 items. The last one contains -100 items or fever. +100 items or fewer. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 3d691c580..cc26ae173 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -243,11 +243,11 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None) + self.storage_batch_item_count = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None) for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured - if not self._batch_deliveries_supported(uri): + if not self._settings_are_valid(uri): raise NotConfigured if not self._exporter_supported(feed['format']): raise NotConfigured @@ -328,7 +328,7 @@ class FeedExporter: slot.exporter.export_item(item) slot.itemcount += 1 # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one - if self.storage_batch_size and slot.itemcount == self.storage_batch_size: + if self.storage_batch_item_count and slot.itemcount == self.storage_batch_item_count: uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot) self._close_slot(slot, spider) slots.append(self._start_new_batch( @@ -357,12 +357,12 @@ class FeedExporter: return True logger.error("Unknown feed format: %(format)s", {'format': format}) - def _batch_deliveries_supported(self, uri): + def _settings_are_valid(self, uri): """ If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ - if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri: + if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri: return True logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified') return False diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d1374f291..88f9a5933 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -986,7 +986,7 @@ class FeedExportTest(FeedExportTestBase): self.assertEqual(data['csv'], b'') -class PartialDeliveriesTest(FeedExportTestBase): +class BatchDeliveriesTest(FeedExportTestBase): __test__ = True _file_mark = '_%(batch_time)s_#%(batch_id)s_' diff --git a/tox.ini b/tox.ini index 6dd944dff..7507a14a6 100644 --- a/tox.ini +++ b/tox.ini @@ -12,9 +12,9 @@ deps = -ctests/constraints.txt -rtests/requirements-py3.txt # Extras + boto3>=1.13.0 botocore>=1.3.23 Pillow>=3.4.2 - boto3>=1.13.0 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID