Minor fixes

This commit is contained in:
BroodingKangaroo 2020-05-15 22:50:54 +03:00
parent 1cdcf8b08b
commit 10ae1a284f
4 changed files with 8 additions and 8 deletions

View File

@ -467,4 +467,4 @@ The command line above can generate a directory tree like::
--->3-filename2020-03-28T14-45-10.046092.json
Where the first and second files contain exactly 100 items. The last one contains
100 items or fever.
100 items or fewer.

View File

@ -243,11 +243,11 @@ class FeedExporter:
self.storages = self._load_components('FEED_STORAGES')
self.exporters = self._load_components('FEED_EXPORTERS')
self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None)
self.storage_batch_item_count = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None)
for uri, feed in self.feeds.items():
if not self._storage_supported(uri):
raise NotConfigured
if not self._batch_deliveries_supported(uri):
if not self._settings_are_valid(uri):
raise NotConfigured
if not self._exporter_supported(feed['format']):
raise NotConfigured
@ -328,7 +328,7 @@ class FeedExporter:
slot.exporter.export_item(item)
slot.itemcount += 1
# create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
if self.storage_batch_size and slot.itemcount == self.storage_batch_size:
if self.storage_batch_item_count and slot.itemcount == self.storage_batch_item_count:
uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
self._close_slot(slot, spider)
slots.append(self._start_new_batch(
@ -357,12 +357,12 @@ class FeedExporter:
return True
logger.error("Unknown feed format: %(format)s", {'format': format})
def _batch_deliveries_supported(self, uri):
def _settings_are_valid(self, uri):
"""
If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s
to distinguish different files of partial output
"""
if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
return True
logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified')
return False

View File

@ -986,7 +986,7 @@ class FeedExportTest(FeedExportTestBase):
self.assertEqual(data['csv'], b'')
class PartialDeliveriesTest(FeedExportTestBase):
class BatchDeliveriesTest(FeedExportTestBase):
__test__ = True
_file_mark = '_%(batch_time)s_#%(batch_id)s_'

View File

@ -12,9 +12,9 @@ deps =
-ctests/constraints.txt
-rtests/requirements-py3.txt
# Extras
boto3>=1.13.0
botocore>=1.3.23
Pillow>=3.4.2
boto3>=1.13.0
passenv =
S3_TEST_FILE_URI
AWS_ACCESS_KEY_ID