mirror of https://github.com/scrapy/scrapy.git
Rename FEED_STORAGE_BATCH_SIZE to FEED_STORAGE_BATCH_ITEM_COUNT
This commit is contained in:
BroodingKangaroo
parent
dad2ea7522
commit
2327ecead0
|
|
@ -220,7 +220,7 @@ These are the settings used for configuring the feed exports:
|
|||
* :setting:`FEED_STORAGE_FTP_ACTIVE`
|
||||
* :setting:`FEED_STORAGE_S3_ACL`
|
||||
* :setting:`FEED_EXPORTERS`
|
||||
* :setting:`FEED_EXPORT_BATCH_SIZE`
|
||||
* :setting:`FEED_STORAGE_BATCH_ITEM_COUNT`
|
||||
|
||||
.. currentmodule:: scrapy.extensions.feedexport
|
||||
|
||||
|
|
@ -431,9 +431,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
|
|||
.. _botocore: https://github.com/boto/botocore
|
||||
.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
|
||||
|
||||
.. setting:: FEED_EXPORT_BATCH_SIZE
|
||||
.. setting:: FEED_STORAGE_BATCH_ITEM_COUNT
|
||||
|
||||
FEED_EXPORT_BATCH_SIZE
|
||||
FEED_STORAGE_BATCH_ITEM_COUNT
|
||||
----------------------
|
||||
Default: ``None``
|
||||
|
||||
|
|
@ -448,7 +448,7 @@ Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI.
|
|||
|
||||
For instance::
|
||||
|
||||
FEED_EXPORT_BATCH_SIZE=100
|
||||
FEED_STORAGE_BATCH_ITEM_COUNT=100
|
||||
|
||||
Your request can be like::
|
||||
|
||||
|
|
|
|||
|
|
@ -242,7 +242,7 @@ class FeedExporter:
|
|||
|
||||
self.storages = self._load_components('FEED_STORAGES')
|
||||
self.exporters = self._load_components('FEED_EXPORTERS')
|
||||
self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_SIZE', None)
|
||||
self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None)
|
||||
for uri, feed in self.feeds.items():
|
||||
if not self._storage_supported(uri):
|
||||
raise NotConfigured
|
||||
|
|
@ -290,7 +290,7 @@ class FeedExporter:
|
|||
def _start_new_batch(self, batch_id, uri, feed, spider, template_uri):
|
||||
"""
|
||||
Redirect the output data stream to a new file.
|
||||
Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified.
|
||||
Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified.
|
||||
:param batch_id: sequence number of current batch
|
||||
:param uri: uri of the new batch to start
|
||||
:param feed: dict with parameters of feed
|
||||
|
|
@ -326,7 +326,7 @@ class FeedExporter:
|
|||
slot.start_exporting()
|
||||
slot.exporter.export_item(item)
|
||||
slot.itemcount += 1
|
||||
# create new slot for each slot with itemcount == FEED_STORAGE_BATCH_SIZE and close the old one
|
||||
# create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
|
||||
if self.storage_batch_size and slot.itemcount == self.storage_batch_size:
|
||||
uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot)
|
||||
self._close_slot(slot, spider)
|
||||
|
|
@ -358,12 +358,12 @@ class FeedExporter:
|
|||
|
||||
def _batch_deliveries_supported(self, uri):
|
||||
"""
|
||||
If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(batch_time)s or %(batch_id)s
|
||||
If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s
|
||||
to distinguish different files of partial output
|
||||
"""
|
||||
if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
|
||||
return True
|
||||
logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
|
||||
logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified')
|
||||
return False
|
||||
|
||||
def _storage_supported(self, uri):
|
||||
|
|
|
|||
|
|
@ -146,7 +146,7 @@ FEED_STORAGES_BASE = {
|
|||
's3': 'scrapy.extensions.feedexport.S3FeedStorage',
|
||||
'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
|
||||
}
|
||||
FEED_STORAGE_BATCH_SIZE = None
|
||||
FEED_STORAGE_BATCH_ITEM_COUNT = None
|
||||
FEED_EXPORTERS = {}
|
||||
FEED_EXPORTERS_BASE = {
|
||||
'json': 'scrapy.exporters.JsonItemExporter',
|
||||
|
|
|
|||
|
|
@ -1025,7 +1025,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
|
|||
os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
|
||||
},
|
||||
})
|
||||
batch_size = settings['FEED_STORAGE_BATCH_SIZE']
|
||||
batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
|
||||
rows = [{k: v for k, v in row.items() if v} for row in rows]
|
||||
data = yield self.exported_data(items, settings)
|
||||
for batch in data['jl']:
|
||||
|
|
@ -1041,7 +1041,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
|
|||
os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
|
||||
},
|
||||
})
|
||||
batch_size = settings['FEED_STORAGE_BATCH_SIZE']
|
||||
batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
|
||||
data = yield self.exported_data(items, settings)
|
||||
for batch in data['csv']:
|
||||
got_batch = csv.DictReader(to_unicode(batch).splitlines())
|
||||
|
|
@ -1057,7 +1057,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
|
|||
os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
|
||||
},
|
||||
})
|
||||
batch_size = settings['FEED_STORAGE_BATCH_SIZE']
|
||||
batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
|
||||
rows = [{k: v for k, v in row.items() if v} for row in rows]
|
||||
data = yield self.exported_data(items, settings)
|
||||
for batch in data['xml']:
|
||||
|
|
@ -1075,7 +1075,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
|
|||
os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
|
||||
},
|
||||
})
|
||||
batch_size = settings['FEED_STORAGE_BATCH_SIZE']
|
||||
batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
|
||||
rows = [{k: v for k, v in row.items() if v} for row in rows]
|
||||
data = yield self.exported_data(items, settings)
|
||||
# XML
|
||||
|
|
@ -1100,7 +1100,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
|
|||
os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
|
||||
},
|
||||
})
|
||||
batch_size = settings['FEED_STORAGE_BATCH_SIZE']
|
||||
batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
|
||||
rows = [{k: v for k, v in row.items() if v} for row in rows]
|
||||
data = yield self.exported_data(items, settings)
|
||||
import pickle
|
||||
|
|
@ -1117,7 +1117,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
|
|||
os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
|
||||
},
|
||||
})
|
||||
batch_size = settings['FEED_STORAGE_BATCH_SIZE']
|
||||
batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
|
||||
rows = [{k: v for k, v in row.items() if v} for row in rows]
|
||||
data = yield self.exported_data(items, settings)
|
||||
import marshal
|
||||
|
|
@ -1140,7 +1140,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
|
|||
{'foo': 'bar3', 'baz': 'quux3', 'egg': ''}
|
||||
]
|
||||
settings = {
|
||||
'FEED_STORAGE_BATCH_SIZE': 2
|
||||
'FEED_STORAGE_BATCH_ITEM_COUNT': 2
|
||||
}
|
||||
header = self.MyItem.fields.keys()
|
||||
yield self.assertExported(items, header, rows, settings=settings)
|
||||
|
|
@ -1151,7 +1151,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
|
|||
'FEEDS': {
|
||||
self._random_temp_filename(): {'format': 'xml'},
|
||||
},
|
||||
'FEED_STORAGE_BATCH_SIZE': 1
|
||||
'FEED_STORAGE_BATCH_ITEM_COUNT': 1
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
self.assertRaises(NotConfigured, FeedExporter, crawler)
|
||||
|
|
@ -1163,7 +1163,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
|
|||
'FEEDS': {
|
||||
os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
|
||||
},
|
||||
'FEED_STORAGE_BATCH_SIZE': 1
|
||||
'FEED_STORAGE_BATCH_ITEM_COUNT': 1
|
||||
}
|
||||
data = yield self.exported_no_data(settings)
|
||||
data = dict(data)
|
||||
|
|
@ -1185,7 +1185,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
|
|||
},
|
||||
'FEED_STORE_EMPTY': True,
|
||||
'FEED_EXPORT_INDENT': None,
|
||||
'FEED_STORAGE_BATCH_SIZE': 1,
|
||||
'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
|
||||
}
|
||||
data = yield self.exported_no_data(settings)
|
||||
data = dict(data)
|
||||
|
|
@ -1225,7 +1225,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
|
|||
'encoding': 'utf-8',
|
||||
},
|
||||
},
|
||||
'FEED_STORAGE_BATCH_SIZE': 1,
|
||||
'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
|
||||
}
|
||||
data = yield self.exported_data(items, settings)
|
||||
for fmt, expected in formats.items():
|
||||
|
|
@ -1249,7 +1249,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
|
|||
'format': 'json',
|
||||
},
|
||||
},
|
||||
'FEED_STORAGE_BATCH_SIZE': 1,
|
||||
'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
|
||||
}
|
||||
data = yield self.exported_data(items, settings)
|
||||
self.assertEqual(len(items) + 1, len(data['json']))
|
||||
|
|
@ -1291,7 +1291,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
|
|||
'format': 'json',
|
||||
},
|
||||
},
|
||||
'FEED_STORAGE_BATCH_SIZE': 1,
|
||||
'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
|
||||
}
|
||||
items = [
|
||||
self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
|
||||
|
|
@ -1309,7 +1309,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
|
|||
|
||||
s3 = boto3.resource('s3')
|
||||
my_bucket = s3.Bucket(s3_test_bucket_name)
|
||||
batch_size = settings['FEED_STORAGE_BATCH_SIZE']
|
||||
batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
|
||||
|
||||
with MockServer() as s:
|
||||
runner = CrawlerRunner(Settings(settings))
|
||||
|
|
|
|||
Loading…
Reference in New Issue