Minor fixes

This commit is contained in:
BroodingKangaroo 2020-05-15 19:46:36 +03:00
parent 0627bf476e
commit 1cdcf8b08b
2 changed files with 20 additions and 19 deletions

View File

@ -437,17 +437,18 @@ FEED_STORAGE_BATCH_ITEM_COUNT
-----------------------------
Default: ``None``
An integer number that represents the number of scraped items stored in each output
file. Whenever the number of items exceeds this setting, a new file is
created and the output is redirected to it. The name of the new file will be selected
based on the timestamp when the feed is being created and/or on the batch sequence number.
Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`.
If assigned an integer number higher than ``0``, Scrapy generates multiple output files
storing up to the specified number of items in each output file.
When generating multiple output files, you must use at least one of the following
placeholders in :setting:`FEED_URI` to indicate how the different output file names are
generated:
* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
(e.g. `2020-03-28T14-45-08.237134`)
(e.g. ``2020-03-28T14-45-08.237134``)
* ``%(batch_id)s`` - gets replaced by the batch sequence number of batch
(e.g. `2` for the second file)
(e.g. ``2`` for the second file)
For instance, if your settings include::
@ -457,7 +458,7 @@ And your :command:`crawl` command line is::
scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
The resulting directory tree of above can be like::
The command line above can generate a directory tree like::
->projectname
-->dirname
@ -466,4 +467,4 @@ The resulting directory tree of above can be like::
--->3-filename2020-03-28T14-45-10.046092.json
Where the first and second files contain exactly 100 items. The last one contains
<= 100 items.
100 items or fever.

View File

@ -180,7 +180,7 @@ class FTPFeedStorage(BlockingFeedStorage):
class _FeedSlot:
def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, template_uri):
def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template):
self.file = file
self.exporter = exporter
self.storage = storage
@ -188,7 +188,7 @@ class _FeedSlot:
self.batch_id = batch_id
self.format = format
self.store_empty = store_empty
self.template_uri = template_uri
self.uri_template = uri_template
self.uri = uri
# flags
self.itemcount = 0
@ -260,7 +260,7 @@ class FeedExporter:
uri=uri % uri_params,
feed=feed,
spider=spider,
template_uri=uri,
uri_template=uri,
))
def close_spider(self, spider):
@ -288,7 +288,7 @@ class FeedExporter:
extra={'spider': spider}))
return d
def _start_new_batch(self, batch_id, uri, feed, spider, template_uri):
def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
"""
Redirect the output data stream to a new file.
Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified.
@ -296,7 +296,7 @@ class FeedExporter:
:param uri: uri of the new batch to start
:param feed: dict with parameters of feed
:param spider: user spider
:param template_uri: template uri which contains %(batch_time)s or %(batch_id)s to create new uri
:param uri_template: template of uri which contains %(batch_time)s or %(batch_id)s to create new uri
"""
storage = self._get_storage(uri)
file = storage.open(spider)
@ -315,7 +315,7 @@ class FeedExporter:
format=feed['format'],
store_empty=feed['store_empty'],
batch_id=batch_id,
template_uri=template_uri,
uri_template=uri_template,
)
if slot.store_empty:
slot.start_exporting()
@ -329,14 +329,14 @@ class FeedExporter:
slot.itemcount += 1
# create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
if self.storage_batch_size and slot.itemcount == self.storage_batch_size:
uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot)
uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
self._close_slot(slot, spider)
slots.append(self._start_new_batch(
batch_id=slot.batch_id + 1,
uri=slot.template_uri % uri_params,
feed=self.feeds[slot.template_uri],
uri=slot.uri_template % uri_params,
feed=self.feeds[slot.uri_template],
spider=spider,
template_uri=slot.template_uri,
uri_template=slot.uri_template,
))
else:
slots.append(slot)