mirror of https://github.com/scrapy/scrapy.git
Minor fixes
This commit is contained in:
parent
0627bf476e
commit
1cdcf8b08b
|
|
@ -437,17 +437,18 @@ FEED_STORAGE_BATCH_ITEM_COUNT
|
|||
-----------------------------
|
||||
Default: ``None``
|
||||
|
||||
An integer number that represents the number of scraped items stored in each output
|
||||
file. Whenever the number of items exceeds this setting, a new file is
|
||||
created and the output is redirected to it. The name of the new file will be selected
|
||||
based on the timestamp when the feed is being created and/or on the batch sequence number.
|
||||
Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`.
|
||||
If assigned an integer number higher than ``0``, Scrapy generates multiple output files
|
||||
storing up to the specified number of items in each output file.
|
||||
|
||||
When generating multiple output files, you must use at least one of the following
|
||||
placeholders in :setting:`FEED_URI` to indicate how the different output file names are
|
||||
generated:
|
||||
|
||||
* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
|
||||
(e.g. `2020-03-28T14-45-08.237134`)
|
||||
(e.g. ``2020-03-28T14-45-08.237134``)
|
||||
|
||||
* ``%(batch_id)s`` - gets replaced by the batch sequence number of batch
|
||||
(e.g. `2` for the second file)
|
||||
(e.g. ``2`` for the second file)
|
||||
|
||||
For instance, if your settings include::
|
||||
|
||||
|
|
@ -457,7 +458,7 @@ And your :command:`crawl` command line is::
|
|||
|
||||
scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
|
||||
|
||||
The resulting directory tree of above can be like::
|
||||
The command line above can generate a directory tree like::
|
||||
|
||||
->projectname
|
||||
-->dirname
|
||||
|
|
@ -466,4 +467,4 @@ The resulting directory tree of above can be like::
|
|||
--->3-filename2020-03-28T14-45-10.046092.json
|
||||
|
||||
Where the first and second files contain exactly 100 items. The last one contains
|
||||
<= 100 items.
|
||||
100 items or fever.
|
||||
|
|
|
|||
|
|
@ -180,7 +180,7 @@ class FTPFeedStorage(BlockingFeedStorage):
|
|||
|
||||
|
||||
class _FeedSlot:
|
||||
def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, template_uri):
|
||||
def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template):
|
||||
self.file = file
|
||||
self.exporter = exporter
|
||||
self.storage = storage
|
||||
|
|
@ -188,7 +188,7 @@ class _FeedSlot:
|
|||
self.batch_id = batch_id
|
||||
self.format = format
|
||||
self.store_empty = store_empty
|
||||
self.template_uri = template_uri
|
||||
self.uri_template = uri_template
|
||||
self.uri = uri
|
||||
# flags
|
||||
self.itemcount = 0
|
||||
|
|
@ -260,7 +260,7 @@ class FeedExporter:
|
|||
uri=uri % uri_params,
|
||||
feed=feed,
|
||||
spider=spider,
|
||||
template_uri=uri,
|
||||
uri_template=uri,
|
||||
))
|
||||
|
||||
def close_spider(self, spider):
|
||||
|
|
@ -288,7 +288,7 @@ class FeedExporter:
|
|||
extra={'spider': spider}))
|
||||
return d
|
||||
|
||||
def _start_new_batch(self, batch_id, uri, feed, spider, template_uri):
|
||||
def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
|
||||
"""
|
||||
Redirect the output data stream to a new file.
|
||||
Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified.
|
||||
|
|
@ -296,7 +296,7 @@ class FeedExporter:
|
|||
:param uri: uri of the new batch to start
|
||||
:param feed: dict with parameters of feed
|
||||
:param spider: user spider
|
||||
:param template_uri: template uri which contains %(batch_time)s or %(batch_id)s to create new uri
|
||||
:param uri_template: template of uri which contains %(batch_time)s or %(batch_id)s to create new uri
|
||||
"""
|
||||
storage = self._get_storage(uri)
|
||||
file = storage.open(spider)
|
||||
|
|
@ -315,7 +315,7 @@ class FeedExporter:
|
|||
format=feed['format'],
|
||||
store_empty=feed['store_empty'],
|
||||
batch_id=batch_id,
|
||||
template_uri=template_uri,
|
||||
uri_template=uri_template,
|
||||
)
|
||||
if slot.store_empty:
|
||||
slot.start_exporting()
|
||||
|
|
@ -329,14 +329,14 @@ class FeedExporter:
|
|||
slot.itemcount += 1
|
||||
# create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
|
||||
if self.storage_batch_size and slot.itemcount == self.storage_batch_size:
|
||||
uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot)
|
||||
uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
|
||||
self._close_slot(slot, spider)
|
||||
slots.append(self._start_new_batch(
|
||||
batch_id=slot.batch_id + 1,
|
||||
uri=slot.template_uri % uri_params,
|
||||
feed=self.feeds[slot.template_uri],
|
||||
uri=slot.uri_template % uri_params,
|
||||
feed=self.feeds[slot.uri_template],
|
||||
spider=spider,
|
||||
template_uri=slot.template_uri,
|
||||
uri_template=slot.uri_template,
|
||||
))
|
||||
else:
|
||||
slots.append(slot)
|
||||
|
|
|
|||
Loading…
Reference in New Issue