RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Change setting name. Add leading zeroes to batch_id. Minor fixes.

This commit is contained in:
BroodingKangaroo 2020-07-02 12:38:08 +03:00
parent 7b1d3c35ea
commit 1e245046ed
6 changed files with 45 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
docs/topics/feed-exports.rst
View File

@ -220,7 +220,7 @@ These are the settings used for configuring the feed exports:
* :setting:`FEED_STORAGE_FTP_ACTIVE`
* :setting:`FEED_STORAGE_S3_ACL`
* :setting:`FEED_EXPORTERS`
* :setting:`FEED_STORAGE_BATCH_ITEM_COUNT`
* :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
.. currentmodule:: scrapy.extensions.feedexport
@ -272,7 +272,7 @@ as a fallback value if that key is not provided for a specific feed definition.
* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
* ``batch_item_count``: falls back to :setting:`FEED_STORAGE_BATCH_ITEM_COUNT`
* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
.. setting:: FEED_EXPORT_ENCODING
@ -432,9 +432,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
.. _botocore: https://github.com/boto/botocore
.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
.. setting:: FEED_STORAGE_BATCH_ITEM_COUNT
.. setting:: FEED_EXPORT_BATCH_ITEM_COUNT
FEED_STORAGE_BATCH_ITEM_COUNT
FEED_EXPORT_BATCH_ITEM_COUNT
-----------------------------
Default: ``0``
@ -448,16 +448,19 @@ generated:
* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
(e.g. ``2020-03-28T14-45-08.237134``)
* ``%(batch_id)s`` - gets replaced by the batch sequence number of batch
(e.g. ``2`` for the second file)
* ``%(batch_id)0xd`` - gets replaced by the sequence number of the batch.
By replacing ``x`` with an integer you set the number of leading zeroes to prevent
inappropriate sorting like this: [``'1'``, ``'10'``, ``'2'``]. Here are some examples:
``%(batch_id)01d`` for the second batch gets replaced by ``2``
``%(batch_id)05d`` for the third batch gets replaced by ``00003``
For instance, if your settings include::
FEED_STORAGE_BATCH_ITEM_COUNT=100
FEED_EXPORT_BATCH_ITEM_COUNT=100
And your :command:`crawl` command line is::
scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
scrapy crawl spidername -o dirname/%(batch_id)d-filename%(batch_time)s.json
The command line above can generate a directory tree like::

24
scrapy/extensions/feedexport.py
View File

@ -6,6 +6,7 @@ See documentation in docs/topics/feed-exports.rst
import logging
import os
import re
import sys
import warnings
from datetime import datetime
@ -25,6 +26,7 @@ from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import without_none_values
logger = logging.getLogger(__name__)
@ -245,7 +247,7 @@ class FeedExporter:
for uri, feed in self.feeds.items():
if not self._storage_supported(uri):
raise NotConfigured
if not self._settings_are_valid(uri):
if not self._settings_are_valid():
raise NotConfigured
if not self._exporter_supported(feed['format']):
raise NotConfigured
@ -298,7 +300,7 @@ class FeedExporter:
def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
"""
Redirect the output data stream to a new file.
Execute multiple times if FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
:param batch_id: sequence number of current batch
:param uri: uri of the new batch to start
:param feed: dict with parameters of feed
@ -334,10 +336,10 @@ class FeedExporter:
slot.start_exporting()
slot.exporter.export_item(item)
slot.itemcount += 1
# create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
# create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one
if (
self.feeds[slot.uri_template]['batch_item_count']
and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count']
self.feeds[slot.uri_template]['batch_item_count']
and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count']
):
uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
self._close_slot(slot, spider)
@ -367,16 +369,18 @@ class FeedExporter:
return True
logger.error("Unknown feed format: %(format)s", {'format': format})
def _settings_are_valid(self, uri):
def _settings_are_valid(self):
"""
If FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
%(batch_time)s or %(batch_id)s to distinguish different files of partial output
"""
for uri_template, values in self.feeds.items():
if values['batch_item_count'] and not any(s in uri_template for s in ['%(batch_time)s', '%(batch_id)s']):
if values['batch_item_count'] and not re.findall(r'(%\(batch_time\)s|(%\(batch_id\)0\d*d))', uri_template):
logger.error(
'%(batch_time)s or %(batch_id)s must be in uri({}) if FEED_STORAGE_BATCH_ITEM_COUNT setting '
'or FEEDS.batch_item_count is specified and greater than 0.'.format(uri_template)
'%(batch_time)s or %(batch_id)0xd must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting '
'or FEEDS.batch_item_count is specified and greater than 0. For more info see:'
'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'
''.format(uri_template)
)
return False
return True

2
scrapy/settings/default_settings.py
View File

@ -146,7 +146,7 @@ FEED_STORAGES_BASE = {
's3': 'scrapy.extensions.feedexport.S3FeedStorage',
'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
}
FEED_STORAGE_BATCH_ITEM_COUNT = 0
FEED_EXPORT_BATCH_ITEM_COUNT = 0
FEED_EXPORTERS = {}
FEED_EXPORTERS_BASE = {
'json': 'scrapy.exporters.JsonItemExporter',

5
scrapy/utils/conf.py
View File

@ -113,12 +113,9 @@ def get_sources(use_closest=True):
def feed_complete_default_values_from_settings(feed, settings):
out = feed.copy()
out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT'))
out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
out.setdefault(
"batch_item_count",
out.get('batch_item_count', settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT'))
)
out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
if settings["FEED_EXPORT_INDENT"] is None:

32
tests/test_feedexport.py
View File

@ -1108,7 +1108,7 @@ class FeedExportTest(FeedExportTestBase):
class BatchDeliveriesTest(FeedExportTestBase):
__test__ = True
_file_mark = '_%(batch_time)s_#%(batch_id)s_'
_file_mark = '_%(batch_time)s_#%(batch_id)02d_'
@defer.inlineCallbacks
def run_and_export(self, spider_cls, settings):
@ -1144,7 +1144,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
},
})
batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
for batch in data['jl']:
@ -1160,7 +1160,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
},
})
batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
data = yield self.exported_data(items, settings)
for batch in data['csv']:
got_batch = csv.DictReader(to_unicode(batch).splitlines())
@ -1176,7 +1176,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
},
})
batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
for batch in data['xml']:
@ -1194,7 +1194,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
},
})
batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
# XML
@ -1219,7 +1219,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
},
})
batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
import pickle
@ -1236,7 +1236,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
},
})
batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
import marshal
@ -1259,18 +1259,18 @@ class BatchDeliveriesTest(FeedExportTestBase):
{'foo': 'bar3', 'baz': 'quux3', 'egg': ''}
]
settings = {
'FEED_STORAGE_BATCH_ITEM_COUNT': 2
'FEED_EXPORT_BATCH_ITEM_COUNT': 2
}
header = self.MyItem.fields.keys()
yield self.assertExported(items, header, rows, settings=Settings(settings))
def test_wrong_path(self):
""" If path is without %(batch_time)s and %(batch_id)s an exception must be raised """
""" If path is without %(batch_time)s and %(batch_id)0xd an exception must be raised """
settings = {
'FEEDS': {
self._random_temp_filename(): {'format': 'xml'},
},
'FEED_STORAGE_BATCH_ITEM_COUNT': 1
'FEED_EXPORT_BATCH_ITEM_COUNT': 1
}
crawler = get_crawler(settings_dict=settings)
self.assertRaises(NotConfigured, FeedExporter, crawler)
@ -1282,7 +1282,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
'FEEDS': {
os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
},
'FEED_STORAGE_BATCH_ITEM_COUNT': 1
'FEED_EXPORT_BATCH_ITEM_COUNT': 1
}
data = yield self.exported_no_data(settings)
data = dict(data)
@ -1304,7 +1304,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
},
'FEED_STORE_EMPTY': True,
'FEED_EXPORT_INDENT': None,
'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
}
data = yield self.exported_no_data(settings)
data = dict(data)
@ -1352,7 +1352,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
'encoding': 'utf-8',
},
},
'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
}
data = yield self.exported_data(items, settings)
for fmt, expected in formats.items():
@ -1398,7 +1398,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
'format': 'json',
},
},
'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
}
data = yield self.exported_data(items, settings)
self.assertEqual(len(items) + 1, len(data['json']))
@ -1440,7 +1440,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
'format': 'json',
},
},
'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
})
items = [
self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
@ -1458,7 +1458,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
s3 = boto3.resource('s3')
my_bucket = s3.Bucket(s3_test_bucket_name)
batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
with MockServer() as s:
runner = CrawlerRunner(Settings(settings))

4
tests/test_utils_conf.py
View File

@ -149,7 +149,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
"FEED_EXPORT_INDENT": 42,
"FEED_STORE_EMPTY": True,
"FEED_URI_PARAMS": (1, 2, 3, 4),
"FEED_STORAGE_BATCH_ITEM_COUNT": 2,
"FEED_EXPORT_BATCH_ITEM_COUNT": 2,
})
new_feed = feed_complete_default_values_from_settings(feed, settings)
self.assertEqual(new_feed, {
@ -171,7 +171,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
"FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],
"FEED_EXPORT_INDENT": 42,
"FEED_STORE_EMPTY": True,
"FEED_STORAGE_BATCH_ITEM_COUNT": 2,
"FEED_EXPORT_BATCH_ITEM_COUNT": 2,
})
new_feed = feed_complete_default_values_from_settings(feed, settings)
self.assertEqual(new_feed, {