From 7b1d3c35ea3bfde2ac7fc69a2a26bbcb94aec1bf Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Wed, 1 Jul 2020 11:54:39 +0300 Subject: [PATCH] Minor updates --- docs/topics/feed-exports.rst | 4 ++-- scrapy/extensions/feedexport.py | 34 +++++++++++++++++---------------- scrapy/utils/conf.py | 4 ++++ tests/test_feedexport.py | 23 ++++++---------------- tests/test_utils_conf.py | 4 ++++ 5 files changed, 34 insertions(+), 35 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 3da56821e..0b659f30e 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -442,7 +442,7 @@ If assigned an integer number higher than ``0``, Scrapy generates multiple outpu storing up to the specified number of items in each output file. When generating multiple output files, you must use at least one of the following -placeholders in :setting:`FEED_URI` to indicate how the different output file names are +placeholders in the feed URI to indicate how the different output file names are generated: * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created @@ -457,7 +457,7 @@ For instance, if your settings include:: And your :command:`crawl` command line is:: - scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json + scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json The command line above can generate a directory tree like:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 2312c994e..5908987a3 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -242,7 +242,6 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - self.storage_batch_item_count = self.settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured @@ -253,7 +252,7 @@ class FeedExporter: def open_spider(self, spider): for uri, feed in self.feeds.items(): - uri_params = self._get_uri_params(spider, feed['uri_params'], None) + uri_params = self._get_uri_params(spider, feed['uri_params']) self.slots.append(self._start_new_batch( batch_id=1, uri=uri % uri_params, @@ -299,7 +298,7 @@ class FeedExporter: def _start_new_batch(self, batch_id, uri, feed, spider, uri_template): """ Redirect the output data stream to a new file. - Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified. + Execute multiple times if FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified :param batch_id: sequence number of current batch :param uri: uri of the new batch to start :param feed: dict with parameters of feed @@ -331,14 +330,15 @@ class FeedExporter: def item_scraped(self, item, spider): slots = [] - for idx, slot in enumerate(self.slots): + for slot in self.slots: slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one - if self.feeds[slot.uri_template].get('batch_item_count', self.storage_batch_item_count) \ - and slot.itemcount == self.feeds[slot.uri_template].get('batch_item_count', - self.storage_batch_item_count): + if ( + self.feeds[slot.uri_template]['batch_item_count'] + and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count'] + ): uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot) self._close_slot(slot, spider) slots.append(self._start_new_batch( @@ -369,15 +369,17 @@ class FeedExporter: def _settings_are_valid(self, uri): """ - If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s - to distinguish different files of partial output + If FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain + %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ - if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri: - return True - logger.error( - '%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified' - ) - return False + for uri_template, values in self.feeds.items(): + if values['batch_item_count'] and not any(s in uri_template for s in ['%(batch_time)s', '%(batch_id)s']): + logger.error( + '%(batch_time)s or %(batch_id)s must be in uri({}) if FEED_STORAGE_BATCH_ITEM_COUNT setting ' + 'or FEEDS.batch_item_count is specified and greater than 0.'.format(uri_template) + ) + return False + return True def _storage_supported(self, uri): scheme = urlparse(uri).scheme @@ -404,7 +406,7 @@ class FeedExporter: def _get_storage(self, uri): return self._get_instance(self.storages[urlparse(uri).scheme], uri) - def _get_uri_params(self, spider, uri_params, slot): + def _get_uri_params(self, spider, uri_params, slot=None): params = {} for k in dir(spider): params[k] = getattr(spider, k) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 5921f82bf..0e02f0f28 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -115,6 +115,10 @@ def feed_complete_default_values_from_settings(feed, settings): out = feed.copy() out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None) + out.setdefault( + "batch_item_count", + out.get('batch_item_count', settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')) + ) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) out.setdefault("uri_params", settings["FEED_URI_PARAMS"]) if settings["FEED_EXPORT_INDENT"] is None: diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 542cce70f..db14b20b9 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1265,7 +1265,7 @@ class BatchDeliveriesTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=Settings(settings)) def test_wrong_path(self): - """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """ + """ If path is without %(batch_time)s and %(batch_id)s an exception must be raised """ settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, @@ -1329,7 +1329,6 @@ class BatchDeliveriesTest(FeedExportTestBase): ], 'csv': ['foo,bar\r\nFOO,BAR\r\n'.encode('utf-8'), 'foo,bar\r\nFOO1,BAR1\r\n'.encode('utf-8')], - 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')], } settings = { @@ -1352,13 +1351,6 @@ class BatchDeliveriesTest(FeedExportTestBase): 'fields': ['foo', 'bar'], 'encoding': 'utf-8', }, - os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): { - 'format': 'jsonlines', - 'indent': None, - 'fields': ['foo', 'bar'], - 'encoding': 'utf-8', - 'batch_item_count': 0, - }, }, 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } @@ -1369,19 +1361,16 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_batch_item_count_feeds_setting(self): - items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})] - + items = [dict({'foo': u'FOO'}), dict({'foo': u'FOO1'})] formats = { - 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n'.encode('utf-8'), - '{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')], + 'json': ['[{"foo": "FOO"}]'.encode('utf-8'), + '[{"foo": "FOO1"}]'.encode('utf-8')], } - settings = { 'FEEDS': { - os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): { - 'format': 'jsonlines', + os.path.join(self._random_temp_filename(), 'json', self._file_mark): { + 'format': 'json', 'indent': None, - 'fields': ['foo', 'bar'], 'encoding': 'utf-8', 'batch_item_count': 1, }, diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index e5d3ef582..95ec2b64a 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -149,6 +149,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "FEED_EXPORT_INDENT": 42, "FEED_STORE_EMPTY": True, "FEED_URI_PARAMS": (1, 2, 3, 4), + "FEED_STORAGE_BATCH_ITEM_COUNT": 2, }) new_feed = feed_complete_default_values_from_settings(feed, settings) self.assertEqual(new_feed, { @@ -157,6 +158,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "indent": 42, "store_empty": True, "uri_params": (1, 2, 3, 4), + "batch_item_count": 2, }) def test_feed_complete_default_values_from_settings_non_empty(self): @@ -169,6 +171,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"], "FEED_EXPORT_INDENT": 42, "FEED_STORE_EMPTY": True, + "FEED_STORAGE_BATCH_ITEM_COUNT": 2, }) new_feed = feed_complete_default_values_from_settings(feed, settings) self.assertEqual(new_feed, { @@ -177,6 +180,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "indent": 42, "store_empty": True, "uri_params": None, + "batch_item_count": 2, })