From 1e245046ed8ac3d9f89860501c2da95b69aaabf6 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Thu, 2 Jul 2020 12:38:08 +0300 Subject: [PATCH] Change setting name. Add leading zeroes to batch_id. Minor fixes. --- docs/topics/feed-exports.rst | 19 +++++++++-------- scrapy/extensions/feedexport.py | 24 +++++++++++++--------- scrapy/settings/default_settings.py | 2 +- scrapy/utils/conf.py | 5 +---- tests/test_feedexport.py | 32 ++++++++++++++--------------- tests/test_utils_conf.py | 4 ++-- 6 files changed, 45 insertions(+), 41 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 0b659f30e..56efa80a7 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -220,7 +220,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_STORAGE_FTP_ACTIVE` * :setting:`FEED_STORAGE_S3_ACL` * :setting:`FEED_EXPORTERS` - * :setting:`FEED_STORAGE_BATCH_ITEM_COUNT` + * :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` .. currentmodule:: scrapy.extensions.feedexport @@ -272,7 +272,7 @@ as a fallback value if that key is not provided for a specific feed definition. * ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS` * ``indent``: falls back to :setting:`FEED_EXPORT_INDENT` * ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY` -* ``batch_item_count``: falls back to :setting:`FEED_STORAGE_BATCH_ITEM_COUNT` +* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` .. setting:: FEED_EXPORT_ENCODING @@ -432,9 +432,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. _botocore: https://github.com/boto/botocore .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl -.. setting:: FEED_STORAGE_BATCH_ITEM_COUNT +.. setting:: FEED_EXPORT_BATCH_ITEM_COUNT -FEED_STORAGE_BATCH_ITEM_COUNT +FEED_EXPORT_BATCH_ITEM_COUNT ----------------------------- Default: ``0`` @@ -448,16 +448,19 @@ generated: * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created (e.g. ``2020-03-28T14-45-08.237134``) -* ``%(batch_id)s`` - gets replaced by the batch sequence number of batch - (e.g. ``2`` for the second file) +* ``%(batch_id)0xd`` - gets replaced by the sequence number of the batch. +By replacing ``x`` with an integer you set the number of leading zeroes to prevent +inappropriate sorting like this: [``'1'``, ``'10'``, ``'2'``]. Here are some examples: + ``%(batch_id)01d`` for the second batch gets replaced by ``2`` + ``%(batch_id)05d`` for the third batch gets replaced by ``00003`` For instance, if your settings include:: - FEED_STORAGE_BATCH_ITEM_COUNT=100 + FEED_EXPORT_BATCH_ITEM_COUNT=100 And your :command:`crawl` command line is:: - scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json + scrapy crawl spidername -o dirname/%(batch_id)d-filename%(batch_time)s.json The command line above can generate a directory tree like:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 5908987a3..adb6ea2e4 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -6,6 +6,7 @@ See documentation in docs/topics/feed-exports.rst import logging import os +import re import sys import warnings from datetime import datetime @@ -25,6 +26,7 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import without_none_values + logger = logging.getLogger(__name__) @@ -245,7 +247,7 @@ class FeedExporter: for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured - if not self._settings_are_valid(uri): + if not self._settings_are_valid(): raise NotConfigured if not self._exporter_supported(feed['format']): raise NotConfigured @@ -298,7 +300,7 @@ class FeedExporter: def _start_new_batch(self, batch_id, uri, feed, spider, uri_template): """ Redirect the output data stream to a new file. - Execute multiple times if FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified + Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified :param batch_id: sequence number of current batch :param uri: uri of the new batch to start :param feed: dict with parameters of feed @@ -334,10 +336,10 @@ class FeedExporter: slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 - # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one + # create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one if ( - self.feeds[slot.uri_template]['batch_item_count'] - and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count'] + self.feeds[slot.uri_template]['batch_item_count'] + and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count'] ): uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot) self._close_slot(slot, spider) @@ -367,16 +369,18 @@ class FeedExporter: return True logger.error("Unknown feed format: %(format)s", {'format': format}) - def _settings_are_valid(self, uri): + def _settings_are_valid(self): """ - If FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain + If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ for uri_template, values in self.feeds.items(): - if values['batch_item_count'] and not any(s in uri_template for s in ['%(batch_time)s', '%(batch_id)s']): + if values['batch_item_count'] and not re.findall(r'(%\(batch_time\)s|(%\(batch_id\)0\d*d))', uri_template): logger.error( - '%(batch_time)s or %(batch_id)s must be in uri({}) if FEED_STORAGE_BATCH_ITEM_COUNT setting ' - 'or FEEDS.batch_item_count is specified and greater than 0.'.format(uri_template) + '%(batch_time)s or %(batch_id)0xd must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting ' + 'or FEEDS.batch_item_count is specified and greater than 0. For more info see:' + 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count' + ''.format(uri_template) ) return False return True diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 810acd5a3..0016bbe1b 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -146,7 +146,7 @@ FEED_STORAGES_BASE = { 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } -FEED_STORAGE_BATCH_ITEM_COUNT = 0 +FEED_EXPORT_BATCH_ITEM_COUNT = 0 FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 0e02f0f28..64f9c824b 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -113,12 +113,9 @@ def get_sources(use_closest=True): def feed_complete_default_values_from_settings(feed, settings): out = feed.copy() + out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')) out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None) - out.setdefault( - "batch_item_count", - out.get('batch_item_count', settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')) - ) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) out.setdefault("uri_params", settings["FEED_URI_PARAMS"]) if settings["FEED_EXPORT_INDENT"] is None: diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index db14b20b9..d20b40e2f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1108,7 +1108,7 @@ class FeedExportTest(FeedExportTestBase): class BatchDeliveriesTest(FeedExportTestBase): __test__ = True - _file_mark = '_%(batch_time)s_#%(batch_id)s_' + _file_mark = '_%(batch_time)s_#%(batch_id)02d_' @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): @@ -1144,7 +1144,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['jl']: @@ -1160,7 +1160,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') data = yield self.exported_data(items, settings) for batch in data['csv']: got_batch = csv.DictReader(to_unicode(batch).splitlines()) @@ -1176,7 +1176,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['xml']: @@ -1194,7 +1194,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) # XML @@ -1219,7 +1219,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import pickle @@ -1236,7 +1236,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import marshal @@ -1259,18 +1259,18 @@ class BatchDeliveriesTest(FeedExportTestBase): {'foo': 'bar3', 'baz': 'quux3', 'egg': ''} ] settings = { - 'FEED_STORAGE_BATCH_ITEM_COUNT': 2 + 'FEED_EXPORT_BATCH_ITEM_COUNT': 2 } header = self.MyItem.fields.keys() yield self.assertExported(items, header, rows, settings=Settings(settings)) def test_wrong_path(self): - """ If path is without %(batch_time)s and %(batch_id)s an exception must be raised """ + """ If path is without %(batch_time)s and %(batch_id)0xd an exception must be raised """ settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, }, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1 + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1 } crawler = get_crawler(settings_dict=settings) self.assertRaises(NotConfigured, FeedExporter, crawler) @@ -1282,7 +1282,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'FEEDS': { os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, }, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1 + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1 } data = yield self.exported_no_data(settings) data = dict(data) @@ -1304,7 +1304,7 @@ class BatchDeliveriesTest(FeedExportTestBase): }, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, } data = yield self.exported_no_data(settings) data = dict(data) @@ -1352,7 +1352,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'encoding': 'utf-8', }, }, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, } data = yield self.exported_data(items, settings) for fmt, expected in formats.items(): @@ -1398,7 +1398,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'format': 'json', }, }, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, } data = yield self.exported_data(items, settings) self.assertEqual(len(items) + 1, len(data['json'])) @@ -1440,7 +1440,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'format': 'json', }, }, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, }) items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), @@ -1458,7 +1458,7 @@ class BatchDeliveriesTest(FeedExportTestBase): s3 = boto3.resource('s3') my_bucket = s3.Bucket(s3_test_bucket_name) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') with MockServer() as s: runner = CrawlerRunner(Settings(settings)) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 95ec2b64a..f3ef36127 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -149,7 +149,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "FEED_EXPORT_INDENT": 42, "FEED_STORE_EMPTY": True, "FEED_URI_PARAMS": (1, 2, 3, 4), - "FEED_STORAGE_BATCH_ITEM_COUNT": 2, + "FEED_EXPORT_BATCH_ITEM_COUNT": 2, }) new_feed = feed_complete_default_values_from_settings(feed, settings) self.assertEqual(new_feed, { @@ -171,7 +171,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"], "FEED_EXPORT_INDENT": 42, "FEED_STORE_EMPTY": True, - "FEED_STORAGE_BATCH_ITEM_COUNT": 2, + "FEED_EXPORT_BATCH_ITEM_COUNT": 2, }) new_feed = feed_complete_default_values_from_settings(feed, settings) self.assertEqual(new_feed, {