From dad2ea75222d6240c569440d3221f5fc00925682 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Sat, 2 May 2020 01:21:03 +0300 Subject: [PATCH] Change time_id to batch_time --- docs/topics/feed-exports.rst | 6 +++--- scrapy/extensions/feedexport.py | 10 +++++----- tests/test_feedexport.py | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 6c463fc27..2106b41f5 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -441,9 +441,9 @@ An integer number which represent number of scraped items stored in each output file. Whenever the number of items exceeds this setting, a new file is created and output redirects to it. The name of the new file will be selected based on timestamp when the feed is being created and/or batch sequence number. -Therefore you must specify %(time_id)s or %(batch_id)s or both in FEED_URI. +Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI. -* ``%(time_id)s`` - gets replaced by a timestamp when the feed is being created +* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created * ``%(batch_id)s`` - gets replaced by sequence number of batch For instance:: @@ -452,7 +452,7 @@ For instance:: Your request can be like:: - scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time_id)s.json + scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json The result directory tree of above can be like:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index fe6061c33..a262f5d18 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -295,7 +295,7 @@ class FeedExporter: :param uri: uri of the new batch to start :param feed: dict with parameters of feed :param spider: user spider - :param template_uri: template uri which contains %(time_id)s or %(batch_id)s to create new uri + :param template_uri: template uri which contains %(batch_time)s or %(batch_id)s to create new uri """ storage = self._get_storage(uri) file = storage.open(spider) @@ -358,12 +358,12 @@ class FeedExporter: def _batch_deliveries_supported(self, uri): """ - If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time_id)s or %(batch_id)s + If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ - if self.storage_batch_size is None or '%(time_id)s' in uri or '%(batch_id)s' in uri: + if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri: return True - logger.warning('%(time_id)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') + logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') return False def _storage_supported(self, uri): @@ -396,7 +396,7 @@ class FeedExporter: for k in dir(spider): params[k] = getattr(spider, k) params['time'] = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-') - params['time_id'] = datetime.utcnow().isoformat().replace(':', '-') + params['batch_time'] = datetime.utcnow().isoformat().replace(':', '-') params['batch_id'] = slot.batch_id + 1 if slot is not None else 1 uripar_function = load_object(uri_params) if uri_params else lambda x, y: None uripar_function(params, spider) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 9fc39c3a6..2217bb4ed 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -989,7 +989,7 @@ class FeedExportTest(FeedExportTestBase): class PartialDeliveriesTest(FeedExportTestBase): __test__ = True - _file_mark = '_%(time_id)s_#%(batch_id)s_' + _file_mark = '_%(batch_time)s_#%(batch_id)s_' @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): @@ -1146,7 +1146,7 @@ class PartialDeliveriesTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): - """ If path is without %(time_id)s or %(batch_id)s an exception must be raised """ + """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """ settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, @@ -1236,7 +1236,7 @@ class PartialDeliveriesTest(FeedExportTestBase): def test_batch_path_differ(self): """ Test that the name of all batch files differ from each other. - So %(time_id)s replaced with the current date. + So %(batch_time)s replaced with the current date. """ items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), @@ -1245,7 +1245,7 @@ class PartialDeliveriesTest(FeedExportTestBase): ] settings = { 'FEEDS': { - os.path.join(self._random_temp_filename(), '%(time_id)s'): { + os.path.join(self._random_temp_filename(), '%(batch_time)s'): { 'format': 'json', }, }, @@ -1281,7 +1281,7 @@ class PartialDeliveriesTest(FeedExportTestBase): chars = [random.choice(ascii_letters + digits) for _ in range(15)] filename = ''.join(chars) prefix = 'tmp/{filename}'.format(filename=filename) - s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(time_id)s.json'.format( + s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(batch_time)s.json'.format( bucket_name=s3_test_bucket_name, prefix=prefix ) storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key)