diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index cd26b5778..da1a88299 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -383,13 +383,20 @@ class FeedExporter: return defer.DeferredList(deferred_list) if deferred_list else None def _close_slot(self, slot, spider): + def get_file(slot_): + if isinstance(slot_.file, PostProcessingManager): + slot_.file.close() + return slot_.file.file + return slot_.file + slot.finish_exporting() if not slot.itemcount and not slot.store_empty: # We need to call slot.storage.store nonetheless to get the file # properly closed. - return defer.maybeDeferred(slot.storage.store, slot.file) + return defer.maybeDeferred(slot.storage.store, get_file(slot)) + logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" - d = defer.maybeDeferred(slot.storage.store, slot.file) + d = defer.maybeDeferred(slot.storage.store, get_file(slot)) d.addCallback( self._handle_store_success, logmsg, spider, type(slot.storage).__name__ diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index eafe1b334..3124d9d67 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1638,6 +1638,57 @@ class FeedExportTest(FeedExportTestBase): data = yield self.exported_data(items, settings) self.assertEqual(row["expected"], data[feed_options["format"]]) + @defer.inlineCallbacks + def test_storage_file_no_postprocessing(self): + @implementer(IFeedStorage) + class Storage: + def __init__(self, uri, *, feed_options=None): + pass + + def open(self, spider): + Storage.open_file = tempfile.NamedTemporaryFile(prefix="feed-") + return Storage.open_file + + def store(self, file): + Storage.store_file = file + file.close() + + settings = { + "FEEDS": {self._random_temp_filename(): {"format": "jsonlines"}}, + "FEED_STORAGES": {"file": Storage}, + } + yield self.exported_no_data(settings) + self.assertIs(Storage.open_file, Storage.store_file) + + @defer.inlineCallbacks + def test_storage_file_postprocessing(self): + @implementer(IFeedStorage) + class Storage: + def __init__(self, uri, *, feed_options=None): + pass + + def open(self, spider): + Storage.open_file = tempfile.NamedTemporaryFile(prefix="feed-") + return Storage.open_file + + def store(self, file): + Storage.store_file = file + file.close() + + settings = { + "FEEDS": { + self._random_temp_filename(): { + "format": "jsonlines", + "postprocessing": [ + "scrapy.extensions.postprocessing.GzipPlugin", + ], + }, + }, + "FEED_STORAGES": {"file": Storage}, + } + yield self.exported_no_data(settings) + self.assertIs(Storage.open_file, Storage.store_file) + class FeedPostProcessedExportsTest(FeedExportTestBase): __test__ = True