From 13c1c1faf81100394cffc6cf088c18c315f09ede Mon Sep 17 00:00:00 2001 From: Ayush Singh <135635937+Ayush442842q@users.noreply.github.com> Date: Thu, 4 Jun 2026 20:41:50 +0530 Subject: [PATCH] Close GCS feed temp files after upload (#7546) --- scrapy/extensions/feedexport.py | 13 ++++++++----- tests/test_feedexport_storages.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index d0430d0db..8029f85c9 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -306,12 +306,15 @@ class GCSFeedStorage(BlockingFeedStorage): def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) - from google.cloud.storage import Client # noqa: PLC0415 + try: + from google.cloud.storage import Client # noqa: PLC0415 - client = Client(project=self.project_id) - bucket = client.get_bucket(self.bucket_name) - blob = bucket.blob(self.blob_name) - blob.upload_from_file(file, predefined_acl=self.acl) + client = Client(project=self.project_id) + bucket = client.get_bucket(self.bucket_name) + blob = bucket.blob(self.blob_name) + blob.upload_from_file(file, predefined_acl=self.acl) + finally: + file.close() class FTPFeedStorage(BlockingFeedStorage): diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py index 5102f8e37..11e79775f 100644 --- a/tests/test_feedexport_storages.py +++ b/tests/test_feedexport_storages.py @@ -513,6 +513,34 @@ class TestGCSFeedStorage: client_mock.get_bucket.assert_called_once_with("mybucket") bucket_mock.blob.assert_called_once_with("export.csv") blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) + f.close.assert_called_once_with() + + @coroutine_test + async def test_store_closes_file_on_upload_error(self): + try: + from google.cloud.storage import Client # noqa: F401,PLC0415 + except ImportError: + pytest.skip("GCSFeedStorage requires google-cloud-storage") + + uri = "gs://mybucket/export.csv" + project_id = "myproject-123" + acl = "publicRead" + (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage() + blob_mock.upload_from_file.side_effect = OSError("Upload failed") + with mock.patch("google.cloud.storage.Client") as m: + m.return_value = client_mock + + f = mock.Mock() + storage = GCSFeedStorage(uri, project_id, acl) + with pytest.raises(OSError, match="Upload failed"): + await maybe_deferred_to_future(storage.store(f)) + + f.seek.assert_called_once_with(0) + m.assert_called_once_with(project=project_id) + client_mock.get_bucket.assert_called_once_with("mybucket") + bucket_mock.blob.assert_called_once_with("export.csv") + blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) + f.close.assert_called_once_with() def test_overwrite_default(self): with LogCapture() as log: