diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 72e210469..d5ed3121d 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -222,8 +222,8 @@ class GCSFilesStore: return {'checksum': checksum, 'last_modified': last_modified} else: return {} - - return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess) + blob_path = self._get_blob_path(path) + return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess) def _get_content_type(self, headers): if headers and 'Content-Type' in headers: @@ -231,8 +231,12 @@ class GCSFilesStore: else: return 'application/octet-stream' + def _get_blob_path(self, path): + return self.prefix + path + def persist_file(self, path, buf, info, meta=None, headers=None): - blob = self.bucket.blob(self.prefix + path) + blob_path = self._get_blob_path(path) + blob = self.bucket.blob(blob_path) blob.cache_control = self.CACHE_CONTROL blob.metadata = {k: str(v) for k, v in (meta or {}).items()} return threads.deferToThread( diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4e1b90787..0ff2045ed 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -525,6 +525,29 @@ class TestGCSFilesStore(unittest.TestCase): self.assertEqual(blob.content_type, 'application/octet-stream') self.assertIn(expected_policy, acl) + @defer.inlineCallbacks + def test_blob_path_consistency(self): + """Test to make sure that paths used to store files is the same as the one used to get + already uploaded files. + """ + assert_gcs_environ() + try: + import google.cloud.storage # noqa + except ModuleNotFoundError: + raise unittest.SkipTest("google-cloud-storage is not installed") + else: + with mock.patch('google.cloud.storage') as _: + with mock.patch('scrapy.pipelines.files.time') as _: + uri = 'gs://my_bucket/my_prefix/' + store = GCSFilesStore(uri) + store.bucket = mock.Mock() + path = 'full/my_data.txt' + yield store.persist_file(path, mock.Mock(), info=None, meta=None, headers=None) + yield store.stat_file(path, info=None) + expected_blob_path = store.prefix + path + store.bucket.blob.assert_called_with(expected_blob_path) + store.bucket.get_blob.assert_called_with(expected_blob_path) + class TestFTPFileStore(unittest.TestCase): @defer.inlineCallbacks diff --git a/tox.ini b/tox.ini index db151f215..6951b6d16 100644 --- a/tox.ini +++ b/tox.ini @@ -12,10 +12,11 @@ deps = -rtests/requirements.txt # mitmproxy does not support PyPy # mitmproxy does not support Windows when running Python < 3.7 - # Python 3.9+ requires https://github.com/mitmproxy/mitmproxy/commit/8e5e43de24c9bc93092b63efc67fbec029a9e7fe + # Python 3.9+ requires mitmproxy >= 5.3.0 # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0 #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' - mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' + # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454 + mitmproxy >= 4.0.4, < 8; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower) markupsafe < 2.1.0; python_version >= '3.6' and python_version < '3.8' and implementation_name != 'pypy' @@ -125,13 +126,14 @@ setenv = deps = {[testenv]deps} boto + google-cloud-storage + # Twisted[http2] currently forces old mitmproxy because of h2 version + # restrictions in their deps, so we need to pin old markupsafe here too. + markupsafe < 2.1.0 reppy robotexclusionrulesparser Pillow>=4.0.0 Twisted[http2]>=17.9.0 - # Twisted[http2] currently forces old mitmproxy because of h2 version restrictions in their deps, - # so we need to pin old markupsafe here too - markupsafe < 2.1.0 [testenv:asyncio] commands =