diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index e89786103..db1f5c419 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -34,11 +34,23 @@ _T = TypeVar("_T") def assert_gcs_environ() -> None: + warnings.warn( + "The assert_gcs_environ() function is deprecated and will be removed in a future version of Scrapy." + " Check GCS_PROJECT_ID directly.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) if "GCS_PROJECT_ID" not in os.environ: raise SkipTest("GCS_PROJECT_ID not found") def skip_if_no_boto() -> None: + warnings.warn( + "The skip_if_no_boto() function is deprecated and will be removed in a future version of Scrapy." + " Check scrapy.utils.boto.is_botocore_available() directly.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) if not is_botocore_available(): raise SkipTest("missing botocore library") @@ -48,6 +60,11 @@ def get_gcs_content_and_delete( ) -> tuple[bytes, list[dict[str, str]], Any]: from google.cloud import storage + warnings.warn( + "The get_gcs_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) client = storage.Client(project=os.environ.get("GCS_PROJECT_ID")) bucket = client.get_bucket(bucket) blob = bucket.get_blob(path) @@ -67,6 +84,11 @@ def get_ftp_content_and_delete( ) -> bytes: from ftplib import FTP + warnings.warn( + "The get_ftp_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) ftp = FTP() ftp.connect(host, port) ftp.login(username, password) @@ -150,6 +172,12 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]: """ from google.cloud.storage import Blob, Bucket, Client + warnings.warn( + "The mock_google_cloud_storage() function is deprecated and will be removed in a future version of Scrapy.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + client_mock = mock.create_autospec(Client) bucket_mock = mock.create_autospec(Bucket) diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 85d7c940f..10f764ab8 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -2,18 +2,27 @@ from __future__ import annotations import os import sys +import warnings from typing import TYPE_CHECKING, cast from twisted.internet.defer import Deferred from twisted.internet.error import ProcessTerminated from twisted.internet.protocol import ProcessProtocol +from scrapy.exceptions import ScrapyDeprecationWarning + if TYPE_CHECKING: from collections.abc import Iterable from twisted.python.failure import Failure +warnings.warn( + "The scrapy.utils.testproc module is deprecated.", + ScrapyDeprecationWarning, +) + + class ProcessTest: command: str | None = None prefix = [sys.executable, "-m", "scrapy.cmdline"] diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index ca1f68116..f12b301fd 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -1,7 +1,15 @@ +import warnings from urllib.parse import urljoin from twisted.web import resource, server, static, util +from scrapy.exceptions import ScrapyDeprecationWarning + +warnings.warn( + "The scrapy.utils.testsite module is deprecated.", + ScrapyDeprecationWarning, +) + class SiteTest: def setUp(self): @@ -48,7 +56,7 @@ def test_site(): if __name__ == "__main__": - from twisted.internet import reactor + from twisted.internet import reactor # pylint: disable=ungrouped-imports port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") print(f"http://localhost:{port.getHost().port}/") diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index d2027d1c2..a4d7fdd30 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -1,8 +1,8 @@ from twisted.internet import defer from twisted.trial import unittest -from scrapy.utils.testproc import ProcessTest -from scrapy.utils.testsite import SiteTest +from tests.utils.testproc import ProcessTest +from tests.utils.testsite import SiteTest class FetchTest(ProcessTest, SiteTest, unittest.TestCase): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 9356d6b79..9f2c7fa13 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -7,9 +7,9 @@ from twisted.internet import defer from scrapy.commands import parse from scrapy.settings import Settings from scrapy.utils.python import to_unicode -from scrapy.utils.testproc import ProcessTest -from scrapy.utils.testsite import SiteTest from tests.test_commands import CommandTest +from tests.utils.testproc import ProcessTest +from tests.utils.testsite import SiteTest def _textmode(bstr): diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 7918d94b2..9ca5e05dc 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -7,10 +7,10 @@ from pexpect.popen_spawn import PopenSpawn from twisted.internet import defer from twisted.trial import unittest -from scrapy.utils.testproc import ProcessTest -from scrapy.utils.testsite import SiteTest from tests import NON_EXISTING_RESOLVABLE, tests_datadir from tests.mockserver import MockServer +from tests.utils.testproc import ProcessTest +from tests.utils.testsite import SiteTest class ShellTest(ProcessTest, SiteTest, unittest.TestCase): diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 18c1c531c..917f457cb 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -4,7 +4,7 @@ from twisted.internet import defer from twisted.trial import unittest import scrapy -from scrapy.utils.testproc import ProcessTest +from tests.utils.testproc import ProcessTest class VersionTest(ProcessTest, unittest.TestCase): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index b4c1b9631..8e008ab98 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -17,7 +17,7 @@ from io import BytesIO from logging import getLogger from pathlib import Path from string import ascii_letters, digits -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from unittest import mock from urllib.parse import quote, urljoin from urllib.request import pathname2url @@ -48,7 +48,7 @@ from scrapy.extensions.feedexport import ( ) from scrapy.settings import Settings from scrapy.utils.python import to_unicode -from scrapy.utils.test import get_crawler, mock_google_cloud_storage +from scrapy.utils.test import get_crawler from tests.mockserver import MockFTPServer, MockServer from tests.spiders import ItemSpider @@ -71,6 +71,23 @@ def build_url(path: str | PathLike) -> str: return urljoin("file:", path_str) +def mock_google_cloud_storage() -> tuple[Any, Any, Any]: + """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob + classes and set their proper return values. + """ + from google.cloud.storage import Blob, Bucket, Client + + client_mock = mock.create_autospec(Client) + + bucket_mock = mock.create_autospec(Bucket) + client_mock.get_bucket.return_value = bucket_mock + + blob_mock = mock.create_autospec(Blob) + bucket_mock.blob.return_value = blob_mock + + return (client_mock, bucket_mock, blob_mock) + + class FileFeedStorageTest(unittest.TestCase): def test_store_file_uri(self): path = Path(self.mktemp()).resolve() diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4c59fcfb7..05fd17207 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -6,8 +6,10 @@ import warnings from datetime import datetime from io import BytesIO from pathlib import Path +from posixpath import split from shutil import rmtree from tempfile import mkdtemp +from typing import Any from unittest import mock from urllib.parse import urlparse @@ -27,16 +29,54 @@ from scrapy.pipelines.files import ( S3FilesStore, ) from scrapy.utils.test import ( - assert_gcs_environ, get_crawler, - get_ftp_content_and_delete, - get_gcs_content_and_delete, ) from tests.mockserver import MockFTPServer from .test_pipeline_media import _mocked_download_func +def get_gcs_content_and_delete( + bucket: Any, path: str +) -> tuple[bytes, list[dict[str, str]], Any]: + from google.cloud import storage + + client = storage.Client(project=os.environ.get("GCS_PROJECT_ID")) + bucket = client.get_bucket(bucket) + blob = bucket.get_blob(path) + content = blob.download_as_string() + acl = list(blob.acl) # loads acl before it will be deleted + bucket.delete_blob(path) + return content, acl, blob + + +def get_ftp_content_and_delete( + path: str, + host: str, + port: int, + username: str, + password: str, + use_active_mode: bool = False, +) -> bytes: + from ftplib import FTP + + ftp = FTP() + ftp.connect(host, port) + ftp.login(username, password) + if use_active_mode: + ftp.set_pasv(False) + ftp_data: list[bytes] = [] + + def buffer_data(data: bytes) -> None: + ftp_data.append(data) + + ftp.retrbinary(f"RETR {path}", buffer_data) + dirname, filename = split(path) + ftp.cwd(dirname) + ftp.delete(filename) + return b"".join(ftp_data) + + class FilesPipelineTestCase(unittest.TestCase): def setUp(self): self.tempdir = mkdtemp() @@ -597,10 +637,12 @@ class TestS3FilesStore(unittest.TestCase): stub.assert_no_pending_responses() +@pytest.mark.skipif( + "GCS_PROJECT_ID" not in os.environ, reason="GCS_PROJECT_ID not found" +) class TestGCSFilesStore(unittest.TestCase): @defer.inlineCallbacks def test_persist(self): - assert_gcs_environ() uri = os.environ.get("GCS_TEST_FILE_URI") if not uri: raise unittest.SkipTest("No GCS URI available for testing") @@ -629,7 +671,6 @@ class TestGCSFilesStore(unittest.TestCase): """Test to make sure that paths used to store files is the same as the one used to get already uploaded files. """ - assert_gcs_environ() try: import google.cloud.storage # noqa: F401 except ModuleNotFoundError: diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/utils/testproc.py b/tests/utils/testproc.py new file mode 100644 index 000000000..85d7c940f --- /dev/null +++ b/tests/utils/testproc.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import os +import sys +from typing import TYPE_CHECKING, cast + +from twisted.internet.defer import Deferred +from twisted.internet.error import ProcessTerminated +from twisted.internet.protocol import ProcessProtocol + +if TYPE_CHECKING: + from collections.abc import Iterable + + from twisted.python.failure import Failure + + +class ProcessTest: + command: str | None = None + prefix = [sys.executable, "-m", "scrapy.cmdline"] + cwd = os.getcwd() # trial chdirs to temp dir # noqa: PTH109 + + def execute( + self, + args: Iterable[str], + check_code: bool = True, + settings: str | None = None, + ) -> Deferred[TestProcessProtocol]: + from twisted.internet import reactor + + env = os.environ.copy() + if settings is not None: + env["SCRAPY_SETTINGS_MODULE"] = settings + assert self.command + cmd = [*self.prefix, self.command, *args] + pp = TestProcessProtocol() + pp.deferred.addCallback(self._process_finished, cmd, check_code) + reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd) + return pp.deferred + + def _process_finished( + self, pp: TestProcessProtocol, cmd: list[str], check_code: bool + ) -> tuple[int, bytes, bytes]: + if pp.exitcode and check_code: + msg = f"process {cmd} exit with code {pp.exitcode}" + msg += f"\n>>> stdout <<<\n{pp.out.decode()}" + msg += "\n" + msg += f"\n>>> stderr <<<\n{pp.err.decode()}" + raise RuntimeError(msg) + return cast(int, pp.exitcode), pp.out, pp.err + + +class TestProcessProtocol(ProcessProtocol): + def __init__(self) -> None: + self.deferred: Deferred[TestProcessProtocol] = Deferred() + self.out: bytes = b"" + self.err: bytes = b"" + self.exitcode: int | None = None + + def outReceived(self, data: bytes) -> None: + self.out += data + + def errReceived(self, data: bytes) -> None: + self.err += data + + def processEnded(self, status: Failure) -> None: + self.exitcode = cast(ProcessTerminated, status.value).exitCode + self.deferred.callback(self) diff --git a/tests/utils/testsite.py b/tests/utils/testsite.py new file mode 100644 index 000000000..473738773 --- /dev/null +++ b/tests/utils/testsite.py @@ -0,0 +1,47 @@ +from urllib.parse import urljoin + +from twisted.web import resource, server, static, util + + +class SiteTest: + def setUp(self): + from twisted.internet import reactor + + super().setUp() + self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1") + self.baseurl = f"http://localhost:{self.site.getHost().port}/" + + def tearDown(self): + super().tearDown() + self.site.stopListening() + + def url(self, path: str) -> str: + return urljoin(self.baseurl, path) + + +class NoMetaRefreshRedirect(util.Redirect): + def render(self, request: server.Request) -> bytes: + content = util.Redirect.render(self, request) + return content.replace( + b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"' + ) + + +def test_site(): + r = resource.Resource() + r.putChild(b"text", static.Data(b"Works", "text/plain")) + r.putChild( + b"html", + static.Data( + b"
Works
World
", + "text/html", + ), + ) + r.putChild( + b"enc-gb18030", + static.Data(b"gb18030 encoding
", "text/html; charset=gb18030"), + ) + r.putChild(b"redirect", util.Redirect(b"/redirected")) + r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")) + r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) + return server.Site(r)