Move most of the test utils inside tests.

scrapy/utils/test.py

@@ -34,11 +34,23 @@ _T = TypeVar("_T")
 
 
 def assert_gcs_environ() -> None:
+    warnings.warn(
+        "The assert_gcs_environ() function is deprecated and will be removed in a future version of Scrapy."
+        " Check GCS_PROJECT_ID directly.",
+        category=ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
     if "GCS_PROJECT_ID" not in os.environ:
         raise SkipTest("GCS_PROJECT_ID not found")
 
 
 def skip_if_no_boto() -> None:
+    warnings.warn(
+        "The skip_if_no_boto() function is deprecated and will be removed in a future version of Scrapy."
+        " Check scrapy.utils.boto.is_botocore_available() directly.",
+        category=ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
     if not is_botocore_available():
         raise SkipTest("missing botocore library")
 
@@ -48,6 +60,11 @@ def get_gcs_content_and_delete(
 ) -> tuple[bytes, list[dict[str, str]], Any]:
     from google.cloud import storage
 
+    warnings.warn(
+        "The get_gcs_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.",
+        category=ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
     client = storage.Client(project=os.environ.get("GCS_PROJECT_ID"))
     bucket = client.get_bucket(bucket)
     blob = bucket.get_blob(path)
@@ -67,6 +84,11 @@ def get_ftp_content_and_delete(
 ) -> bytes:
     from ftplib import FTP
 
+    warnings.warn(
+        "The get_ftp_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.",
+        category=ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
     ftp = FTP()
     ftp.connect(host, port)
     ftp.login(username, password)
@@ -150,6 +172,12 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]:
     """
     from google.cloud.storage import Blob, Bucket, Client
 
+    warnings.warn(
+        "The mock_google_cloud_storage() function is deprecated and will be removed in a future version of Scrapy.",
+        category=ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
+
     client_mock = mock.create_autospec(Client)
 
     bucket_mock = mock.create_autospec(Bucket)

scrapy/utils/testproc.py

@@ -2,18 +2,27 @@ from __future__ import annotations
 
 import os
 import sys
+import warnings
 from typing import TYPE_CHECKING, cast
 
 from twisted.internet.defer import Deferred
 from twisted.internet.error import ProcessTerminated
 from twisted.internet.protocol import ProcessProtocol
 
+from scrapy.exceptions import ScrapyDeprecationWarning
+
 if TYPE_CHECKING:
     from collections.abc import Iterable
 
     from twisted.python.failure import Failure
 
 
+warnings.warn(
+    "The scrapy.utils.testproc module is deprecated.",
+    ScrapyDeprecationWarning,
+)
+
+
 class ProcessTest:
     command: str | None = None
     prefix = [sys.executable, "-m", "scrapy.cmdline"]

scrapy/utils/testsite.py

@@ -1,7 +1,15 @@
+import warnings
 from urllib.parse import urljoin
 
 from twisted.web import resource, server, static, util
 
+from scrapy.exceptions import ScrapyDeprecationWarning
+
+warnings.warn(
+    "The scrapy.utils.testsite module is deprecated.",
+    ScrapyDeprecationWarning,
+)
+
 
 class SiteTest:
     def setUp(self):
@@ -48,7 +56,7 @@ def test_site():
 
 
 if __name__ == "__main__":
-    from twisted.internet import reactor
+    from twisted.internet import reactor  # pylint: disable=ungrouped-imports
 
     port = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
     print(f"http://localhost:{port.getHost().port}/")

tests/test_command_fetch.py

@@ -1,8 +1,8 @@
 from twisted.internet import defer
 from twisted.trial import unittest
 
-from scrapy.utils.testproc import ProcessTest
-from scrapy.utils.testsite import SiteTest
+from tests.utils.testproc import ProcessTest
+from tests.utils.testsite import SiteTest
 
 
 class FetchTest(ProcessTest, SiteTest, unittest.TestCase):

tests/test_command_parse.py

@@ -7,9 +7,9 @@ from twisted.internet import defer
 from scrapy.commands import parse
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
-from scrapy.utils.testproc import ProcessTest
-from scrapy.utils.testsite import SiteTest
 from tests.test_commands import CommandTest
+from tests.utils.testproc import ProcessTest
+from tests.utils.testsite import SiteTest
 
 
 def _textmode(bstr):

tests/test_command_shell.py

@@ -7,10 +7,10 @@ from pexpect.popen_spawn import PopenSpawn
 from twisted.internet import defer
 from twisted.trial import unittest
 
-from scrapy.utils.testproc import ProcessTest
-from scrapy.utils.testsite import SiteTest
 from tests import NON_EXISTING_RESOLVABLE, tests_datadir
 from tests.mockserver import MockServer
+from tests.utils.testproc import ProcessTest
+from tests.utils.testsite import SiteTest
 
 
 class ShellTest(ProcessTest, SiteTest, unittest.TestCase):

tests/test_command_version.py

@@ -4,7 +4,7 @@ from twisted.internet import defer
 from twisted.trial import unittest
 
 import scrapy
-from scrapy.utils.testproc import ProcessTest
+from tests.utils.testproc import ProcessTest
 
 
 class VersionTest(ProcessTest, unittest.TestCase):

tests/test_feedexport.py

@@ -17,7 +17,7 @@ from io import BytesIO
 from logging import getLogger
 from pathlib import Path
 from string import ascii_letters, digits
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 from unittest import mock
 from urllib.parse import quote, urljoin
 from urllib.request import pathname2url
@@ -48,7 +48,7 @@ from scrapy.extensions.feedexport import (
 )
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
-from scrapy.utils.test import get_crawler, mock_google_cloud_storage
+from scrapy.utils.test import get_crawler
 from tests.mockserver import MockFTPServer, MockServer
 from tests.spiders import ItemSpider
 
@@ -71,6 +71,23 @@ def build_url(path: str | PathLike) -> str:
     return urljoin("file:", path_str)
 
 
+def mock_google_cloud_storage() -> tuple[Any, Any, Any]:
+    """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
+    classes and set their proper return values.
+    """
+    from google.cloud.storage import Blob, Bucket, Client
+
+    client_mock = mock.create_autospec(Client)
+
+    bucket_mock = mock.create_autospec(Bucket)
+    client_mock.get_bucket.return_value = bucket_mock
+
+    blob_mock = mock.create_autospec(Blob)
+    bucket_mock.blob.return_value = blob_mock
+
+    return (client_mock, bucket_mock, blob_mock)
+
+
 class FileFeedStorageTest(unittest.TestCase):
     def test_store_file_uri(self):
         path = Path(self.mktemp()).resolve()

tests/test_pipeline_files.py

@@ -6,8 +6,10 @@ import warnings
 from datetime import datetime
 from io import BytesIO
 from pathlib import Path
+from posixpath import split
 from shutil import rmtree
 from tempfile import mkdtemp
+from typing import Any
 from unittest import mock
 from urllib.parse import urlparse
 
@@ -27,16 +29,54 @@ from scrapy.pipelines.files import (
     S3FilesStore,
 )
 from scrapy.utils.test import (
-    assert_gcs_environ,
     get_crawler,
-    get_ftp_content_and_delete,
-    get_gcs_content_and_delete,
 )
 from tests.mockserver import MockFTPServer
 
 from .test_pipeline_media import _mocked_download_func
 
 
+def get_gcs_content_and_delete(
+    bucket: Any, path: str
+) -> tuple[bytes, list[dict[str, str]], Any]:
+    from google.cloud import storage
+
+    client = storage.Client(project=os.environ.get("GCS_PROJECT_ID"))
+    bucket = client.get_bucket(bucket)
+    blob = bucket.get_blob(path)
+    content = blob.download_as_string()
+    acl = list(blob.acl)  # loads acl before it will be deleted
+    bucket.delete_blob(path)
+    return content, acl, blob
+
+
+def get_ftp_content_and_delete(
+    path: str,
+    host: str,
+    port: int,
+    username: str,
+    password: str,
+    use_active_mode: bool = False,
+) -> bytes:
+    from ftplib import FTP
+
+    ftp = FTP()
+    ftp.connect(host, port)
+    ftp.login(username, password)
+    if use_active_mode:
+        ftp.set_pasv(False)
+    ftp_data: list[bytes] = []
+
+    def buffer_data(data: bytes) -> None:
+        ftp_data.append(data)
+
+    ftp.retrbinary(f"RETR {path}", buffer_data)
+    dirname, filename = split(path)
+    ftp.cwd(dirname)
+    ftp.delete(filename)
+    return b"".join(ftp_data)
+
+
 class FilesPipelineTestCase(unittest.TestCase):
     def setUp(self):
         self.tempdir = mkdtemp()
@@ -597,10 +637,12 @@ class TestS3FilesStore(unittest.TestCase):
             stub.assert_no_pending_responses()
 
 
+@pytest.mark.skipif(
+    "GCS_PROJECT_ID" not in os.environ, reason="GCS_PROJECT_ID not found"
+)
 class TestGCSFilesStore(unittest.TestCase):
     @defer.inlineCallbacks
     def test_persist(self):
-        assert_gcs_environ()
         uri = os.environ.get("GCS_TEST_FILE_URI")
         if not uri:
             raise unittest.SkipTest("No GCS URI available for testing")
@@ -629,7 +671,6 @@ class TestGCSFilesStore(unittest.TestCase):
         """Test to make sure that paths used to store files is the same as the one used to get
         already uploaded files.
         """
-        assert_gcs_environ()
         try:
             import google.cloud.storage  # noqa: F401
         except ModuleNotFoundError:

tests/utils/testproc.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import os
+import sys
+from typing import TYPE_CHECKING, cast
+
+from twisted.internet.defer import Deferred
+from twisted.internet.error import ProcessTerminated
+from twisted.internet.protocol import ProcessProtocol
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+    from twisted.python.failure import Failure
+
+
+class ProcessTest:
+    command: str | None = None
+    prefix = [sys.executable, "-m", "scrapy.cmdline"]
+    cwd = os.getcwd()  # trial chdirs to temp dir  # noqa: PTH109
+
+    def execute(
+        self,
+        args: Iterable[str],
+        check_code: bool = True,
+        settings: str | None = None,
+    ) -> Deferred[TestProcessProtocol]:
+        from twisted.internet import reactor
+
+        env = os.environ.copy()
+        if settings is not None:
+            env["SCRAPY_SETTINGS_MODULE"] = settings
+        assert self.command
+        cmd = [*self.prefix, self.command, *args]
+        pp = TestProcessProtocol()
+        pp.deferred.addCallback(self._process_finished, cmd, check_code)
+        reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd)
+        return pp.deferred
+
+    def _process_finished(
+        self, pp: TestProcessProtocol, cmd: list[str], check_code: bool
+    ) -> tuple[int, bytes, bytes]:
+        if pp.exitcode and check_code:
+            msg = f"process {cmd} exit with code {pp.exitcode}"
+            msg += f"\n>>> stdout <<<\n{pp.out.decode()}"
+            msg += "\n"
+            msg += f"\n>>> stderr <<<\n{pp.err.decode()}"
+            raise RuntimeError(msg)
+        return cast(int, pp.exitcode), pp.out, pp.err
+
+
+class TestProcessProtocol(ProcessProtocol):
+    def __init__(self) -> None:
+        self.deferred: Deferred[TestProcessProtocol] = Deferred()
+        self.out: bytes = b""
+        self.err: bytes = b""
+        self.exitcode: int | None = None
+
+    def outReceived(self, data: bytes) -> None:
+        self.out += data
+
+    def errReceived(self, data: bytes) -> None:
+        self.err += data
+
+    def processEnded(self, status: Failure) -> None:
+        self.exitcode = cast(ProcessTerminated, status.value).exitCode
+        self.deferred.callback(self)

tests/utils/testsite.py

@@ -0,0 +1,47 @@
+from urllib.parse import urljoin
+
+from twisted.web import resource, server, static, util
+
+
+class SiteTest:
+    def setUp(self):
+        from twisted.internet import reactor
+
+        super().setUp()
+        self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
+        self.baseurl = f"http://localhost:{self.site.getHost().port}/"
+
+    def tearDown(self):
+        super().tearDown()
+        self.site.stopListening()
+
+    def url(self, path: str) -> str:
+        return urljoin(self.baseurl, path)
+
+
+class NoMetaRefreshRedirect(util.Redirect):
+    def render(self, request: server.Request) -> bytes:
+        content = util.Redirect.render(self, request)
+        return content.replace(
+            b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"'
+        )
+
+
+def test_site():
+    r = resource.Resource()
+    r.putChild(b"text", static.Data(b"Works", "text/plain"))
+    r.putChild(
+        b"html",
+        static.Data(
+            b"<body><p class='one'>Works</p><p class='two'>World</p></body>",
+            "text/html",
+        ),
+    )
+    r.putChild(
+        b"enc-gb18030",
+        static.Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"),
+    )
+    r.putChild(b"redirect", util.Redirect(b"/redirected"))
+    r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected"))
+    r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
+    return server.Site(r)