Move imported stuff around.

tests/spiders.py

@@ -571,3 +571,19 @@ class HeadersReceivedErrbackSpider(HeadersReceivedCallbackSpider):
     def headers_received(self, headers, body_length, request, spider):
         self.meta["headers_received"] = headers
         raise StopDownload(fail=True)
+
+
+class ExceptionSpider(Spider):
+    name = "exception"
+
+    @classmethod
+    def from_crawler(cls, crawler, *args, **kwargs):
+        raise ValueError("Exception in from_crawler method")
+
+
+class NoRequestsSpider(Spider):
+    name = "no_request"
+
+    async def start(self):
+        return
+        yield

tests/test_command_runspider.py

@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING
 
 import pytest
 
-from tests.test_crawler import ExceptionSpider, NoRequestsSpider
+from tests.spiders import ExceptionSpider, NoRequestsSpider
 from tests.utils.cmdline import proc
 
 if TYPE_CHECKING:
@@ -65,13 +65,14 @@ class BadSpider(scrapy.Spider):
 
     def test_run_fail_spider(self, tmp_path: Path) -> None:
         ret, _, _ = self.runspider(
-            tmp_path, "import scrapy\n" + inspect.getsource(ExceptionSpider)
+            tmp_path, "from scrapy import Spider\n" + inspect.getsource(ExceptionSpider)
         )
         assert ret != 0
 
     def test_run_good_spider(self, tmp_path: Path) -> None:
         ret, _, _ = self.runspider(
-            tmp_path, "import scrapy\n" + inspect.getsource(NoRequestsSpider)
+            tmp_path,
+            "from scrapy import Spider\n" + inspect.getsource(NoRequestsSpider),
         )
         assert ret == 0
 

tests/test_crawler.py

@@ -33,6 +33,7 @@ from scrapy.utils.log import (
 )
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler, get_reactor_settings
+from tests.spiders import ExceptionSpider, NoRequestsSpider
 from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
@@ -849,22 +850,6 @@ def test_crawl_rejects_spider_object(runner_cls: type[CrawlerRunnerBase]) -> Non
         runner.crawl(DefaultSpider())  # type: ignore[arg-type]
 
 
-class ExceptionSpider(scrapy.Spider):
-    name = "exception"
-
-    @classmethod
-    def from_crawler(cls, crawler, *args, **kwargs):
-        raise ValueError("Exception in from_crawler method")
-
-
-class NoRequestsSpider(scrapy.Spider):
-    name = "no_request"
-
-    async def start(self):
-        return
-        yield
-
-
 @pytest.mark.requires_reactor  # CrawlerRunner requires a reactor
 class TestCrawlerRunnerHasSpider:
     @pytest.fixture

tests/test_downloadermiddleware_redirect.py

@@ -16,11 +16,8 @@ from scrapy.spiders import Spider
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
-from tests.test_downloadermiddleware_redirect_base import (
-    REDIRECT_SCHEME_CASES,
-    SCHEME_PARAMS,
-    Base,
-)
+from tests.test_downloadermiddleware_redirect_base import Base
+from tests.utils.redirect import REDIRECT_SCHEME_CASES, SCHEME_PARAMS
 
 
 class TestRedirectMiddleware(Base.Test):

tests/test_downloadermiddleware_redirect_base.py

@@ -1,7 +1,5 @@
 from __future__ import annotations
 
-from itertools import product
-
 import pytest
 
 from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
@@ -10,43 +8,6 @@ from scrapy.http import Request, Response
 from scrapy.utils.misc import set_environ
 from scrapy.utils.test import get_crawler
 
-SCHEME_PARAMS = ("url", "location", "target")
-HTTP_SCHEMES = ("http", "https")
-NON_HTTP_SCHEMES = ("data", "file", "ftp", "s3", "foo")
-REDIRECT_SCHEME_CASES = (
-    # http/https → http/https redirects
-    *(
-        (
-            f"{input_scheme}://example.com/a",
-            f"{output_scheme}://example.com/b",
-            f"{output_scheme}://example.com/b",
-        )
-        for input_scheme, output_scheme in product(HTTP_SCHEMES, repeat=2)
-    ),
-    # http/https → data/file/ftp/s3/foo does not redirect
-    *(
-        (
-            f"{input_scheme}://example.com/a",
-            f"{output_scheme}://example.com/b",
-            None,
-        )
-        for input_scheme in HTTP_SCHEMES
-        for output_scheme in NON_HTTP_SCHEMES
-    ),
-    # http/https → relative redirects
-    *(
-        (
-            f"{scheme}://example.com/a",
-            location,
-            f"{scheme}://example.com/b",
-        )
-        for scheme in HTTP_SCHEMES
-        for location in ("//example.com/b", "/b")
-    ),
-    # Note: We do not test data/file/ftp/s3 schemes for the initial URL
-    # because their download handlers cannot return a status code of 3xx.
-)
-
 
 class Base:
     class Test:

tests/test_downloadermiddleware_redirect_metarefresh.py

@@ -12,12 +12,12 @@ from scrapy.http import HtmlResponse, Request, Response
 from scrapy.spiders import Spider
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.test import get_crawler
-from tests.test_downloadermiddleware_redirect_base import (
+from tests.test_downloadermiddleware_redirect_base import Base
+from tests.utils.redirect import (
     HTTP_SCHEMES,
     NON_HTTP_SCHEMES,
     REDIRECT_SCHEME_CASES,
     SCHEME_PARAMS,
-    Base,
 )
 
 

tests/test_downloadermiddleware_robotstxt.py

@@ -15,8 +15,8 @@ from scrapy.http.request import NO_CALLBACK
 from scrapy.settings import Settings
 from scrapy.utils.asyncio import call_later
 from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
-from tests.test_robotstxt_interface import rerp_available
 from tests.utils.decorators import coroutine_test
+from tests.utils.robotstxt import rerp_available
 
 if TYPE_CHECKING:
     from scrapy.crawler import Crawler

tests/test_engine_loop.py

@@ -2,13 +2,13 @@ from __future__ import annotations
 
 from collections import deque
 from logging import ERROR
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from scrapy import Request, Spider, signals
+from scrapy.core.scheduler import BaseScheduler
 from scrapy.utils.asyncio import call_later
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
-from tests.test_scheduler import MemoryScheduler
 from tests.utils import async_sleep
 from tests.utils.decorators import coroutine_test
 
@@ -18,6 +18,38 @@ if TYPE_CHECKING:
     from scrapy.http import Response
 
 
+class MemoryScheduler(BaseScheduler):
+    paused = False
+
+    def __init__(self, *args: Any, **kwargs: Any):
+        super().__init__(*args, **kwargs)
+        self.queue: deque[Request] = deque(
+            Request(value) if isinstance(value, str) else value
+            for value in getattr(self, "queue", [])
+        )
+
+    def enqueue_request(self, request: Request) -> bool:
+        self.queue.append(request)
+        return True
+
+    def has_pending_requests(self) -> bool:
+        return self.paused or bool(self.queue)
+
+    def next_request(self) -> Request | None:
+        if self.paused:
+            return None
+        try:
+            return self.queue.pop()
+        except IndexError:
+            return None
+
+    def pause(self) -> None:
+        self.paused = True
+
+    def unpause(self) -> None:
+        self.paused = False
+
+
 class TestMain:
     @coroutine_test
     async def test_sleep(self):

tests/test_feedexport.py

@@ -13,8 +13,6 @@ from pathlib import Path
 from string import ascii_letters, digits
 from typing import IO, TYPE_CHECKING, Any
 from unittest import mock
-from urllib.parse import urljoin
-from urllib.request import pathname2url
 
 import lxml.etree
 import pytest
@@ -39,19 +37,12 @@ from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
 from tests.spiders import ItemSpider
 from tests.utils.decorators import coroutine_test, inline_callbacks_test
+from tests.utils.feedexport import path_to_url, printf_escape
 
 if TYPE_CHECKING:
     from collections.abc import Awaitable, Callable, Iterable
 
 
-def path_to_url(path: str | Path) -> str:
-    return urljoin("file:", pathname2url(str(path)))
-
-
-def printf_escape(s: str) -> str:
-    return s.replace("%", "%%")
-
-
 class FromCrawlerMixin:
     init_with_crawler = False
 

tests/test_feedexport_postprocess.py

@@ -13,8 +13,9 @@ from typing import TYPE_CHECKING, Any
 import pytest
 
 from scrapy.utils.test import get_crawler
-from tests.test_feedexport import TestFeedExportBase, path_to_url, printf_escape
+from tests.test_feedexport import TestFeedExportBase
 from tests.utils.decorators import coroutine_test
+from tests.utils.feedexport import path_to_url, printf_escape
 
 if TYPE_CHECKING:
     from scrapy import Spider

tests/test_robotstxt_interface.py

@@ -7,17 +7,7 @@ from scrapy.robotstxt import (
     decode_robotstxt,
 )
 from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER
-
-
-def rerp_available() -> bool:
-    # check if robotexclusionrulesparser is installed
-    try:
-        from robotexclusionrulesparser import (  # noqa: PLC0415
-            RobotExclusionRulesParser,  # noqa: F401
-        )
-    except ImportError:
-        return False
-    return True
+from tests.utils.robotstxt import rerp_available
 
 
 class BaseRobotParserTest:

tests/test_scheduler.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import warnings
 from abc import ABC, abstractmethod
-from collections import deque
 from contextlib import AbstractAsyncContextManager, asynccontextmanager
 from typing import TYPE_CHECKING, Any, NamedTuple, cast
 from unittest.mock import Mock
@@ -10,7 +9,7 @@ from unittest.mock import Mock
 import pytest
 
 from scrapy.core.downloader import Downloader
-from scrapy.core.scheduler import BaseScheduler, Scheduler
+from scrapy.core.scheduler import Scheduler
 from scrapy.crawler import Crawler
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
@@ -27,38 +26,6 @@ if TYPE_CHECKING:
     from pathlib import Path
 
 
-class MemoryScheduler(BaseScheduler):
-    paused = False
-
-    def __init__(self, *args: Any, **kwargs: Any):
-        super().__init__(*args, **kwargs)
-        self.queue: deque[Request] = deque(
-            Request(value) if isinstance(value, str) else value
-            for value in getattr(self, "queue", [])
-        )
-
-    def enqueue_request(self, request: Request) -> bool:
-        self.queue.append(request)
-        return True
-
-    def has_pending_requests(self) -> bool:
-        return self.paused or bool(self.queue)
-
-    def next_request(self) -> Request | None:
-        if self.paused:
-            return None
-        try:
-            return self.queue.pop()
-        except IndexError:
-            return None
-
-    def pause(self) -> None:
-        self.paused = True
-
-    def unpause(self) -> None:
-        self.paused = False
-
-
 class MockSlot(NamedTuple):
     active: list[Any]
 

tests/utils/feedexport.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from urllib.parse import urljoin
+from urllib.request import pathname2url
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+def path_to_url(path: str | Path) -> str:
+    return urljoin("file:", pathname2url(str(path)))
+
+
+def printf_escape(s: str) -> str:
+    return s.replace("%", "%%")

tests/utils/redirect.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from itertools import product
+
+SCHEME_PARAMS = ("url", "location", "target")
+HTTP_SCHEMES = ("http", "https")
+NON_HTTP_SCHEMES = ("data", "file", "ftp", "s3", "foo")
+REDIRECT_SCHEME_CASES = (
+    # http/https → http/https redirects
+    *(
+        (
+            f"{input_scheme}://example.com/a",
+            f"{output_scheme}://example.com/b",
+            f"{output_scheme}://example.com/b",
+        )
+        for input_scheme, output_scheme in product(HTTP_SCHEMES, repeat=2)
+    ),
+    # http/https → data/file/ftp/s3/foo does not redirect
+    *(
+        (
+            f"{input_scheme}://example.com/a",
+            f"{output_scheme}://example.com/b",
+            None,
+        )
+        for input_scheme in HTTP_SCHEMES
+        for output_scheme in NON_HTTP_SCHEMES
+    ),
+    # http/https → relative redirects
+    *(
+        (
+            f"{scheme}://example.com/a",
+            location,
+            f"{scheme}://example.com/b",
+        )
+        for scheme in HTTP_SCHEMES
+        for location in ("//example.com/b", "/b")
+    ),
+    # Note: We do not test data/file/ftp/s3 schemes for the initial URL
+    # because their download handlers cannot return a status code of 3xx.
+)

tests/utils/robotstxt.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+
+def rerp_available() -> bool:
+    # check if robotexclusionrulesparser is installed
+    try:
+        from robotexclusionrulesparser import (  # noqa: PLC0415
+            RobotExclusionRulesParser,  # noqa: F401
+        )
+    except ImportError:
+        return False
+    return True