Assorted test fixes (#7616)

scrapy/utils/_deps_compat.py

@@ -1,7 +1,15 @@
+import sys
+
 from OpenSSL import __version__ as PYOPENSSL_VERSION_STRING
 from packaging.version import Version
 from twisted import version as TWISTED_VERSION
 from twisted.python.versions import Version as TxVersion
+from w3lib import __version__ as W3LIB_VERSION_STRING
+
+# improved urllib.robotparser, https://github.com/python/cpython/pull/149374
+STDLIB_IMPROVED_ROBOTFILEPARSER = sys.version_info >= (3, 14, 5) or (
+    (3, 13, 14) <= sys.version_info < (3, 14)
+)
 
 TWISTED_FAILURE_HAS_STACK = TWISTED_VERSION < TxVersion("twisted", 24, 10, 0)
 # changes to private _sslverify code, https://github.com/twisted/twisted/pull/12506
@@ -14,3 +22,7 @@ PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING)
 PYOPENSSL_WANTS_X509_PKEY = PYOPENSSL_VERSION < Version("24.3.0")
 # SSL.Context.set_cipher_list() creates a temporary connection, making the context immutable
 PYOPENSSL_SET_CIPHER_LIST_TMP_CONN = PYOPENSSL_VERSION < Version("25.2.0")
+
+W3LIB_VERSION = Version(W3LIB_VERSION_STRING)
+# safe_url_string() strips the input, https://github.com/scrapy/w3lib/pull/207
+W3LIB_STRIPS_URLS = W3LIB_VERSION >= Version("2.1.1")

tests/ignores.txt

@@ -1,3 +1 @@
-scrapy/downloadermiddlewares/cookies.py
 scrapy/extensions/statsmailer.py
-scrapy/extensions/memusage.py

tests/mockserver/simple_https.py

@@ -33,7 +33,7 @@ class SimpleMockServer(BaseMockServer):
         super().__init__()
         self.keyfile = keyfile
         self.certfile = certfile
-        self.cipher_string = cipher_string or ""
+        self.cipher_string = cipher_string
         self.tls_min_version = tls_min_version
         self.tls_max_version = tls_max_version
 

tests/spiders.py

@@ -222,7 +222,7 @@ class AsyncDefDeferredWrappedSpider(SimpleSpider):
 
 
 class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):
-    name = "asyncdef_deferred_wrapped"
+    name = "asyncdef_deferred_maybe_wrapped"
 
     async def parse(self, response):
         await maybe_deferred_to_future(defer.succeed(None))

tests/test_addons.py

@@ -161,6 +161,7 @@ class TestAddonManager:
         settings.set("KEY", 0, priority="default")
         runner = runner_cls(settings)
         crawler = runner.create_crawler(Spider)
+        crawler._apply_settings()
         assert crawler.settings.getint("KEY") == 20
 
     def test_fallback_workflow(self):

tests/test_cmdline/extensions.py

@@ -1,14 +1,2 @@
-"""A test extension used to check the settings loading order"""
-
-
-class TestExtension:
-    def __init__(self, settings):
-        settings.set("TEST1", f"{settings['TEST1']} + started")
-
-    @classmethod
-    def from_crawler(cls, crawler):
-        return cls(crawler.settings)
-
-
 class DummyExtension:
     pass

tests/test_cmdline/settings.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 
 EXTENSIONS = {
-    "tests.test_cmdline.extensions.TestExtension": 0,
+    "tests.test_cmdline.extensions.DummyExtension": 0,
 }
 
 TEST1 = "default"

tests/test_command_startproject.py

@@ -257,7 +257,7 @@ class TestStartprojectTemplates:
 
         assert actual_permissions == expected_permissions
 
-    def test_startproject_permissions_umask_022(self, tmp_path: Path) -> None:
+    def test_startproject_permissions_umask_002(self, tmp_path: Path) -> None:
         """Check that generated files have the right permissions when the
         system uses a umask value that causes new files to have different
         permissions than those from the template folder."""

tests/test_commands.py

@@ -214,9 +214,7 @@ class MySpider(scrapy.Spider):
         self._append_settings(proj_path / self.project_name, "TWISTED_REACTOR = None\n")
 
         self._assert_spider_works(self.NORMAL_MSG, proj_path, "sp")
-        self._assert_spider_asyncio_fail(
+        self._assert_spider_asyncio_fail(self.NORMAL_MSG, proj_path, "aiosp")
-            self.NORMAL_MSG, proj_path, "aiosp", "-s", "TWISTED_REACTOR="
-        )
 
     def test_spider_settings_asyncio(self, proj_path: Path) -> None:
         """The reactor is set via the spider settings to the asyncio value.

tests/test_contracts.py

@@ -388,7 +388,7 @@ class TestContractsManager:
         request = self.conman.from_method(spider.returns_item_meta, self.results)
         assert request.meta["key"] == "example"
         response.meta = request.meta
-        request.callback(ResponseMetaMock)
+        request.callback(response)
         assert response.meta["key"] == "example"
         self.should_succeed()
 
@@ -476,14 +476,14 @@ class TestContractsManager:
 
         # invalid regex
         request = self.conman.from_method(spider.invalid_regex, self.results)
-        self.should_succeed()
+        assert request is None
 
         # invalid regex with valid contract
         request = self.conman.from_method(
             spider.invalid_regex_with_valid_contract, self.results
         )
-        self.should_succeed()
         request.callback(response)
+        self.should_succeed()
 
     def test_custom_contracts(self):
         self.conman.from_spider(CustomContractSuccessSpider(), self.results)
@@ -578,7 +578,7 @@ class TestCustomContractPrePostProcess:
         spider = DemoSpider()
         response = ResponseMock()
         contract = CustomFailContractPreProcess(spider.returns_request)
-        conman = ContractsManager([contract])
+        conman = ContractsManager([UrlContract, ReturnsContract, contract])
 
         request = conman.from_method(spider.returns_request, self.results)
         contract.add_pre_hook(request, self.results)
@@ -592,7 +592,7 @@ class TestCustomContractPrePostProcess:
         spider = DemoSpider()
         response = ResponseMock()
         contract = CustomFailContractPostProcess(spider.returns_request)
-        conman = ContractsManager([contract])
+        conman = ContractsManager([UrlContract, ReturnsContract, contract])
 
         request = conman.from_method(spider.returns_request, self.results)
         contract.add_post_hook(request, self.results)

tests/test_core_downloader.py

@@ -36,7 +36,6 @@ from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
     from twisted.internet.defer import Deferred
-    from twisted.internet.ssl import ContextFactory
     from twisted.web.iweb import IBodyProducer
 
 
@@ -48,8 +47,6 @@ class TestSlot:
 
 @pytest.mark.requires_reactor  # this test is related to the Twisted HTTP code
 class TestContextFactoryBase:
-    context_factory: ContextFactory | None = None
-
     @async_yield_fixture
     async def server_url(self, tmp_path):
         (tmp_path / "file").write_bytes(b"0123456789")
@@ -69,7 +66,7 @@ class TestContextFactoryBase:
         return reactor.listenSSL(
             0,
             site,
-            contextFactory=self.context_factory or ssl_context_factory(),
+            contextFactory=ssl_context_factory(),
             interface="127.0.0.1",
         )
 

tests/test_crawl.py

@@ -342,7 +342,7 @@ with multiples lines
         assert "responses" in crawler.spider.meta
         assert "failures" not in crawler.spider.meta
         # start() doesn't set Referer header
-        echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body))
+        echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][0].body))
         assert "Referer" not in echo0["headers"]
         # following request sets Referer to the source request url
         echo1 = json.loads(to_unicode(crawler.spider.meta["responses"][1].body))
@@ -390,7 +390,7 @@ with multiples lines
         est = [x for sublist in est for x in sublist]  # flatten
         est = [x.lstrip().rstrip() for x in est]
         it = iter(est)
-        s = dict(zip(it, it, strict=False))
+        s = dict(zip(it, it, strict=True))
 
         assert s["engine.spider.name"] == crawler.spider.name
         assert s["len(engine.scraper.slot.active)"] == "1"

tests/test_crawler.py

@@ -240,11 +240,7 @@ class TestCrawler(TestBaseCrawler):
 
             @classmethod
             def from_crawler(cls, crawler):
-                try:
+                crawler.get_downloader_middleware(DefaultSpider)
-                    crawler.get_downloader_middleware(DefaultSpider)
-                except Exception as e:
-                    MySpider.result = e
-                    raise
 
         crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
         with pytest.raises(RuntimeError):
@@ -322,11 +318,7 @@ class TestCrawler(TestBaseCrawler):
 
             @classmethod
             def from_crawler(cls, crawler):
-                try:
+                crawler.get_extension(DefaultSpider)
-                    crawler.get_extension(DefaultSpider)
-                except Exception as e:
-                    MySpider.result = e
-                    raise
 
         crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
         with pytest.raises(RuntimeError):
@@ -404,11 +396,7 @@ class TestCrawler(TestBaseCrawler):
 
             @classmethod
             def from_crawler(cls, crawler):
-                try:
+                crawler.get_item_pipeline(DefaultSpider)
-                    crawler.get_item_pipeline(DefaultSpider)
-                except Exception as e:
-                    MySpider.result = e
-                    raise
 
         crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
         with pytest.raises(RuntimeError):
@@ -486,11 +474,7 @@ class TestCrawler(TestBaseCrawler):
 
             @classmethod
             def from_crawler(cls, crawler):
-                try:
+                crawler.get_spider_middleware(DefaultSpider)
-                    crawler.get_spider_middleware(DefaultSpider)
-                except Exception as e:
-                    MySpider.result = e
-                    raise
 
         crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
         with pytest.raises(RuntimeError):
@@ -755,11 +739,12 @@ class TestCrawlerRunnerHasSpider:
             ):
                 await self._crawl(runner, NoRequestsSpider)
         else:
-            CrawlerRunner(
+            runner = CrawlerRunner(
                 settings={
                     "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
                 }
             )
+            await self._crawl(runner, NoRequestsSpider)
 
 
 @pytest.mark.only_asyncio

tests/test_downloader_handler_twisted_ftp.py

@@ -59,7 +59,7 @@ class TestFTPBase(ABC):
         port = reactor.listenTCP(0, factory, interface="127.0.0.1")
         portno = port.getHost().port
 
-        yield f"https://127.0.0.1:{portno}/"
+        yield f"ftp://127.0.0.1:{portno}/"
 
         await port.stopListening()
 
@@ -142,15 +142,11 @@ class TestFTPBase(ABC):
         server_url: str,
         dh: FTPDownloadHandler,
     ) -> None:
-        f, local_fname = mkstemp()
-        local_fname_path = Path(local_fname)
-        os.close(f)
         meta = {}
         meta.update(self.req_meta)
         request = Request(url=server_url + filename, meta=meta)
         r = await dh.download_request(request)
         assert type(r) is response_class  # pylint: disable=unidiomatic-typecheck
-        local_fname_path.unlink()
 
 
 class TestFTP(TestFTPBase):

tests/test_downloader_handler_twisted_http2.py

@@ -24,6 +24,7 @@ from tests.test_downloader_handlers_http_base import (
     TestHttpWithCrawlerBase,
     TestMitmProxyBase,
     TestRealWebsiteBase,
+    TestSimpleHttpsBase,
 )
 from tests.utils.decorators import coroutine_test
 
@@ -156,6 +157,10 @@ class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):
                 await download_handler.download_request(request)
 
 
+class TestSimpleHttp2(H2DownloadHandlerMixin, TestSimpleHttpsBase):
+    pass
+
+
 class TestHttp2WrongHostname(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase):
     pass
 

tests/test_downloader_handlers.py

@@ -191,17 +191,14 @@ class TestS3:
 
     @contextlib.contextmanager
     def _mocked_date(self, date):
-        try:
+        import botocore.auth  # noqa: F401,PLC0415
-            import botocore.auth  # noqa: F401,PLC0415
+
-        except ImportError:
+        # We need to mock botocore.auth.formatdate, because otherwise
+        # botocore overrides Date header with current date and time
+        # and Authorization header is different each time
+        with mock.patch("botocore.auth.formatdate") as mock_formatdate:
+            mock_formatdate.return_value = date
             yield
-        else:
-            # We need to mock botocore.auth.formatdate, because otherwise
-            # botocore overrides Date header with current date and time
-            # and Authorization header is different each time
-            with mock.patch("botocore.auth.formatdate") as mock_formatdate:
-                mock_formatdate.return_value = date
-                yield
 
     @coroutine_test
     async def test_request_signing1(self):

tests/test_downloader_handlers_http_base.py

@@ -563,7 +563,7 @@ class TestHttpBase(ABC):
     ) -> None:
         request = Request(mockserver.url("/text", is_secure=self.is_secure))
 
-        # 10 is minimal size for this request and the limit is only counted on
+        # 5 is minimal size for this request and the limit is only counted on
         # response body. (regardless of headers)
         async with self.get_dh({"DOWNLOAD_MAXSIZE": 5}) as download_handler:
             response = await download_handler.download_request(request)

tests/test_downloadermiddleware_httpcache.py

@@ -203,7 +203,7 @@ class DummyPolicyTestMixin(PolicyTestMixin):
             assert mw.process_request(req) is None
 
         # s3 scheme response is cached by default
-        req, res = Request("s3://bucket/key"), Response("http://bucket/key")
+        req, res = Request("s3://bucket/key"), Response("s3://bucket/key")
         with self._middleware() as mw:
             assert mw.process_request(req) is None
             mw.process_response(req, res)
@@ -214,7 +214,7 @@ class DummyPolicyTestMixin(PolicyTestMixin):
             assert "cached" in cached.flags
 
         # ignore s3 scheme
-        req, res = Request("s3://bucket/key2"), Response("http://bucket/key2")
+        req, res = Request("s3://bucket/key2"), Response("s3://bucket/key2")
         with self._middleware(HTTPCACHE_IGNORE_SCHEMES=["s3"]) as mw:
             assert mw.process_request(req) is None
             mw.process_response(req, res)

tests/test_downloadermiddleware_httpcompression.py

@@ -361,7 +361,7 @@ class TestHttpCompression:
         zf.write(plainbody)
         zf.close()
         response = Response(
-            "http;//www.example.com/", headers=headers, body=f.getvalue()
+            "http://www.example.com/", headers=headers, body=f.getvalue()
         )
         request = Request("http://www.example.com/")
 
@@ -386,7 +386,7 @@ class TestHttpCompression:
         zf.write(plainbody)
         zf.close()
         response = HtmlResponse(
-            "http;//www.example.com/page.html", headers=headers, body=f.getvalue()
+            "http://www.example.com/page.html", headers=headers, body=f.getvalue()
         )
         request = Request("http://www.example.com/")
 
@@ -493,7 +493,7 @@ class TestHttpCompression:
         gz_resp.close()
 
         response = Response(
-            "http;//www.example.com/", headers=headers, body=r.getvalue()
+            "http://www.example.com/", headers=headers, body=r.getvalue()
         )
         request = Request("http://www.example.com/")
 

tests/test_downloadermiddleware_redirect_base.py

@@ -122,7 +122,7 @@ class Base:
             req1 = Request("http://a.example/first")
             rsp1 = self.get_response(req1, "/redirected")
             req2 = self.mw.process_response(req1, rsp1)
-            rsp2 = self.get_response(req1, "/redirected2")
+            rsp2 = self.get_response(req2, "/redirected2")
             req3 = self.mw.process_response(req2, rsp2)
 
             assert req2.url == "http://a.example/redirected"

tests/test_downloadermiddleware_stats.py

@@ -16,7 +16,7 @@ class TestDownloaderStats:
         self.crawler.stats.open_spider()
 
         self.req = Request("http://scrapytest.org")
-        self.res = Response("scrapytest.org", status=400)
+        self.res = Response("http://scrapytest.org", status=400)
 
     def assertStatsEqual(self, key, value):
         assert self.crawler.stats.get_value(key) == value, str(

tests/test_downloaderslotssettings.py

@@ -5,7 +5,6 @@ import pytest
 
 from scrapy import Request
 from scrapy.core.downloader import Downloader, Slot
-from scrapy.crawler import CrawlerRunner
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
@@ -65,9 +64,6 @@ class TestCrawl:
     def teardown_class(cls):
         cls.mockserver.__exit__(None, None, None)
 
-    def setup_method(self):
-        self.runner = CrawlerRunner()
-
     @inline_callbacks_test
     def test_delay(self):
         crawler = get_crawler(DownloaderSlotsSettingsTestSpider)

tests/test_dupefilters.py

@@ -86,7 +86,7 @@ class TestRFPDupeFilter:
                 df.close("finished")
 
             df2 = _get_dupefilter(settings={"JOBDIR": path}, open_=False)
-            assert df != df2
+            assert df is not df2
             try:
                 df2.open()
                 assert df2.request_seen(r1)

tests/test_engine.py

@@ -6,7 +6,6 @@ import subprocess
 import sys
 from collections import defaultdict
 from dataclasses import dataclass
-from logging import DEBUG
 from typing import TYPE_CHECKING, Any, cast
 from unittest.mock import Mock, call
 from urllib.parse import urlparse
@@ -395,6 +394,7 @@ class TestEngine(TestEngineBase):
             self._assert_downloaded_responses(run, count=9)
             self._assert_scraped_items(run)
             self._assert_signals_caught(run)
+            self._assert_headers_received(run)
             self._assert_bytes_received(run)
 
     @coroutine_test
@@ -606,7 +606,7 @@ class TestEngineDownload(TestEngineDownloadAsync):
 
 
 @coroutine_test
-async def test_request_scheduled_signal(caplog):
+async def test_request_scheduled_signal():
     class TestScheduler(BaseScheduler):
         def __init__(self):
             self.enqueued = []
@@ -633,7 +633,6 @@ async def test_request_scheduled_signal(caplog):
     keep_request = Request("https://keep.example")
     engine._schedule_request(keep_request)
     drop_request = Request("https://drop.example")
-    caplog.set_level(DEBUG)
     engine._schedule_request(drop_request)
     assert scheduler.enqueued == [keep_request], (
         f"{scheduler.enqueued!r} != [{keep_request!r}]"

tests/test_feedexport.py

@@ -32,7 +32,6 @@ from scrapy.extensions.feedexport import (
     FeedSlot,
     FileFeedStorage,
     IFeedStorage,
-    S3FeedStorage,
 )
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_crawler
@@ -499,8 +498,7 @@ class TestFeedExport(TestFeedExportBase):
             },
         }
         crawler = get_crawler(ItemSpider, settings)
-        with mock.patch.object(S3FeedStorage, "store"):
+        yield crawler.crawl(mockserver=self.mockserver)
-            yield crawler.crawl(mockserver=self.mockserver)
         assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()
         assert "feedexport/success_count/StdoutFeedStorage" in crawler.stats.get_stats()
         assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1

tests/test_feedexport_batch.py

@@ -315,7 +315,7 @@ class TestBatchDeliveries(TestFeedExportBase):
         }
         data = await self.exported_data(items, settings)
         for fmt, expected in formats.items():
-            for expected_batch, got_batch in zip(expected, data[fmt], strict=False):
+            for expected_batch, got_batch in zip(expected, data[fmt], strict=True):
                 assert got_batch == expected_batch
 
     @coroutine_test
@@ -339,7 +339,7 @@ class TestBatchDeliveries(TestFeedExportBase):
         }
         data = await self.exported_data(items, settings)
         for fmt, expected in formats.items():
-            for expected_batch, got_batch in zip(expected, data[fmt], strict=False):
+            for expected_batch, got_batch in zip(expected, data[fmt], strict=True):
                 assert got_batch == expected_batch
 
     @coroutine_test
@@ -447,7 +447,7 @@ class TestBatchDeliveries(TestFeedExportBase):
         yield crawler.crawl()
 
         assert len(CustomS3FeedStorage.stubs) == len(items)
-        for stub in CustomS3FeedStorage.stubs[:-1]:
+        for stub in CustomS3FeedStorage.stubs:
             stub.assert_no_pending_responses()
         assert (
             "feedexport/success_count/CustomS3FeedStorage" in crawler.stats.get_stats()

tests/test_feedexport_postprocess.py

@@ -270,7 +270,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
             self._named_tempfile("check_CHECK_NONE"): lzma.compress(
                 self.expected, check=lzma.CHECK_NONE
             ),
-            self._named_tempfile("check_CHECK_CRC256"): lzma.compress(
+            self._named_tempfile("CHECK_SHA256"): lzma.compress(
                 self.expected, check=lzma.CHECK_SHA256
             ),
         }
@@ -282,7 +282,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
                     "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
                     "lzma_check": lzma.CHECK_NONE,
                 },
-                self._named_tempfile("check_CHECK_CRC256"): {
+                self._named_tempfile("CHECK_SHA256"): {
                     "format": "csv",
                     "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
                     "lzma_check": lzma.CHECK_SHA256,

tests/test_http2_client_protocol.py

@@ -141,7 +141,7 @@ class Dataloss(LeafResource):
 
 class NoContentLengthHeader(LeafResource):
     def render_GET(self, request: TxRequest):
-        request.requestHeaders.removeHeader("Content-Length")
+        request.responseHeaders.removeHeader("Content-Length")
         self.deferRequest(request, 0, self._delayed_render, request)
         return NOT_DONE_YET
 
@@ -460,9 +460,7 @@ class TestHttps2ClientProtocol:
     def test_invalid_negotiated_protocol(
         self, server_port: int, client: H2ClientProtocol
     ) -> Generator[Deferred[Any], Any, None]:
-        with mock.patch(
+        with mock.patch("scrapy.core.http2.protocol.PROTOCOL_NAME", new=b"not-h2"):
-            "scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2"
-        ):
             request = Request(url=self.get_url(server_port, "/status?n=200"))
             with pytest.raises(ResponseFailed):
                 yield make_request_dfd(client, request)

tests/test_http_request.py

@@ -23,7 +23,6 @@ class TestRequest:
         # url argument must be basestring
         with pytest.raises(TypeError):
             self.request_class(123)
-        r = self.request_class("http://www.example.com")
 
         r = self.request_class("http://www.example.com")
         assert isinstance(r.url, str)
@@ -211,11 +210,11 @@ class TestRequest:
         r1.cb_kwargs["key"] = "value"
         r2 = r1.copy()
 
-        # make sure copy does not propagate callbacks
+        # make sure callbaclks are copied
         assert r1.callback is somecallback
         assert r1.errback is somecallback
         assert r2.callback is r1.callback
-        assert r2.errback is r2.errback
+        assert r2.errback is r1.errback
 
         # make sure flags list is shallow copied
         assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"

tests/test_http_response.py

@@ -1,13 +1,19 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
 import pytest
-from packaging.version import Version as parse_version
-from w3lib import __version__ as w3lib_version
 from w3lib.encoding import resolve_encoding
 
 from scrapy.exceptions import NotSupported
 from scrapy.http import Headers, Request, Response
 from scrapy.link import Link
+from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS
 from tests import get_testdata
 
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
 
 class TestResponse:
     response_class = Response
@@ -249,7 +255,7 @@ class TestResponse:
             r.follow(None)
 
     @pytest.mark.xfail(
-        parse_version(w3lib_version) < parse_version("2.1.1"),
+        not W3LIB_STRIPS_URLS,
         reason="https://github.com/scrapy/w3lib/pull/207",
         strict=True,
     )
@@ -257,7 +263,7 @@ class TestResponse:
         self._assert_followed_url("foo ", "http://example.com/foo")
 
     @pytest.mark.xfail(
-        parse_version(w3lib_version) < parse_version("2.1.1"),
+        not W3LIB_STRIPS_URLS,
         reason="https://github.com/scrapy/w3lib/pull/207",
         strict=True,
     )
@@ -325,16 +331,26 @@ class TestResponse:
             with pytest.raises(ValueError, match="url can't be None"):
                 list(r.follow_all(urls=[None]))
 
+    @pytest.mark.xfail(
+        not W3LIB_STRIPS_URLS,
+        reason="https://github.com/scrapy/w3lib/pull/207",
+        strict=True,
+    )
     def test_follow_all_whitespace(self):
         relative = ["foo ", "bar ", "foo/bar ", "bar/foo "]
         absolute = [
-            "http://example.com/foo%20",
+            "http://example.com/foo",
-            "http://example.com/bar%20",
+            "http://example.com/bar",
-            "http://example.com/foo/bar%20",
+            "http://example.com/foo/bar",
-            "http://example.com/bar/foo%20",
+            "http://example.com/bar/foo",
         ]
         self._assert_followed_all_urls(relative, absolute)
 
+    @pytest.mark.xfail(
+        not W3LIB_STRIPS_URLS,
+        reason="https://github.com/scrapy/w3lib/pull/207",
+        strict=True,
+    )
     def test_follow_all_whitespace_links(self):
         absolute = [
             "http://example.com/foo ",
@@ -342,8 +358,8 @@ class TestResponse:
             "http://example.com/foo/bar ",
             "http://example.com/bar/foo ",
         ]
-        links = map(Link, absolute)
+        links = [Link(u) for u in absolute]
-        expected = [u.replace(" ", "%20") for u in absolute]
+        expected = [u.strip() for u in absolute]
         self._assert_followed_all_urls(links, expected)
 
     def test_follow_all_flags(self):
@@ -357,25 +373,36 @@ class TestResponse:
         for req in fol:
             assert req.flags == ["cached", "allowed"]
 
-    def _assert_followed_url(self, follow_obj, target_url, response=None):
+    def _assert_followed_url(
+        self,
+        follow_obj: str | Link,
+        target_url: str,
+        response: Response | None = None,
+        encoding: str | None = None,
+    ) -> None:
         if response is None:
             response = self._links_response()
         req = response.follow(follow_obj)
         assert req.url == target_url
-        return req
+        if encoding is not None:
+            assert req.encoding == encoding
 
-    def _assert_followed_all_urls(self, follow_obj, target_urls, response=None):
+    def _assert_followed_all_urls(
+        self,
+        follow_obj: Iterable[str | Link],
+        target_urls: Iterable[str],
+        response: Response | None = None,
+    ) -> None:
         if response is None:
             response = self._links_response()
         followed = response.follow_all(follow_obj)
-        for req, target in zip(followed, target_urls, strict=False):
+        for req, target in zip(followed, target_urls, strict=True):
             assert req.url == target
-            yield req
 
-    def _links_response(self):
+    def _links_response(self) -> Response:
         body = get_testdata("link_extractor", "linkextractor.html")
         return self.response_class("http://example.com/index", body=body)
 
-    def _links_response_no_href(self):
+    def _links_response_no_href(self) -> Response:
         body = get_testdata("link_extractor", "linkextractor_no_href.html")
         return self.response_class("http://example.com/index", body=body)

tests/test_http_response_text.py

@@ -179,7 +179,6 @@ class TestTextResponse(TestResponse):
         # Inferring encoding from body also cache decoded body as sideeffect,
         # this test tries to ensure that calling response.encoding and
         # response.text in indistinct order doesn't affect final
-        # response.text in indistinct order doesn't affect final
         # values for encoding and decoded body.
         url = "http://example.com"
         body = b"\xef\xbb\xbfWORD"
@@ -308,11 +307,12 @@ class TestTextResponse(TestResponse):
             "http://example.com/sample3.html#foo",
             "http://www.google.com/something",
             "http://example.com/innertag.html",
+            "http://example.com/page%204.html",
         ]
 
         # select <a> elements
         for sellist in [resp.css("a"), resp.xpath("//a")]:
-            for sel, url in zip(sellist, urls, strict=False):
+            for sel, url in zip(sellist, urls, strict=True):
                 self._assert_followed_url(sel, url, response=resp)
 
         # select <link> elements
@@ -324,7 +324,7 @@ class TestTextResponse(TestResponse):
 
         # href attributes should work
         for sellist in [resp.css("a::attr(href)"), resp.xpath("//a/@href")]:
-            for sel, url in zip(sellist, urls, strict=False):
+            for sel, url in zip(sellist, urls, strict=True):
                 self._assert_followed_url(sel, url, response=resp)
 
         # non-a elements are not supported
@@ -376,12 +376,12 @@ class TestTextResponse(TestResponse):
             encoding="utf8",
             body='<html><body><a href="foo?привет">click me</a></body></html>'.encode(),
         )
-        req = self._assert_followed_url(
+        self._assert_followed_url(
             resp1.css("a")[0],
             "http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82",
             response=resp1,
+            encoding="utf8",
         )
-        assert req.encoding == "utf8"
 
         resp2 = self.response_class(
             "http://example.com",
@@ -390,12 +390,12 @@ class TestTextResponse(TestResponse):
                 "cp1251"
             ),
         )
-        req = self._assert_followed_url(
+        self._assert_followed_url(
             resp2.css("a")[0],
             "http://example.com/foo?%EF%F0%E8%E2%E5%F2",
             response=resp2,
+            encoding="cp1251",
         )
-        assert req.encoding == "cp1251"
 
     def test_follow_flags(self):
         res = self.response_class("http://example.com/")

tests/test_logformatter.py

@@ -28,14 +28,14 @@ class TestLogFormatter:
         self.spider = Spider("default")
         self.spider.crawler = get_crawler()
 
-    def test_crawled_with_referer(self):
+    def test_crawled_without_referer(self):
         req = Request("http://www.example.com")
         res = Response("http://www.example.com")
         logkws = self.formatter.crawled(req, res, self.spider)
         logline = logkws["msg"] % logkws["args"]
         assert logline == "Crawled (200) <GET http://www.example.com> (referer: None)"
 
-    def test_crawled_without_referer(self):
+    def test_crawled_with_referer(self):
         req = Request(
             "http://www.example.com", headers={"referer": "http://example.com"}
         )
@@ -198,7 +198,7 @@ class TestLogformatterSubclass(TestLogFormatter):
         self.spider = Spider("default")
         self.spider.crawler = get_crawler(Spider)
 
-    def test_crawled_with_referer(self):
+    def test_crawled_without_referer(self):
         req = Request("http://www.example.com")
         res = Response("http://www.example.com")
         logkws = self.formatter.crawled(req, res, self.spider)
@@ -207,7 +207,7 @@ class TestLogformatterSubclass(TestLogFormatter):
             logline == "Crawled (200) <GET http://www.example.com> (referer: None) []"
         )
 
-    def test_crawled_without_referer(self):
+    def test_crawled_with_referer(self):
         req = Request(
             "http://www.example.com",
             headers={"referer": "http://example.com"},

tests/test_pipeline_media.py

@@ -175,10 +175,6 @@ class MockedMediaPipeline(UserDefinedPipeline):
         super().__init__(*args, crawler=crawler, **kwargs)
         self._mockcalled = []
 
-    def download(self, request, info):
-        self._mockcalled.append("download")
-        return super().download(request, info)
-
     def media_to_download(self, request, info, *, item=None):
         self._mockcalled.append("media_to_download")
         if "result" in request.meta:

tests/test_robotstxt_interface.py

@@ -1,5 +1,3 @@
-import sys
-
 import pytest
 
 from scrapy.robotstxt import (
@@ -8,6 +6,7 @@ from scrapy.robotstxt import (
     RerpRobotParser,
     decode_robotstxt,
 )
+from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER
 
 
 def rerp_available() -> bool:
@@ -139,28 +138,25 @@ class TestDecodeRobotsTxt:
 
 
 class TestPythonRobotParser(BaseRobotParserTest):
-    # https://github.com/python/cpython/pull/149374 improves it
-    IMPROVED_ROBOTFILEPARSER = sys.version_info >= (3, 14, 5)
-
     def setup_method(self):
         super()._setUp(PythonRobotParser)
 
     @pytest.mark.skipif(
-        not IMPROVED_ROBOTFILEPARSER,
+        not STDLIB_IMPROVED_ROBOTFILEPARSER,
         reason="RobotFileParser from this Python version does not support length based directives precedence.",
     )
     def test_length_based_precedence(self):
         super().test_length_based_precedence()
 
     @pytest.mark.skipif(
-        IMPROVED_ROBOTFILEPARSER,
+        STDLIB_IMPROVED_ROBOTFILEPARSER,
         reason="RobotFileParser from this Python version does not support order based directives precedence.",
     )
     def test_order_based_precedence(self):
         super().test_order_based_precedence()
 
     @pytest.mark.skipif(
-        not IMPROVED_ROBOTFILEPARSER,
+        not STDLIB_IMPROVED_ROBOTFILEPARSER,
         reason="RobotFileParser from this Python version does not support wildcards.",
     )
     def test_allowed_wildcards(self):

tests/test_scheduler_base.py

@@ -104,7 +104,7 @@ class TestMinimalScheduler(InterfaceCheckMixin):
         for url in URLS:
             assert self.scheduler.enqueue_request(Request(url))
             assert not self.scheduler.enqueue_request(Request(url))
-        assert self.scheduler.has_pending_requests
+        assert self.scheduler.has_pending_requests()
 
         dequeued = []
         while self.scheduler.has_pending_requests():

tests/test_spidermiddleware.py

@@ -238,16 +238,6 @@ class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple):
             yield item
 
 
-class ProcessSpiderOutputNonIterableMiddleware:
-    def process_spider_output(self, response, result):
-        return
-
-
-class ProcessSpiderOutputCoroutineMiddleware:
-    async def process_spider_output(self, response, result):
-        return result
-
-
 class ProcessStartSimpleMiddleware:
     async def process_start(self, start):
         async for item_or_request in start:
@@ -423,20 +413,12 @@ class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple):
 class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):
     ITEM_TYPE = dict
     MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
-    MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
     MW_EXC_SIMPLE = ProcessSpiderExceptionSimpleIterableMiddleware
     MW_EXC_ASYNCGEN = ProcessSpiderExceptionAsyncIteratorMiddleware
 
     def _callback(self) -> Any:
         1 / 0
 
-    async def _test_asyncgen_nodowngrade(self, *mw_classes: type[Any]) -> None:
-        with pytest.raises(
-            _InvalidOutput,
-            match=r"Async iterable returned from .+ cannot be downgraded",
-        ):
-            await self._get_middleware_result(*mw_classes)
-
     @coroutine_test
     async def test_exc_simple(self):
         """Simple exc mw"""

tests/test_spidermiddleware_process_start.py

@@ -14,7 +14,6 @@ from .utils.decorators import coroutine_test
 ITEM_A = {"id": "a"}
 ITEM_B = {"id": "b"}
 ITEM_C = {"id": "c"}
-ITEM_D = {"id": "d"}
 
 
 class AsyncioSleepSpiderMiddleware:
@@ -47,10 +46,6 @@ class ModernWrapSpider(Spider):
         yield ITEM_B
 
 
-class ModernWrapSpiderSubclass(ModernWrapSpider):
-    name = "test"
-
-
 class ModernWrapSpiderMiddleware:
     async def process_start(self, start):
         yield ITEM_A
@@ -79,10 +74,6 @@ class TestMain:
         expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C]
         await self._test([spider_middleware], spider_cls, expected_items)
 
-    async def _test_douple_wrap(self, smw1, smw2, spider_cls, expected_items=None):
-        expected_items = expected_items or [ITEM_A, ITEM_A, ITEM_B, ITEM_C, ITEM_C]
-        await self._test([smw1, smw2], spider_cls, expected_items)
-
     @coroutine_test
     async def test_modern_mw_modern_spider(self):
         with warnings.catch_warnings():

tests/test_spidermiddleware_referer.py

@@ -836,7 +836,7 @@ class TestRequestMetaSettingFallback:
             request_meta,
             policy_class,
             check_warning,
-        ) in self.params[3:]:
+        ) in self.params:
             mw = RefererMiddleware(Settings(settings))
 
             response = Response(origin, headers=response_headers)

tests/test_spiderstate.py

@@ -1,4 +1,7 @@
+from __future__ import annotations
+
 from datetime import datetime, timezone
+from typing import TYPE_CHECKING
 
 import pytest
 
@@ -7,8 +10,11 @@ from scrapy.extensions.spiderstate import SpiderState
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 
+if TYPE_CHECKING:
+    from pathlib import Path
 
-def test_store_load(tmp_path):
+
+def test_store_load(tmp_path: Path) -> None:
     jobdir = str(tmp_path)
 
     spider = Spider(name="default")
@@ -16,6 +22,7 @@ def test_store_load(tmp_path):
 
     ss = SpiderState(jobdir)
     ss.spider_opened(spider)
+    assert hasattr(spider, "state")
     spider.state["one"] = 1
     spider.state["dt"] = dt
     ss.spider_closed(spider)
@@ -23,21 +30,23 @@ def test_store_load(tmp_path):
     spider2 = Spider(name="default")
     ss2 = SpiderState(jobdir)
     ss2.spider_opened(spider2)
-    assert spider.state == {"one": 1, "dt": dt}
+    assert hasattr(spider2, "state")
+    assert spider2.state == {"one": 1, "dt": dt}
     ss2.spider_closed(spider2)
 
 
-def test_state_attribute():
+def test_state_attribute() -> None:
     # state attribute must be present if jobdir is not set, to provide a
     # consistent interface
     spider = Spider(name="default")
     ss = SpiderState()
     ss.spider_opened(spider)
+    assert hasattr(spider, "state")
     assert spider.state == {}
     ss.spider_closed(spider)
 
 
-def test_not_configured():
+def test_not_configured() -> None:
     crawler = get_crawler(Spider)
     with pytest.raises(NotConfigured):
         SpiderState.from_crawler(crawler)

tests/test_utils_asyncio.py

@@ -83,6 +83,7 @@ class TestParallelAsyncio:
                 max_parallel_count,
             )
             assert list(range(length)) == sorted(results)
+            assert parallel_count[0] == 0
             assert max_parallel_count[0] <= self.CONCURRENT_ITEMS
 
     @coroutine_test
@@ -101,6 +102,7 @@ class TestParallelAsyncio:
                 max_parallel_count,
             )
             assert list(range(length)) == sorted(results)
+            assert parallel_count[0] == 0
             assert max_parallel_count[0] <= self.CONCURRENT_ITEMS
 
 

tests/test_utils_deprecate.py

@@ -22,9 +22,7 @@ class NewName(SomeBaseClass):
 
 
 class TestWarnWhenSubclassed:
-    def _mywarnings(
+    def _mywarnings(self, w: list[WarningMessage]) -> list[WarningMessage]:
-        self, w: list[WarningMessage], category: type[Warning] = MyWarning
-    ) -> list[WarningMessage]:
         return [x for x in w if x.category is MyWarning]
 
     def test_no_warning_on_definition(self):

tests/test_utils_httpobj.py

@@ -17,4 +17,4 @@ def test_urlparse_cached():
     assert req1a == urlp
     assert req1a is req1b
     assert req1a is not req2
-    assert req1a is not req2
+    assert req1b is not req2

tests/test_utils_log.py

@@ -76,15 +76,16 @@ class TestLogCounterHandler:
     @pytest.fixture
     def logger(self, crawler: Crawler) -> Generator[logging.Logger]:
         logger = logging.getLogger("test")
-        logger.setLevel(logging.NOTSET)
+        logger.setLevel(logging.DEBUG)
         logger.propagate = False
-        handler = LogCounterHandler(crawler)
+        handler = LogCounterHandler(crawler, level=crawler.settings.get("LOG_LEVEL"))
         logger.addHandler(handler)
-
+        try:
-        yield logger
+            yield logger
-
+        finally:
-        logger.propagate = True
+            logger.propagate = True
-        logger.removeHandler(handler)
+            logger.setLevel(logging.NOTSET)
+            logger.removeHandler(handler)
 
     def test_init(self, crawler: Crawler, logger: logging.Logger) -> None:
         assert crawler.stats
@@ -102,7 +103,7 @@ class TestLogCounterHandler:
     def test_filtered_out_level(self, crawler: Crawler, logger: logging.Logger) -> None:
         logger.debug("test log msg")
         assert crawler.stats
-        assert crawler.stats.get_value("log_count/INFO") is None
+        assert crawler.stats.get_value("log_count/DEBUG") is None
 
 
 class TestStreamLogger:

tests/test_utils_response.py

@@ -23,7 +23,7 @@ def _read_browser_output(burl: str):
 
 
 def test_open_in_browser():
-    url = "http:///www.example.com/some/page.html"
+    url = "http://www.example.com/some/page.html"
     body = (
         b"<html> <head> <title>test page</title> </head> <body>test body</body> </html>"
     )