From 5aa254ba0481581833bb1c79d3e4be1c353f05b5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 15 Jul 2026 20:02:33 +0500 Subject: [PATCH 1/2] Assorted test fixes, part 3. --- tests/test_crawler_subprocess.py | 10 +++++----- tests/test_downloader_handlers_http_base.py | 14 +++++++------- tests/test_downloadermiddleware_redirect.py | 2 +- tests/test_downloadermiddleware_retry.py | 9 +++++---- tests/test_engine_loop.py | 2 +- tests/test_extension_periodic_log.py | 1 - tests/test_feedexport.py | 2 +- tests/test_http_response_text.py | 16 ---------------- tests/test_linkextractors.py | 13 ------------- tests/test_pipeline_media.py | 2 +- tests/test_spider.py | 2 +- tests/test_spiderloader/__init__.py | 1 - tests/test_spidermiddleware_referer.py | 10 +++++----- tests/test_stats.py | 6 +++++- tests/test_utils_iterators.py | 5 ----- tests/test_utils_url.py | 10 ---------- 16 files changed, 32 insertions(+), 73 deletions(-) diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py index 018a2b31b..6e43615e4 100644 --- a/tests/test_crawler_subprocess.py +++ b/tests/test_crawler_subprocess.py @@ -352,7 +352,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): ) in log @pytest.mark.requires_uvloop - def test_asyncio_enabled_reactor_same_loop(self) -> None: + def test_asyncio_custom_loop_custom_settings_same(self) -> None: log = self.run_script("asyncio_custom_loop_custom_settings_same.py") assert "Spider closed (finished)" in log assert ( @@ -362,7 +362,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): assert "Using asyncio event loop: uvloop.Loop" in log @pytest.mark.requires_uvloop - def test_asyncio_enabled_reactor_different_loop(self) -> None: + def test_asyncio_custom_loop_custom_settings_different(self) -> None: log = self.run_script("asyncio_custom_loop_custom_settings_different.py") assert "Spider closed (finished)" not in log assert ( @@ -437,14 +437,14 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): in log ) - def test_shutdown_graceful(self) -> None: + def test_reactorless_shutdown_graceful(self) -> None: self._test_shutdown_graceful("reactorless_sleeping.py") @coroutine_test - async def test_shutdown_forced(self) -> None: + async def test_reactorless_shutdown_forced(self) -> None: await self._test_shutdown_forced("reactorless_sleeping.py") - def test_shutdown_graceful_reactorless_no_stop(self) -> None: + def test_reactorless_shutdown_graceful_no_stop(self) -> None: self._test_shutdown_graceful("reactorless_sleeping.py", "--no-stop") def test_asyncio_enabled_reactor_same_loop_default(self) -> None: diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index c4a8193c8..29e1d9757 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -538,13 +538,6 @@ class TestHttpBase(ABC): else: assert latency > 0 - @coroutine_test - async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None: - request = Request(mockserver.url("/text", is_secure=self.is_secure)) - async with self.get_dh() as download_handler: - response = await download_handler.download_request(request) - assert response.body == b"Works" - @coroutine_test async def test_response_class_choosing_request( self, mockserver: MockServer @@ -560,6 +553,13 @@ class TestHttpBase(ABC): response = await download_handler.download_request(request) assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck + @coroutine_test + async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/text", is_secure=self.is_secure)) + async with self.get_dh({"DOWNLOAD_MAXSIZE": 0}) as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"Works" + @coroutine_test async def test_download_with_maxsize( self, caplog: pytest.LogCaptureFixture, mockserver: MockServer diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 1da7bbf3e..f8ec40ccc 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -36,7 +36,7 @@ class TestRedirectMiddleware(Base.Test): headers = {"Location": location} return Response(request.url, status=status, headers=headers) - def test_redirect_3xx_permanent(self): + def test_redirect_307_308_preserve_method(self): def _test(method, status: int): url = f"http://www.example.com/{status}" url2 = "http://www.example.com/redirected" diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 56d21a4d2..825efce2f 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -42,17 +42,18 @@ class TestRetry: req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True}) rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) - # first retry + # no retry r = self.mw.process_response(req, rsp) assert r is rsp # Test retry when dont_retry set to False req = Request("http://www.scrapytest.org/503", meta={"dont_retry": False}) - rsp = Response("http://www.scrapytest.org/503") + rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) # first retry - r = self.mw.process_response(req, rsp) - assert r is rsp + req = self.mw.process_response(req, rsp) + assert isinstance(req, Request) + assert req.meta["retry_times"] == 1 def test_dont_retry_exc(self): req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True}) diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 6cc8c0650..ae7afdbde 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -120,7 +120,7 @@ class TestRequestSendOrder: @classmethod def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) # increase if flaky + cls.mockserver.__exit__(None, None, None) def request(self, num, response_seconds, download_slots, priority=0): url = self.mockserver.url(f"/delay?n={response_seconds}&{num}") diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 0fe2caca4..ffe7a0dc7 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -73,7 +73,6 @@ def extension(settings: dict[str, Any] | None = None) -> CustomPeriodicLog: class TestPeriodicLog: def test_extension_enabled(self): # Expected that settings for this extension loaded successfully - # And on certain conditions - extension raising NotConfigured # "PERIODIC_LOG_STATS": True -> set to {"enabled": True} # due to TypeError exception from settings.getdict diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 5ab1a54c1..28f01a45b 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -724,7 +724,7 @@ class TestFeedExport(TestFeedExportBase): @coroutine_test async def test_export_no_items_multiple_feeds(self): - """Make sure that `storage.store` is called for every feed.""" + """Make sure that `storage.store` is not called.""" settings = { "FEEDS": { self._random_temp_filename(): {"format": "json"}, diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py index 5ef89fe4a..daf6e66e5 100644 --- a/tests/test_http_response_text.py +++ b/tests/test_http_response_text.py @@ -405,22 +405,6 @@ class TestTextResponse(TestResponse): encoding="cp1251", ) - def test_follow_flags(self): - res = self.response_class("http://example.com/") - fol = res.follow("http://example.com/", flags=["cached", "allowed"]) - assert fol.flags == ["cached", "allowed"] - - def test_follow_all_flags(self): - re = self.response_class("http://www.example.com/") - urls = [ - "http://www.example.com/", - "http://www.example.com/2", - "http://www.example.com/foo", - ] - fol = re.follow_all(urls, flags=["cached", "allowed"]) - for req in fol: - assert req.flags == ["cached", "allowed"] - def test_follow_all_css(self): expected = [ "http://example.com/sample3.html", diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index db23e9aef..95a6aee54 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -798,19 +798,6 @@ class Base: class TestLxmlLinkExtractor(Base.TestLinkExtractorBase): extractor_cls = LxmlLinkExtractor - def test_link_wrong_href(self): - html = b""" - Item 1 - Item 2 - Item 3 - """ - response = HtmlResponse("http://example.org/index.html", body=html) - lx = self.extractor_cls() - assert list(lx.extract_links(response)) == [ - Link(url="http://example.org/item1.html", text="Item 1", nofollow=False), - Link(url="http://example.org/item3.html", text="Item 3", nofollow=False), - ] - def test_link_restrict_text(self): html = b""" Pic of a cat diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 8ac949da7..6862c34eb 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -394,7 +394,7 @@ class TestMediaPipelineAllowRedirectSettings: # the downloader to handle itself (301, False), (302, False), - (302, False), + (303, False), (307, False), (308, False), # we still want to get 4xx and 5xx diff --git a/tests/test_spider.py b/tests/test_spider.py index 526cc8f23..9a4790dba 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -30,7 +30,7 @@ class TestSpider: assert spider.foo == "bar" def test_spider_without_name(self): - """``__init__`` method arguments are assigned to spider attributes""" + """``__init__`` raises when the name is not provided.""" msg = "must have a name" with pytest.raises(ValueError, match=msg): self.spider_class() diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index d85942f90..81e49bb75 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -192,7 +192,6 @@ class TestDuplicateSpiderNameLoader: def test_multiple_dupename_warning(self, spider_loader_env): settings, spiders_dir = spider_loader_env # copy 2 spider modules so as to have duplicate spider name - # This should issue 2 warning, 1 for each duplicate spider name shutil.copyfile(spiders_dir / "spider1.py", spiders_dir / "spider1dupe.py") shutil.copyfile(spiders_dir / "spider2.py", spiders_dir / "spider2dupe.py") diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 7591c0e7b..5ca27d15c 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -239,7 +239,7 @@ class MixinSameOrigin: ("https://example.com/page.html", "http://not.example.com/", None), ("ftps://example.com/urls.zip", "https://example.com/not-page.html", None), ("ftp://example.com/urls.zip", "http://example.com/not-page.html", None), - ("ftps://example.com/urls.zip", "https://example.com/not-page.html", None), + ("ftps://example.com/urls.zip", "http://example.com/not-page.html", None), # test for user/password stripping ( "https://user:password@example.com/page.html", @@ -395,7 +395,7 @@ class MixinOriginWhenCrossOrigin: ), ( "ftps://example4.com/urls.zip", - "https://example4.com/not-page.html", + "http://example4.com/not-page.html", b"ftps://example4.com/", ), # test for user/password stripping @@ -506,9 +506,9 @@ class MixinStrictOriginWhenCrossOrigin: b"ftps://example4.com/", ), ( - "ftps://example4.com/urls.zip", - "https://example4.com/not-page.html", - b"ftps://example4.com/", + "ftp://example4.com/urls.zip", + "http://example4.com/not-page.html", + b"ftp://example4.com/", ), # test for user/password stripping ( diff --git a/tests/test_stats.py b/tests/test_stats.py index 05f609fda..c33a00090 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -32,7 +32,11 @@ class TestCoreStatsExtension: @mock.patch("scrapy.extensions.corestats.monotonic", return_value=0) @mock.patch("scrapy.extensions.corestats.datetime") def test_core_stats_default_stats_collector( - self, mock_datetime: mock.Mock, crawler: Crawler, spider: Spider + self, + mock_datetime: mock.Mock, + mock_monotonic: mock.Mock, + crawler: Crawler, + spider: Spider, ) -> None: fixed_datetime = datetime(2019, 12, 1, 11, 38) mock_datetime.now = mock.Mock(return_value=fixed_datetime) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index a59541edb..4fb18305e 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -352,11 +352,6 @@ class TestLxmlXmliter(TestXmliterBase): node = next(my_iter) assert node.xpath("f:name/text()").getall() == ["African Coffee Table"] - def test_xmliter_objtype_exception(self): - i = self.xmliter(42, "product") # type: ignore[arg-type] - with pytest.raises(TypeError): - next(i) - class TestUtilsCsv: def test_csviter_defaults(self): diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index a74b9a41d..5b98131a1 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -48,16 +48,6 @@ def test_url_is_from_spider(): assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider) -def test_url_is_from_spider_class_attributes(): - class MySpider(Spider): - name = "example.com" - - assert url_is_from_spider("http://www.example.com/some/page.html", MySpider) - assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider) - assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider) - assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider) - - def test_url_is_from_spider_with_allowed_domains(): class MySpider(Spider): name = "example.com" From 295f674c6a58d427371061038dee67ad9ba4c87f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 15 Jul 2026 20:29:45 +0500 Subject: [PATCH 2/2] Fix asyncio_enabled_reactor_different_loop.py on 3.14. --- .../asyncio_enabled_reactor_different_loop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py index 4257bc0ac..13a0850f9 100644 --- a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py +++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -8,7 +8,7 @@ from scrapy.crawler import AsyncCrawlerProcess if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) -asyncioreactor.install(asyncio.get_event_loop()) +asyncioreactor.install() class NoRequestsSpider(scrapy.Spider):