Merge 295f674c6a into d8ba1571e7

tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py

@@ -8,7 +8,7 @@ from scrapy.crawler import AsyncCrawlerProcess
 
 if sys.platform == "win32":
     asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
-asyncioreactor.install(asyncio.get_event_loop())
+asyncioreactor.install()
 
 
 class NoRequestsSpider(scrapy.Spider):

tests/test_crawler_subprocess.py

@@ -352,7 +352,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
         ) in log
 
     @pytest.mark.requires_uvloop
-    def test_asyncio_enabled_reactor_same_loop(self) -> None:
+    def test_asyncio_custom_loop_custom_settings_same(self) -> None:
         log = self.run_script("asyncio_custom_loop_custom_settings_same.py")
         assert "Spider closed (finished)" in log
         assert (
@@ -362,7 +362,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
         assert "Using asyncio event loop: uvloop.Loop" in log
 
     @pytest.mark.requires_uvloop
-    def test_asyncio_enabled_reactor_different_loop(self) -> None:
+    def test_asyncio_custom_loop_custom_settings_different(self) -> None:
         log = self.run_script("asyncio_custom_loop_custom_settings_different.py")
         assert "Spider closed (finished)" not in log
         assert (
@@ -437,14 +437,14 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
             in log
         )
 
-    def test_shutdown_graceful(self) -> None:
+    def test_reactorless_shutdown_graceful(self) -> None:
         self._test_shutdown_graceful("reactorless_sleeping.py")
 
     @coroutine_test
-    async def test_shutdown_forced(self) -> None:
+    async def test_reactorless_shutdown_forced(self) -> None:
         await self._test_shutdown_forced("reactorless_sleeping.py")
 
-    def test_shutdown_graceful_reactorless_no_stop(self) -> None:
+    def test_reactorless_shutdown_graceful_no_stop(self) -> None:
         self._test_shutdown_graceful("reactorless_sleeping.py", "--no-stop")
 
     def test_asyncio_enabled_reactor_same_loop_default(self) -> None:

tests/test_downloader_handlers_http_base.py

@@ -538,13 +538,6 @@ class TestHttpBase(ABC):
         else:
             assert latency > 0
 
-    @coroutine_test
-    async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None:
-        request = Request(mockserver.url("/text", is_secure=self.is_secure))
-        async with self.get_dh() as download_handler:
-            response = await download_handler.download_request(request)
-        assert response.body == b"Works"
-
     @coroutine_test
     async def test_response_class_choosing_request(
         self, mockserver: MockServer
@@ -560,6 +553,13 @@ class TestHttpBase(ABC):
             response = await download_handler.download_request(request)
         assert type(response) is TextResponse  # pylint: disable=unidiomatic-typecheck
 
+    @coroutine_test
+    async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None:
+        request = Request(mockserver.url("/text", is_secure=self.is_secure))
+        async with self.get_dh({"DOWNLOAD_MAXSIZE": 0}) as download_handler:
+            response = await download_handler.download_request(request)
+        assert response.body == b"Works"
+
     @coroutine_test
     async def test_download_with_maxsize(
         self, caplog: pytest.LogCaptureFixture, mockserver: MockServer

tests/test_downloadermiddleware_redirect.py

@@ -36,7 +36,7 @@ class TestRedirectMiddleware(Base.Test):
         headers = {"Location": location}
         return Response(request.url, status=status, headers=headers)
 
-    def test_redirect_3xx_permanent(self):
+    def test_redirect_307_308_preserve_method(self):
         def _test(method, status: int):
             url = f"http://www.example.com/{status}"
             url2 = "http://www.example.com/redirected"

tests/test_downloadermiddleware_retry.py

@@ -42,17 +42,18 @@ class TestRetry:
         req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True})
         rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
 
-        # first retry
+        # no retry
         r = self.mw.process_response(req, rsp)
         assert r is rsp
 
         # Test retry when dont_retry set to False
         req = Request("http://www.scrapytest.org/503", meta={"dont_retry": False})
-        rsp = Response("http://www.scrapytest.org/503")
+        rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
 
         # first retry
-        r = self.mw.process_response(req, rsp)
-        assert r is rsp
+        req = self.mw.process_response(req, rsp)
+        assert isinstance(req, Request)
+        assert req.meta["retry_times"] == 1
 
     def test_dont_retry_exc(self):
         req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True})

tests/test_engine_loop.py

@@ -120,7 +120,7 @@ class TestRequestSendOrder:
 
     @classmethod
     def teardown_class(cls):
-        cls.mockserver.__exit__(None, None, None)  # increase if flaky
+        cls.mockserver.__exit__(None, None, None)
 
     def request(self, num, response_seconds, download_slots, priority=0):
         url = self.mockserver.url(f"/delay?n={response_seconds}&{num}")

tests/test_extension_periodic_log.py

@@ -73,7 +73,6 @@ def extension(settings: dict[str, Any] | None = None) -> CustomPeriodicLog:
 class TestPeriodicLog:
     def test_extension_enabled(self):
         # Expected that settings for this extension loaded successfully
-        # And on certain conditions - extension raising NotConfigured
 
         # "PERIODIC_LOG_STATS": True -> set to {"enabled": True}
         # due to TypeError exception from settings.getdict

tests/test_feedexport.py

@@ -724,7 +724,7 @@ class TestFeedExport(TestFeedExportBase):
 
     @coroutine_test
     async def test_export_no_items_multiple_feeds(self):
-        """Make sure that `storage.store` is called for every feed."""
+        """Make sure that `storage.store` is not called."""
         settings = {
             "FEEDS": {
                 self._random_temp_filename(): {"format": "json"},

tests/test_http_response_text.py

@@ -405,22 +405,6 @@ class TestTextResponse(TestResponse):
             encoding="cp1251",
         )
 
-    def test_follow_flags(self):
-        res = self.response_class("http://example.com/")
-        fol = res.follow("http://example.com/", flags=["cached", "allowed"])
-        assert fol.flags == ["cached", "allowed"]
-
-    def test_follow_all_flags(self):
-        re = self.response_class("http://www.example.com/")
-        urls = [
-            "http://www.example.com/",
-            "http://www.example.com/2",
-            "http://www.example.com/foo",
-        ]
-        fol = re.follow_all(urls, flags=["cached", "allowed"])
-        for req in fol:
-            assert req.flags == ["cached", "allowed"]
-
     def test_follow_all_css(self):
         expected = [
             "http://example.com/sample3.html",

tests/test_linkextractors.py

@@ -798,19 +798,6 @@ class Base:
 class TestLxmlLinkExtractor(Base.TestLinkExtractorBase):
     extractor_cls = LxmlLinkExtractor
 
-    def test_link_wrong_href(self):
-        html = b"""
-        <a href="http://example.org/item1.html">Item 1</a>
-        <a href="http://[example.org/item2.html">Item 2</a>
-        <a href="http://example.org/item3.html">Item 3</a>
-        """
-        response = HtmlResponse("http://example.org/index.html", body=html)
-        lx = self.extractor_cls()
-        assert list(lx.extract_links(response)) == [
-            Link(url="http://example.org/item1.html", text="Item 1", nofollow=False),
-            Link(url="http://example.org/item3.html", text="Item 3", nofollow=False),
-        ]
-
     def test_link_restrict_text(self):
         html = b"""
         <a href="http://example.org/item1.html">Pic of a cat</a>

tests/test_pipeline_media.py

@@ -394,7 +394,7 @@ class TestMediaPipelineAllowRedirectSettings:
             # the downloader to handle itself
             (301, False),
             (302, False),
-            (302, False),
+            (303, False),
             (307, False),
             (308, False),
             # we still want to get 4xx and 5xx

tests/test_spider.py

@@ -30,7 +30,7 @@ class TestSpider:
         assert spider.foo == "bar"
 
     def test_spider_without_name(self):
-        """``__init__`` method arguments are assigned to spider attributes"""
+        """``__init__`` raises when the name is not provided."""
         msg = "must have a name"
         with pytest.raises(ValueError, match=msg):
             self.spider_class()

tests/test_spiderloader/__init__.py

@@ -192,7 +192,6 @@ class TestDuplicateSpiderNameLoader:
     def test_multiple_dupename_warning(self, spider_loader_env):
         settings, spiders_dir = spider_loader_env
         # copy 2 spider modules so as to have duplicate spider name
-        # This should issue 2 warning, 1 for each duplicate spider name
         shutil.copyfile(spiders_dir / "spider1.py", spiders_dir / "spider1dupe.py")
         shutil.copyfile(spiders_dir / "spider2.py", spiders_dir / "spider2dupe.py")
 

tests/test_spidermiddleware_referer.py

@@ -239,7 +239,7 @@ class MixinSameOrigin:
         ("https://example.com/page.html", "http://not.example.com/", None),
         ("ftps://example.com/urls.zip", "https://example.com/not-page.html", None),
         ("ftp://example.com/urls.zip", "http://example.com/not-page.html", None),
-        ("ftps://example.com/urls.zip", "https://example.com/not-page.html", None),
+        ("ftps://example.com/urls.zip", "http://example.com/not-page.html", None),
         # test for user/password stripping
         (
             "https://user:password@example.com/page.html",
@@ -395,7 +395,7 @@ class MixinOriginWhenCrossOrigin:
         ),
         (
             "ftps://example4.com/urls.zip",
-            "https://example4.com/not-page.html",
+            "http://example4.com/not-page.html",
             b"ftps://example4.com/",
         ),
         # test for user/password stripping
@@ -506,9 +506,9 @@ class MixinStrictOriginWhenCrossOrigin:
             b"ftps://example4.com/",
         ),
         (
-            "ftps://example4.com/urls.zip",
-            "https://example4.com/not-page.html",
-            b"ftps://example4.com/",
+            "ftp://example4.com/urls.zip",
+            "http://example4.com/not-page.html",
+            b"ftp://example4.com/",
         ),
         # test for user/password stripping
         (

tests/test_stats.py

@@ -32,7 +32,11 @@ class TestCoreStatsExtension:
     @mock.patch("scrapy.extensions.corestats.monotonic", return_value=0)
     @mock.patch("scrapy.extensions.corestats.datetime")
     def test_core_stats_default_stats_collector(
-        self, mock_datetime: mock.Mock, crawler: Crawler, spider: Spider
+        self,
+        mock_datetime: mock.Mock,
+        mock_monotonic: mock.Mock,
+        crawler: Crawler,
+        spider: Spider,
     ) -> None:
         fixed_datetime = datetime(2019, 12, 1, 11, 38)
         mock_datetime.now = mock.Mock(return_value=fixed_datetime)

tests/test_utils_iterators.py

@@ -352,11 +352,6 @@ class TestLxmlXmliter(TestXmliterBase):
         node = next(my_iter)
         assert node.xpath("f:name/text()").getall() == ["African Coffee Table"]
 
-    def test_xmliter_objtype_exception(self):
-        i = self.xmliter(42, "product")  # type: ignore[arg-type]
-        with pytest.raises(TypeError):
-            next(i)
-
 
 class TestUtilsCsv:
     def test_csviter_defaults(self):

tests/test_utils_url.py

@@ -48,16 +48,6 @@ def test_url_is_from_spider():
     assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider)
 
 
-def test_url_is_from_spider_class_attributes():
-    class MySpider(Spider):
-        name = "example.com"
-
-    assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)
-    assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
-    assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)
-    assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider)
-
-
 def test_url_is_from_spider_with_allowed_domains():
     class MySpider(Spider):
         name = "example.com"