diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py
index 4257bc0ac..13a0850f9 100644
--- a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py
+++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py
@@ -8,7 +8,7 @@ from scrapy.crawler import AsyncCrawlerProcess
if sys.platform == "win32":
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
-asyncioreactor.install(asyncio.get_event_loop())
+asyncioreactor.install()
class NoRequestsSpider(scrapy.Spider):
diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py
index 018a2b31b..6e43615e4 100644
--- a/tests/test_crawler_subprocess.py
+++ b/tests/test_crawler_subprocess.py
@@ -352,7 +352,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
) in log
@pytest.mark.requires_uvloop
- def test_asyncio_enabled_reactor_same_loop(self) -> None:
+ def test_asyncio_custom_loop_custom_settings_same(self) -> None:
log = self.run_script("asyncio_custom_loop_custom_settings_same.py")
assert "Spider closed (finished)" in log
assert (
@@ -362,7 +362,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
assert "Using asyncio event loop: uvloop.Loop" in log
@pytest.mark.requires_uvloop
- def test_asyncio_enabled_reactor_different_loop(self) -> None:
+ def test_asyncio_custom_loop_custom_settings_different(self) -> None:
log = self.run_script("asyncio_custom_loop_custom_settings_different.py")
assert "Spider closed (finished)" not in log
assert (
@@ -437,14 +437,14 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
in log
)
- def test_shutdown_graceful(self) -> None:
+ def test_reactorless_shutdown_graceful(self) -> None:
self._test_shutdown_graceful("reactorless_sleeping.py")
@coroutine_test
- async def test_shutdown_forced(self) -> None:
+ async def test_reactorless_shutdown_forced(self) -> None:
await self._test_shutdown_forced("reactorless_sleeping.py")
- def test_shutdown_graceful_reactorless_no_stop(self) -> None:
+ def test_reactorless_shutdown_graceful_no_stop(self) -> None:
self._test_shutdown_graceful("reactorless_sleeping.py", "--no-stop")
def test_asyncio_enabled_reactor_same_loop_default(self) -> None:
diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py
index c4a8193c8..29e1d9757 100644
--- a/tests/test_downloader_handlers_http_base.py
+++ b/tests/test_downloader_handlers_http_base.py
@@ -538,13 +538,6 @@ class TestHttpBase(ABC):
else:
assert latency > 0
- @coroutine_test
- async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None:
- request = Request(mockserver.url("/text", is_secure=self.is_secure))
- async with self.get_dh() as download_handler:
- response = await download_handler.download_request(request)
- assert response.body == b"Works"
-
@coroutine_test
async def test_response_class_choosing_request(
self, mockserver: MockServer
@@ -560,6 +553,13 @@ class TestHttpBase(ABC):
response = await download_handler.download_request(request)
assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck
+ @coroutine_test
+ async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None:
+ request = Request(mockserver.url("/text", is_secure=self.is_secure))
+ async with self.get_dh({"DOWNLOAD_MAXSIZE": 0}) as download_handler:
+ response = await download_handler.download_request(request)
+ assert response.body == b"Works"
+
@coroutine_test
async def test_download_with_maxsize(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 1da7bbf3e..f8ec40ccc 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -36,7 +36,7 @@ class TestRedirectMiddleware(Base.Test):
headers = {"Location": location}
return Response(request.url, status=status, headers=headers)
- def test_redirect_3xx_permanent(self):
+ def test_redirect_307_308_preserve_method(self):
def _test(method, status: int):
url = f"http://www.example.com/{status}"
url2 = "http://www.example.com/redirected"
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 56d21a4d2..825efce2f 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -42,17 +42,18 @@ class TestRetry:
req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True})
rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
- # first retry
+ # no retry
r = self.mw.process_response(req, rsp)
assert r is rsp
# Test retry when dont_retry set to False
req = Request("http://www.scrapytest.org/503", meta={"dont_retry": False})
- rsp = Response("http://www.scrapytest.org/503")
+ rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
# first retry
- r = self.mw.process_response(req, rsp)
- assert r is rsp
+ req = self.mw.process_response(req, rsp)
+ assert isinstance(req, Request)
+ assert req.meta["retry_times"] == 1
def test_dont_retry_exc(self):
req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True})
diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py
index 6cc8c0650..ae7afdbde 100644
--- a/tests/test_engine_loop.py
+++ b/tests/test_engine_loop.py
@@ -120,7 +120,7 @@ class TestRequestSendOrder:
@classmethod
def teardown_class(cls):
- cls.mockserver.__exit__(None, None, None) # increase if flaky
+ cls.mockserver.__exit__(None, None, None)
def request(self, num, response_seconds, download_slots, priority=0):
url = self.mockserver.url(f"/delay?n={response_seconds}&{num}")
diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py
index 0fe2caca4..ffe7a0dc7 100644
--- a/tests/test_extension_periodic_log.py
+++ b/tests/test_extension_periodic_log.py
@@ -73,7 +73,6 @@ def extension(settings: dict[str, Any] | None = None) -> CustomPeriodicLog:
class TestPeriodicLog:
def test_extension_enabled(self):
# Expected that settings for this extension loaded successfully
- # And on certain conditions - extension raising NotConfigured
# "PERIODIC_LOG_STATS": True -> set to {"enabled": True}
# due to TypeError exception from settings.getdict
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 5ab1a54c1..28f01a45b 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -724,7 +724,7 @@ class TestFeedExport(TestFeedExportBase):
@coroutine_test
async def test_export_no_items_multiple_feeds(self):
- """Make sure that `storage.store` is called for every feed."""
+ """Make sure that `storage.store` is not called."""
settings = {
"FEEDS": {
self._random_temp_filename(): {"format": "json"},
diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py
index 5ef89fe4a..daf6e66e5 100644
--- a/tests/test_http_response_text.py
+++ b/tests/test_http_response_text.py
@@ -405,22 +405,6 @@ class TestTextResponse(TestResponse):
encoding="cp1251",
)
- def test_follow_flags(self):
- res = self.response_class("http://example.com/")
- fol = res.follow("http://example.com/", flags=["cached", "allowed"])
- assert fol.flags == ["cached", "allowed"]
-
- def test_follow_all_flags(self):
- re = self.response_class("http://www.example.com/")
- urls = [
- "http://www.example.com/",
- "http://www.example.com/2",
- "http://www.example.com/foo",
- ]
- fol = re.follow_all(urls, flags=["cached", "allowed"])
- for req in fol:
- assert req.flags == ["cached", "allowed"]
-
def test_follow_all_css(self):
expected = [
"http://example.com/sample3.html",
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index db23e9aef..95a6aee54 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -798,19 +798,6 @@ class Base:
class TestLxmlLinkExtractor(Base.TestLinkExtractorBase):
extractor_cls = LxmlLinkExtractor
- def test_link_wrong_href(self):
- html = b"""
- Item 1
- Item 2
- Item 3
- """
- response = HtmlResponse("http://example.org/index.html", body=html)
- lx = self.extractor_cls()
- assert list(lx.extract_links(response)) == [
- Link(url="http://example.org/item1.html", text="Item 1", nofollow=False),
- Link(url="http://example.org/item3.html", text="Item 3", nofollow=False),
- ]
-
def test_link_restrict_text(self):
html = b"""
Pic of a cat
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 8ac949da7..6862c34eb 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -394,7 +394,7 @@ class TestMediaPipelineAllowRedirectSettings:
# the downloader to handle itself
(301, False),
(302, False),
- (302, False),
+ (303, False),
(307, False),
(308, False),
# we still want to get 4xx and 5xx
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 526cc8f23..9a4790dba 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -30,7 +30,7 @@ class TestSpider:
assert spider.foo == "bar"
def test_spider_without_name(self):
- """``__init__`` method arguments are assigned to spider attributes"""
+ """``__init__`` raises when the name is not provided."""
msg = "must have a name"
with pytest.raises(ValueError, match=msg):
self.spider_class()
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index d85942f90..81e49bb75 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -192,7 +192,6 @@ class TestDuplicateSpiderNameLoader:
def test_multiple_dupename_warning(self, spider_loader_env):
settings, spiders_dir = spider_loader_env
# copy 2 spider modules so as to have duplicate spider name
- # This should issue 2 warning, 1 for each duplicate spider name
shutil.copyfile(spiders_dir / "spider1.py", spiders_dir / "spider1dupe.py")
shutil.copyfile(spiders_dir / "spider2.py", spiders_dir / "spider2dupe.py")
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 7591c0e7b..5ca27d15c 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -239,7 +239,7 @@ class MixinSameOrigin:
("https://example.com/page.html", "http://not.example.com/", None),
("ftps://example.com/urls.zip", "https://example.com/not-page.html", None),
("ftp://example.com/urls.zip", "http://example.com/not-page.html", None),
- ("ftps://example.com/urls.zip", "https://example.com/not-page.html", None),
+ ("ftps://example.com/urls.zip", "http://example.com/not-page.html", None),
# test for user/password stripping
(
"https://user:password@example.com/page.html",
@@ -395,7 +395,7 @@ class MixinOriginWhenCrossOrigin:
),
(
"ftps://example4.com/urls.zip",
- "https://example4.com/not-page.html",
+ "http://example4.com/not-page.html",
b"ftps://example4.com/",
),
# test for user/password stripping
@@ -506,9 +506,9 @@ class MixinStrictOriginWhenCrossOrigin:
b"ftps://example4.com/",
),
(
- "ftps://example4.com/urls.zip",
- "https://example4.com/not-page.html",
- b"ftps://example4.com/",
+ "ftp://example4.com/urls.zip",
+ "http://example4.com/not-page.html",
+ b"ftp://example4.com/",
),
# test for user/password stripping
(
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 05f609fda..c33a00090 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -32,7 +32,11 @@ class TestCoreStatsExtension:
@mock.patch("scrapy.extensions.corestats.monotonic", return_value=0)
@mock.patch("scrapy.extensions.corestats.datetime")
def test_core_stats_default_stats_collector(
- self, mock_datetime: mock.Mock, crawler: Crawler, spider: Spider
+ self,
+ mock_datetime: mock.Mock,
+ mock_monotonic: mock.Mock,
+ crawler: Crawler,
+ spider: Spider,
) -> None:
fixed_datetime = datetime(2019, 12, 1, 11, 38)
mock_datetime.now = mock.Mock(return_value=fixed_datetime)
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index a59541edb..4fb18305e 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -352,11 +352,6 @@ class TestLxmlXmliter(TestXmliterBase):
node = next(my_iter)
assert node.xpath("f:name/text()").getall() == ["African Coffee Table"]
- def test_xmliter_objtype_exception(self):
- i = self.xmliter(42, "product") # type: ignore[arg-type]
- with pytest.raises(TypeError):
- next(i)
-
class TestUtilsCsv:
def test_csviter_defaults(self):
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index a74b9a41d..5b98131a1 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -48,16 +48,6 @@ def test_url_is_from_spider():
assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider)
-def test_url_is_from_spider_class_attributes():
- class MySpider(Spider):
- name = "example.com"
-
- assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)
- assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
- assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)
- assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider)
-
-
def test_url_is_from_spider_with_allowed_domains():
class MySpider(Spider):
name = "example.com"