Preparations for switching to direct pytest. (#6678)

conftest.py

@@ -41,7 +41,7 @@ if not H2_ENABLED:
     )
 
 
-@pytest.fixture()
+@pytest.fixture
 def chdir(tmpdir):
     """Change to pytest-provided temporary directory"""
     tmpdir.chdir()

pyproject.toml

@@ -200,8 +200,7 @@ disable = [
 [tool.pytest.ini_options]
 xfail_strict = true
 usefixtures = "chdir"
-python_files = ["test_*.py", "__init__.py"]
+python_files = ["test_*.py", "test_*/__init__.py"]
-python_classes = []
 addopts = [
     "--assert=plain",
     "--ignore=docs/_ext",
@@ -254,6 +253,8 @@ extend-select = [
     "PIE",
     # pylint
     "PL",
+    # flake8-pytest-style
+    "PT",
     # flake8-use-pathlib
     "PTH",
     # flake8-pyi
@@ -373,6 +374,12 @@ ignore = [
     "B904",
     # Use capitalized environment variable
     "SIM112",
+
+    # Temporarily silenced PT rules
+    # Use a regular `assert` instead of unittest-style `assertEqual`
+    "PT009",
+    # Use `pytest.raises` instead of unittest-style `assertRaises`
+    "PT027",
 ]
 
 [tool.ruff.lint.per-file-ignores]

scrapy/utils/test.py

@@ -15,9 +15,10 @@ from unittest import TestCase, mock
 
 from twisted.trial.unittest import SkipTest
 
-from scrapy import Spider
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.boto import is_botocore_available
+from scrapy.utils.deprecate import create_deprecated_class
+from scrapy.utils.spider import DefaultSpider
 
 if TYPE_CHECKING:
     from collections.abc import Awaitable
@@ -25,6 +26,7 @@ if TYPE_CHECKING:
     from twisted.internet.defer import Deferred
     from twisted.web.client import Response as TxResponse
 
+    from scrapy import Spider
     from scrapy.crawler import Crawler
 
 
@@ -82,8 +84,7 @@ def get_ftp_content_and_delete(
     return b"".join(ftp_data)
 
 
-class TestSpider(Spider):
+TestSpider = create_deprecated_class("TestSpider", DefaultSpider)
-    name = "test"
 
 
 def get_crawler(
@@ -101,7 +102,7 @@ def get_crawler(
     settings: dict[str, Any] = {}
     settings.update(settings_dict or {})
     runner = CrawlerRunner(settings)
-    crawler = runner.create_crawler(spidercls or TestSpider)
+    crawler = runner.create_crawler(spidercls or DefaultSpider)
     crawler._apply_settings()
     return crawler
 

tests/test_commands.py

@@ -19,7 +19,7 @@ from threading import Timer
 from typing import TYPE_CHECKING
 from unittest import mock, skipIf
 
-from pytest import mark
+import pytest
 from twisted.trial import unittest
 
 import scrapy
@@ -822,7 +822,7 @@ class MySpider(scrapy.Spider):
             "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log
         )
 
-    @mark.requires_uvloop
+    @pytest.mark.requires_uvloop
     def test_custom_asyncio_loop_enabled_true(self):
         log = self.get_log(
             self.debug_log_spider,

tests/test_contracts.py

@@ -21,7 +21,7 @@ from scrapy.utils.test import get_crawler
 from tests.mockserver import MockServer
 
 
-class TestItem(Item):
+class DemoItem(Item):
     name = Field()
     url = Field()
 
@@ -58,7 +58,7 @@ class CustomFormContract(Contract):
         return args
 
 
-class TestSpider(Spider):
+class DemoSpider(Spider):
     name = "demo_spider"
 
     def returns_request(self, response):
@@ -80,7 +80,7 @@ class TestSpider(Spider):
         @url http://scrapy.org
         @returns items 1 1
         """
-        return TestItem(url=response.url)
+        return DemoItem(url=response.url)
 
     def returns_request_cb_kwargs(self, response, url):
         """method which returns request
@@ -96,7 +96,7 @@ class TestSpider(Spider):
         @cb_kwargs {"name": "Scrapy"}
         @returns items 1 1
         """
-        return TestItem(name=name, url=response.url)
+        return DemoItem(name=name, url=response.url)
 
     def returns_item_cb_kwargs_error_unexpected_keyword(self, response):
         """method which returns item
@@ -104,14 +104,14 @@ class TestSpider(Spider):
         @cb_kwargs {"arg": "value"}
         @returns items 1 1
         """
-        return TestItem(url=response.url)
+        return DemoItem(url=response.url)
 
     def returns_item_cb_kwargs_error_missing_argument(self, response, arg):
         """method which returns item
         @url http://scrapy.org
         @returns items 1 1
         """
-        return TestItem(url=response.url)
+        return DemoItem(url=response.url)
 
     def returns_dict_item(self, response):
         """method which returns item
@@ -125,7 +125,7 @@ class TestSpider(Spider):
         @url http://scrapy.org
         @returns items 0 0
         """
-        return TestItem(url=response.url)
+        return DemoItem(url=response.url)
 
     def returns_dict_fail(self, response):
         """method which returns item
@@ -140,7 +140,7 @@ class TestSpider(Spider):
         @returns items 1 1
         @scrapes name url
         """
-        return TestItem(name="test", url=response.url)
+        return DemoItem(name="test", url=response.url)
 
     def scrapes_dict_item_ok(self, response):
         """returns item with name and url
@@ -156,7 +156,7 @@ class TestSpider(Spider):
         @returns items 1 1
         @scrapes name url
         """
-        return TestItem(url=response.url)
+        return DemoItem(url=response.url)
 
     def scrapes_dict_item_fail(self, response):
         """returns item with no name
@@ -212,7 +212,7 @@ class TestSpider(Spider):
         @meta {"key": "example"}
         @returns items 1 1
         """
-        return TestItem(name="example", url=response.url)
+        return DemoItem(name="example", url=response.url)
 
     def returns_error_missing_meta(self, response):
         """method which depends of metadata be defined
@@ -242,7 +242,7 @@ class CustomContractFailSpider(Spider):
         """
 
 
-class InheritsTestSpider(TestSpider):
+class InheritsDemoSpider(DemoSpider):
     name = "inherits_demo_spider"
 
 
@@ -274,7 +274,7 @@ class ContractsManagerTest(unittest.TestCase):
         self.assertTrue(self.results.errors)
 
     def test_contracts(self):
-        spider = TestSpider()
+        spider = DemoSpider()
 
         # extract contracts correctly
         contracts = self.conman.extract_contracts(spider.returns_request)
@@ -293,7 +293,7 @@ class ContractsManagerTest(unittest.TestCase):
         self.assertEqual(request, None)
 
     def test_cb_kwargs(self):
-        spider = TestSpider()
+        spider = DemoSpider()
         response = ResponseMock()
 
         # extract contracts correctly
@@ -356,7 +356,7 @@ class ContractsManagerTest(unittest.TestCase):
         self.should_error()
 
     def test_meta(self):
-        spider = TestSpider()
+        spider = DemoSpider()
 
         # extract contracts correctly
         contracts = self.conman.extract_contracts(spider.returns_request_meta)
@@ -402,7 +402,7 @@ class ContractsManagerTest(unittest.TestCase):
         self.should_error()
 
     def test_returns(self):
-        spider = TestSpider()
+        spider = DemoSpider()
         response = ResponseMock()
 
         # returns_item
@@ -431,7 +431,7 @@ class ContractsManagerTest(unittest.TestCase):
         self.should_fail()
 
     def test_returns_async(self):
-        spider = TestSpider()
+        spider = DemoSpider()
         response = ResponseMock()
 
         request = self.conman.from_method(spider.returns_request_async, self.results)
@@ -439,7 +439,7 @@ class ContractsManagerTest(unittest.TestCase):
         self.should_error()
 
     def test_scrapes(self):
-        spider = TestSpider()
+        spider = DemoSpider()
         response = ResponseMock()
 
         # scrapes_item_ok
@@ -472,7 +472,7 @@ class ContractsManagerTest(unittest.TestCase):
         assert message in self.results.failures[-1][-1]
 
     def test_regex(self):
-        spider = TestSpider()
+        spider = DemoSpider()
         response = ResponseMock()
 
         # invalid regex
@@ -494,7 +494,7 @@ class ContractsManagerTest(unittest.TestCase):
         self.should_error()
 
     def test_errback(self):
-        spider = TestSpider()
+        spider = DemoSpider()
         response = ResponseMock()
 
         try:
@@ -522,11 +522,11 @@ class ContractsManagerTest(unittest.TestCase):
 
             def parse_first(self, response):
                 self.visited += 1
-                return TestItem()
+                return DemoItem()
 
             def parse_second(self, response):
                 self.visited += 1
-                return TestItem()
+                return DemoItem()
 
         with MockServer() as mockserver:
             contract_doc = f"@url {mockserver.url('/status?n=200')}"
@@ -540,13 +540,13 @@ class ContractsManagerTest(unittest.TestCase):
         self.assertEqual(crawler.spider.visited, 2)
 
     def test_form_contract(self):
-        spider = TestSpider()
+        spider = DemoSpider()
         request = self.conman.from_method(spider.custom_form, self.results)
         self.assertEqual(request.method, "POST")
         self.assertIsInstance(request, FormRequest)
 
     def test_inherited_contracts(self):
-        spider = InheritsTestSpider()
+        spider = InheritsDemoSpider()
 
         requests = self.conman.from_spider(spider, self.results)
         self.assertTrue(requests)
@@ -571,7 +571,7 @@ class CustomContractPrePostProcess(unittest.TestCase):
         self.results = TextTestResult(stream=None, descriptions=False, verbosity=0)
 
     def test_pre_hook_keyboard_interrupt(self):
-        spider = TestSpider()
+        spider = DemoSpider()
         response = ResponseMock()
         contract = CustomFailContractPreProcess(spider.returns_request)
         conman = ContractsManager([contract])
@@ -590,7 +590,7 @@ class CustomContractPrePostProcess(unittest.TestCase):
         self.assertFalse(self.results.errors)
 
     def test_post_hook_keyboard_interrupt(self):
-        spider = TestSpider()
+        spider = DemoSpider()
         response = ResponseMock()
         contract = CustomFailContractPostProcess(spider.returns_request)
         conman = ContractsManager([contract])

tests/test_crawl.py

@@ -6,7 +6,7 @@ from ipaddress import IPv4Address
 from socket import gethostbyname
 from urllib.parse import urlparse
 
-from pytest import mark
+import pytest
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.internet.ssl import Certificate
@@ -536,7 +536,7 @@ class CrawlSpiderTestCase(TestCase):
             )
         self.assertIn("Got response 200", str(log))
 
-    @mark.only_asyncio()
+    @pytest.mark.only_asyncio
     @defer.inlineCallbacks
     def test_async_def_asyncio_parse(self):
         crawler = get_crawler(
@@ -551,7 +551,7 @@ class CrawlSpiderTestCase(TestCase):
             )
         self.assertIn("Got response 200", str(log))
 
-    @mark.only_asyncio()
+    @pytest.mark.only_asyncio
     @defer.inlineCallbacks
     def test_async_def_asyncio_parse_items_list(self):
         log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider)
@@ -559,7 +559,7 @@ class CrawlSpiderTestCase(TestCase):
         self.assertIn({"id": 1}, items)
         self.assertIn({"id": 2}, items)
 
-    @mark.only_asyncio()
+    @pytest.mark.only_asyncio
     @defer.inlineCallbacks
     def test_async_def_asyncio_parse_items_single_element(self):
         items = []
@@ -576,7 +576,7 @@ class CrawlSpiderTestCase(TestCase):
         self.assertIn("Got response 200", str(log))
         self.assertIn({"foo": 42}, items)
 
-    @mark.only_asyncio()
+    @pytest.mark.only_asyncio
     @defer.inlineCallbacks
     def test_async_def_asyncgen_parse(self):
         log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider)
@@ -584,7 +584,7 @@ class CrawlSpiderTestCase(TestCase):
         itemcount = stats.get_value("item_scraped_count")
         self.assertEqual(itemcount, 1)
 
-    @mark.only_asyncio()
+    @pytest.mark.only_asyncio
     @defer.inlineCallbacks
     def test_async_def_asyncgen_parse_loop(self):
         log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider)
@@ -594,7 +594,7 @@ class CrawlSpiderTestCase(TestCase):
         for i in range(10):
             self.assertIn({"foo": i}, items)
 
-    @mark.only_asyncio()
+    @pytest.mark.only_asyncio
     @defer.inlineCallbacks
     def test_async_def_asyncgen_parse_exc(self):
         log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider)
@@ -606,7 +606,7 @@ class CrawlSpiderTestCase(TestCase):
         for i in range(7):
             self.assertIn({"foo": i}, items)
 
-    @mark.only_asyncio()
+    @pytest.mark.only_asyncio
     @defer.inlineCallbacks
     def test_async_def_asyncgen_parse_complex(self):
         _, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider)
@@ -618,20 +618,20 @@ class CrawlSpiderTestCase(TestCase):
         for i in [10, 30, 122]:
             self.assertIn({"index2": i}, items)
 
-    @mark.only_asyncio()
+    @pytest.mark.only_asyncio
     @defer.inlineCallbacks
     def test_async_def_asyncio_parse_reqs_list(self):
         log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider)
         for req_id in range(3):
             self.assertIn(f"Got response 200, req_id {req_id}", str(log))
 
-    @mark.only_not_asyncio()
+    @pytest.mark.only_not_asyncio
     @defer.inlineCallbacks
     def test_async_def_deferred_direct(self):
         _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider)
         self.assertEqual(items, [{"code": 200}])
 
-    @mark.only_asyncio()
+    @pytest.mark.only_asyncio
     @defer.inlineCallbacks
     def test_async_def_deferred_wrapped(self):
         log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider)
@@ -659,7 +659,9 @@ class CrawlSpiderTestCase(TestCase):
         self.assertEqual(cert.getSubject().commonName, b"localhost")
         self.assertEqual(cert.getIssuer().commonName, b"localhost")
 
-    @mark.xfail(reason="Responses with no body return early and contain no certificate")
+    @pytest.mark.xfail(
+        reason="Responses with no body return early and contain no certificate"
+    )
     @defer.inlineCallbacks
     def test_response_ssl_certificate_empty_response(self):
         crawler = get_crawler(SingleRequestSpider)

tests/test_crawler.py

@@ -8,9 +8,9 @@ import warnings
 from pathlib import Path
 from typing import Any
 
+import pytest
 from packaging.version import parse as parse_version
 from pexpect.popen_spawn import PopenSpawn
-from pytest import mark, raises
 from twisted.internet.defer import Deferred, inlineCallbacks
 from twisted.trial import unittest
 from w3lib import __version__ as w3lib_version
@@ -77,14 +77,14 @@ class CrawlerTestCase(BaseCrawlerTest):
         self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
 
     def test_crawler_rejects_spider_objects(self):
-        with raises(ValueError):
+        with pytest.raises(ValueError, match="spidercls argument must be a class"):
             Crawler(DefaultSpider())
 
     @inlineCallbacks
     def test_crawler_crawl_twice_unsupported(self):
         crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
         yield crawler.crawl()
-        with raises(RuntimeError, match="more than once on the same instance"):
+        with pytest.raises(RuntimeError, match="more than once on the same instance"):
             yield crawler.crawl()
 
     def test_get_addon(self):
@@ -203,7 +203,7 @@ class CrawlerTestCase(BaseCrawlerTest):
                     raise
 
         crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
-        with raises(RuntimeError):
+        with pytest.raises(RuntimeError):
             yield crawler.crawl()
 
     @inlineCallbacks
@@ -282,7 +282,7 @@ class CrawlerTestCase(BaseCrawlerTest):
                     raise
 
         crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
-        with raises(RuntimeError):
+        with pytest.raises(RuntimeError):
             yield crawler.crawl()
 
     @inlineCallbacks
@@ -361,7 +361,7 @@ class CrawlerTestCase(BaseCrawlerTest):
                     raise
 
         crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
-        with raises(RuntimeError):
+        with pytest.raises(RuntimeError):
             yield crawler.crawl()
 
     @inlineCallbacks
@@ -440,7 +440,7 @@ class CrawlerTestCase(BaseCrawlerTest):
                     raise
 
         crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
-        with raises(RuntimeError):
+        with pytest.raises(RuntimeError):
             yield crawler.crawl()
 
 
@@ -575,7 +575,7 @@ class NoRequestsSpider(scrapy.Spider):
         return []
 
 
-@mark.usefixtures("reactor_pytest")
+@pytest.mark.usefixtures("reactor_pytest")
 class CrawlerRunnerHasSpider(unittest.TestCase):
     def _runner(self):
         return CrawlerRunner()
@@ -744,7 +744,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
             "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log
         )
 
-    @mark.skipif(
+    @pytest.mark.skipif(
         parse_version(w3lib_version) >= parse_version("2.0.0"),
         reason="w3lib 2.0.0 and later do not allow invalid domains.",
     )
@@ -781,7 +781,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
             "Using reactor: twisted.internet.selectreactor.SelectReactor", log
         )
 
-    @mark.skipif(
+    @pytest.mark.skipif(
         platform.system() == "Windows", reason="PollReactor is not supported on Windows"
     )
     def test_twisted_reactor_poll(self):
@@ -820,7 +820,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
             log,
         )
 
-    @mark.requires_uvloop
+    @pytest.mark.requires_uvloop
     def test_custom_loop_asyncio(self):
         log = self.run_script("asyncio_custom_loop.py")
         self.assertIn("Spider closed (finished)", log)
@@ -829,7 +829,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         )
         self.assertIn("Using asyncio event loop: uvloop.Loop", log)
 
-    @mark.requires_uvloop
+    @pytest.mark.requires_uvloop
     def test_custom_loop_asyncio_deferred_signal(self):
         log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop")
         self.assertIn("Spider closed (finished)", log)
@@ -839,7 +839,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("Using asyncio event loop: uvloop.Loop", log)
         self.assertIn("async pipeline opened!", log)
 
-    @mark.requires_uvloop
+    @pytest.mark.requires_uvloop
     def test_asyncio_enabled_reactor_same_loop(self):
         log = self.run_script("asyncio_enabled_reactor_same_loop.py")
         self.assertIn("Spider closed (finished)", log)
@@ -848,7 +848,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         )
         self.assertIn("Using asyncio event loop: uvloop.Loop", log)
 
-    @mark.requires_uvloop
+    @pytest.mark.requires_uvloop
     def test_asyncio_enabled_reactor_different_loop(self):
         log = self.run_script("asyncio_enabled_reactor_different_loop.py")
         self.assertNotIn("Spider closed (finished)", log)
@@ -924,13 +924,13 @@ class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("DEBUG: Using asyncio event loop", log)
 
 
-@mark.parametrize(
+@pytest.mark.parametrize(
-    ["settings", "items"],
+    ("settings", "items"),
-    (
+    [
         ({}, default_settings.LOG_VERSIONS),
         ({"LOG_VERSIONS": ["itemadapter"]}, ["itemadapter"]),
         ({"LOG_VERSIONS": []}, None),
-    ),
+    ],
 )
 def test_log_scrapy_info(settings, items, caplog):
     with caplog.at_level("INFO"):

tests/test_downloadermiddleware.py

@@ -1,7 +1,7 @@
 import asyncio
 from unittest import mock
 
-from pytest import mark
+import pytest
 from twisted.internet import defer
 from twisted.internet.defer import Deferred
 from twisted.python.failure import Failure
@@ -220,7 +220,7 @@ class MiddlewareUsingDeferreds(ManagerTestCase):
         self.assertFalse(download_func.called)
 
 
-@mark.usefixtures("reactor_pytest")
+@pytest.mark.usefixtures("reactor_pytest")
 class MiddlewareUsingCoro(ManagerTestCase):
     """Middlewares using asyncio coroutines should work"""
 
@@ -243,7 +243,7 @@ class MiddlewareUsingCoro(ManagerTestCase):
         self.assertIs(results[0], resp)
         self.assertFalse(download_func.called)
 
-    @mark.only_asyncio()
+    @pytest.mark.only_asyncio
     def test_asyncdef_asyncio(self):
         resp = Response("http://example.com/index.html")
 

tests/test_downloadermiddleware_ajaxcrawlable.py

@@ -7,8 +7,6 @@ from scrapy.http import HtmlResponse, Request, Response
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 
-__doctests__ = ["scrapy.downloadermiddlewares.ajaxcrawl"]
-
 
 @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
 class AjaxCrawlMiddlewareTest(unittest.TestCase):

tests/test_downloadermiddleware_httpauth.py

@@ -7,18 +7,18 @@ from scrapy.http import Request
 from scrapy.spiders import Spider
 
 
-class TestSpiderLegacy(Spider):
+class LegacySpider(Spider):
     http_user = "foo"
     http_pass = "bar"
 
 
-class TestSpider(Spider):
+class DomainSpider(Spider):
     http_user = "foo"
     http_pass = "bar"
     http_auth_domain = "example.com"
 
 
-class TestSpiderAny(Spider):
+class AnyDomainSpider(Spider):
     http_user = "foo"
     http_pass = "bar"
     http_auth_domain = None
@@ -26,7 +26,7 @@ class TestSpiderAny(Spider):
 
 class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
     def setUp(self):
-        self.spider = TestSpiderLegacy("foo")
+        self.spider = LegacySpider("foo")
 
     def test_auth(self):
         with self.assertRaises(AttributeError):
@@ -37,7 +37,7 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
 class HttpAuthMiddlewareTest(unittest.TestCase):
     def setUp(self):
         self.mw = HttpAuthMiddleware()
-        self.spider = TestSpider("foo")
+        self.spider = DomainSpider("foo")
         self.mw.spider_opened(self.spider)
 
     def tearDown(self):
@@ -67,7 +67,7 @@ class HttpAuthMiddlewareTest(unittest.TestCase):
 class HttpAuthAnyMiddlewareTest(unittest.TestCase):
     def setUp(self):
         self.mw = HttpAuthMiddleware()
-        self.spider = TestSpiderAny("foo")
+        self.spider = AnyDomainSpider("foo")
         self.mw.spider_opened(self.spider)
 
     def tearDown(self):

tests/test_downloadermiddleware_httpcache.py

@@ -353,7 +353,8 @@ class RFC2616PolicyTest(DefaultStorageTest):
                 resc = mw.storage.retrieve_response(self.spider, req0)
                 if shouldcache:
                     self.assertEqualResponse(resc, res1)
-                    assert "cached" in res2.flags and res2.status != 304
+                    assert "cached" in res2.flags
+                    assert res2.status != 304
                 else:
                     self.assertFalse(resc)
                     assert "cached" not in res2.flags
@@ -376,7 +377,8 @@ class RFC2616PolicyTest(DefaultStorageTest):
                 resc = mw.storage.retrieve_response(self.spider, req0)
                 if shouldcache:
                     self.assertEqualResponse(resc, res1)
-                    assert "cached" in res2.flags and res2.status != 304
+                    assert "cached" in res2.flags
+                    assert res2.status != 304
                 else:
                     self.assertFalse(resc)
                     assert "cached" not in res2.flags

tests/test_downloadermiddleware_httpproxy.py

@@ -131,7 +131,8 @@ class TestHttpProxyMiddleware(TestCase):
         mw = HttpProxyMiddleware()
         req = Request("http://noproxy.com", meta={"proxy": None})
         assert mw.process_request(req, spider) is None
-        assert "proxy" in req.meta and req.meta["proxy"] is None
+        assert "proxy" in req.meta
+        assert req.meta["proxy"] is None
 
     def test_no_proxy(self):
         os.environ["http_proxy"] = "https://proxy.for.http:3128"

tests/test_downloadermiddleware_offsite.py

@@ -12,7 +12,7 @@ UNSET = object()
 
 @pytest.mark.parametrize(
     ("allowed_domain", "url", "allowed"),
-    (
+    [
         ("example.com", "http://example.com/1", True),
         ("example.com", "http://example.org/1", False),
         ("example.com", "http://sub.example.com/1", True),
@@ -24,7 +24,7 @@ UNSET = object()
         ("example.com", "http://example.com.example", False),
         ("a.example", "http://nota.example", False),
         ("b.a.example", "http://notb.a.example", False),
-    ),
+    ],
 )
 def test_process_request_domain_filtering(allowed_domain, url, allowed):
     crawler = get_crawler(Spider)
@@ -41,12 +41,12 @@ def test_process_request_domain_filtering(allowed_domain, url, allowed):
 
 @pytest.mark.parametrize(
     ("value", "filtered"),
-    (
+    [
         (UNSET, True),
         (None, True),
         (False, True),
         (True, False),
-    ),
+    ],
 )
 def test_process_request_dont_filter(value, filtered):
     crawler = get_crawler(Spider)
@@ -66,7 +66,7 @@ def test_process_request_dont_filter(value, filtered):
 
 @pytest.mark.parametrize(
     ("allow_offsite", "dont_filter", "filtered"),
-    (
+    [
         (True, UNSET, False),
         (True, None, False),
         (True, False, False),
@@ -75,7 +75,7 @@ def test_process_request_dont_filter(value, filtered):
         (False, None, True),
         (False, False, True),
         (False, True, False),
-    ),
+    ],
 )
 def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):
     crawler = get_crawler(Spider)
@@ -97,11 +97,11 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):
 
 @pytest.mark.parametrize(
     "value",
-    (
+    [
         UNSET,
         None,
         [],
-    ),
+    ],
 )
 def test_process_request_no_allowed_domains(value):
     crawler = get_crawler(Spider)
@@ -133,7 +133,7 @@ def test_process_request_invalid_domains():
 
 @pytest.mark.parametrize(
     ("allowed_domain", "url", "allowed"),
-    (
+    [
         ("example.com", "http://example.com/1", True),
         ("example.com", "http://example.org/1", False),
         ("example.com", "http://sub.example.com/1", True),
@@ -145,7 +145,7 @@ def test_process_request_invalid_domains():
         ("example.com", "http://example.com.example", False),
         ("a.example", "http://nota.example", False),
         ("b.a.example", "http://notb.a.example", False),
-    ),
+    ],
 )
 def test_request_scheduled_domain_filtering(allowed_domain, url, allowed):
     crawler = get_crawler(Spider)
@@ -162,12 +162,12 @@ def test_request_scheduled_domain_filtering(allowed_domain, url, allowed):
 
 @pytest.mark.parametrize(
     ("value", "filtered"),
-    (
+    [
         (UNSET, True),
         (None, True),
         (False, True),
         (True, False),
-    ),
+    ],
 )
 def test_request_scheduled_dont_filter(value, filtered):
     crawler = get_crawler(Spider)
@@ -187,11 +187,11 @@ def test_request_scheduled_dont_filter(value, filtered):
 
 @pytest.mark.parametrize(
     "value",
-    (
+    [
         UNSET,
         None,
         [],
-    ),
+    ],
 )
 def test_request_scheduled_no_allowed_domains(value):
     crawler = get_crawler(Spider)

tests/test_downloadermiddleware_redirect.py

@@ -1278,7 +1278,7 @@ class MetaRefreshMiddlewareTest(Base.Test):
 
 @pytest.mark.parametrize(
     SCHEME_PARAMS,
-    (
+    [
         *REDIRECT_SCHEME_CASES,
         # data/file/ftp/s3/foo → * does not redirect
         *(
@@ -1300,7 +1300,7 @@ class MetaRefreshMiddlewareTest(Base.Test):
             for scheme in NON_HTTP_SCHEMES
             for location in ("//example.com/b", "/b")
         ),
-    ),
+    ],
 )
 def test_meta_refresh_schemes(url, location, target):
     crawler = get_crawler(Spider)

tests/test_engine.py

@@ -42,7 +42,7 @@ from scrapy.utils.test import get_crawler
 from tests import get_testdata, tests_datadir
 
 
-class TestItem(Item):
+class MyItem(Item):
     name = Field()
     url = Field()
     price = Field()
@@ -62,7 +62,7 @@ class DataClassItem:
     price: int = 0
 
 
-class TestSpider(Spider):
+class MySpider(Spider):
     name = "scrapytest.org"
     allowed_domains = ["scrapytest.org", "localhost"]
 
@@ -70,7 +70,7 @@ class TestSpider(Spider):
     name_re = re.compile(r"<h1>(.*?)</h1>", re.MULTILINE)
     price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE)
 
-    item_cls: type = TestItem
+    item_cls: type = MyItem
 
     def parse(self, response):
         xlink = LinkExtractor()
@@ -91,24 +91,24 @@ class TestSpider(Spider):
         return adapter.item
 
 
-class TestDupeFilterSpider(TestSpider):
+class DupeFilterSpider(MySpider):
     def start_requests(self):
         return (Request(url) for url in self.start_urls)  # no dont_filter=True
 
 
-class DictItemsSpider(TestSpider):
+class DictItemsSpider(MySpider):
     item_cls = dict
 
 
-class AttrsItemsSpider(TestSpider):
+class AttrsItemsSpider(MySpider):
     item_cls = AttrsItem
 
 
-class DataClassItemsSpider(TestSpider):
+class DataClassItemsSpider(MySpider):
     item_cls = DataClassItem
 
 
-class ItemZeroDivisionErrorSpider(TestSpider):
+class ItemZeroDivisionErrorSpider(MySpider):
     custom_settings = {
         "ITEM_PIPELINES": {
             "tests.pipelines.ProcessWithZeroDivisionErrorPipeline": 300,
@@ -116,7 +116,7 @@ class ItemZeroDivisionErrorSpider(TestSpider):
     }
 
 
-class ChangeCloseReasonSpider(TestSpider):
+class ChangeCloseReasonSpider(MySpider):
     @classmethod
     def from_crawler(cls, crawler, *args, **kwargs):
         spider = cls(*args, **kwargs)
@@ -388,7 +388,7 @@ class EngineTest(EngineTestBase):
     @defer.inlineCallbacks
     def test_crawler(self):
         for spider in (
-            TestSpider,
+            MySpider,
             DictItemsSpider,
             AttrsItemsSpider,
             DataClassItemsSpider,
@@ -404,7 +404,7 @@ class EngineTest(EngineTestBase):
 
     @defer.inlineCallbacks
     def test_crawler_dupefilter(self):
-        run = CrawlerRun(TestDupeFilterSpider)
+        run = CrawlerRun(DupeFilterSpider)
         yield run.run()
         self._assert_scheduled_requests(run, count=8)
         self._assert_dropped_requests(run)
@@ -426,13 +426,13 @@ class EngineTest(EngineTestBase):
 
     @defer.inlineCallbacks
     def test_close_downloader(self):
-        e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
+        e = ExecutionEngine(get_crawler(MySpider), lambda _: None)
         yield e.close()
 
     @defer.inlineCallbacks
     def test_start_already_running_exception(self):
-        e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
+        e = ExecutionEngine(get_crawler(MySpider), lambda _: None)
-        yield e.open_spider(TestSpider(), [])
+        yield e.open_spider(MySpider(), [])
         e.start()
         try:
             yield self.assertFailure(e.start(), RuntimeError).addBoth(
@@ -486,7 +486,7 @@ def test_request_scheduled_signal(caplog):
         if "drop" in request.url:
             raise IgnoreRequest
 
-    spider = TestSpider()
+    spider = MySpider()
     crawler = get_crawler(spider.__class__)
     engine = ExecutionEngine(crawler, lambda _: None)
     engine.downloader._slot_gc_loop.stop()

tests/test_engine_stop_download_bytes.py

@@ -8,7 +8,7 @@ from tests.test_engine import (
     DataClassItemsSpider,
     DictItemsSpider,
     EngineTestBase,
-    TestSpider,
+    MySpider,
 )
 
 
@@ -22,7 +22,7 @@ class BytesReceivedEngineTest(EngineTestBase):
     @defer.inlineCallbacks
     def test_crawler(self):
         for spider in (
-            TestSpider,
+            MySpider,
             DictItemsSpider,
             AttrsItemsSpider,
             DataClassItemsSpider,

tests/test_engine_stop_download_headers.py

@@ -8,7 +8,7 @@ from tests.test_engine import (
     DataClassItemsSpider,
     DictItemsSpider,
     EngineTestBase,
-    TestSpider,
+    MySpider,
 )
 
 
@@ -22,7 +22,7 @@ class HeadersReceivedEngineTest(EngineTestBase):
     @defer.inlineCallbacks
     def test_crawler(self):
         for spider in (
-            TestSpider,
+            MySpider,
             DictItemsSpider,
             AttrsItemsSpider,
             DataClassItemsSpider,

tests/test_exporters.py

@@ -31,7 +31,7 @@ def custom_serializer(value):
     return str(int(value) + 2)
 
 
-class TestItem(Item):
+class MyItem(Item):
     name = Field()
     age = Field()
 
@@ -42,7 +42,7 @@ class CustomFieldItem(Item):
 
 
 @dataclasses.dataclass
-class TestDataClass:
+class MyDataClass:
     name: str
     age: int
 
@@ -54,7 +54,7 @@ class CustomFieldDataclass:
 
 
 class BaseItemExporterTest(unittest.TestCase):
-    item_class: type = TestItem
+    item_class: type = MyItem
     custom_field_item_class: type = CustomFieldItem
 
     def setUp(self):
@@ -138,7 +138,7 @@ class BaseItemExporterTest(unittest.TestCase):
 
 
 class BaseItemExporterDataclassTest(BaseItemExporterTest):
-    item_class = TestDataClass
+    item_class = MyDataClass
     custom_field_item_class = CustomFieldDataclass
 
 
@@ -207,7 +207,7 @@ class PythonItemExporterTest(BaseItemExporterTest):
 
 
 class PythonItemExporterDataclassTest(PythonItemExporterTest):
-    item_class = TestDataClass
+    item_class = MyDataClass
     custom_field_item_class = CustomFieldDataclass
 
 
@@ -222,7 +222,7 @@ class PprintItemExporterTest(BaseItemExporterTest):
 
 
 class PprintItemExporterDataclassTest(PprintItemExporterTest):
-    item_class = TestDataClass
+    item_class = MyDataClass
     custom_field_item_class = CustomFieldDataclass
 
 
@@ -259,7 +259,7 @@ class PickleItemExporterTest(BaseItemExporterTest):
 
 
 class PickleItemExporterDataclassTest(PickleItemExporterTest):
-    item_class = TestDataClass
+    item_class = MyDataClass
     custom_field_item_class = CustomFieldDataclass
 
 
@@ -286,7 +286,7 @@ class MarshalItemExporterTest(BaseItemExporterTest):
 
 
 class MarshalItemExporterDataclassTest(MarshalItemExporterTest):
-    item_class = TestDataClass
+    item_class = MyDataClass
     custom_field_item_class = CustomFieldDataclass
 
 
@@ -406,7 +406,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
 
 
 class CsvItemExporterDataclassTest(CsvItemExporterTest):
-    item_class = TestDataClass
+    item_class = MyDataClass
     custom_field_item_class = CustomFieldDataclass
 
 
@@ -517,7 +517,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
 
 
 class XmlItemExporterDataclassTest(XmlItemExporterTest):
-    item_class = TestDataClass
+    item_class = MyDataClass
     custom_field_item_class = CustomFieldDataclass
 
 
@@ -563,7 +563,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
 
 
 class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest):
-    item_class = TestDataClass
+    item_class = MyDataClass
     custom_field_item_class = CustomFieldDataclass
 
 
@@ -595,11 +595,11 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.assertTwoItemsExported(ItemAdapter(self.i).asdict())
 
     def test_two_items_with_failure_between(self):
-        i1 = TestItem(name="Joseph\xa3", age="22")
+        i1 = MyItem(name="Joseph\xa3", age="22")
-        i2 = TestItem(
+        i2 = MyItem(
             name="Maria", age=1j
         )  # Invalid datetimes didn't consistently fail between Python versions
-        i3 = TestItem(name="Jesus", age="44")
+        i3 = MyItem(name="Jesus", age="44")
         self.ie.start_exporting()
         self.ie.export_item(i1)
         self.assertRaises(TypeError, self.ie.export_item, i2)
@@ -652,9 +652,9 @@ class JsonItemExporterToBytesTest(BaseItemExporterTest):
         return JsonItemExporter(self.output, **kwargs)
 
     def test_two_items_with_failure_between(self):
-        i1 = TestItem(name="Joseph", age="22")
+        i1 = MyItem(name="Joseph", age="22")
-        i2 = TestItem(name="\u263a", age="11")
+        i2 = MyItem(name="\u263a", age="11")
-        i3 = TestItem(name="Jesus", age="44")
+        i3 = MyItem(name="Jesus", age="44")
         self.ie.start_exporting()
         self.ie.export_item(i1)
         self.assertRaises(UnicodeEncodeError, self.ie.export_item, i2)
@@ -665,12 +665,12 @@ class JsonItemExporterToBytesTest(BaseItemExporterTest):
 
 
 class JsonItemExporterDataclassTest(JsonItemExporterTest):
-    item_class = TestDataClass
+    item_class = MyDataClass
     custom_field_item_class = CustomFieldDataclass
 
 
 class CustomExporterItemTest(unittest.TestCase):
-    item_class: type = TestItem
+    item_class: type = MyItem
 
     def setUp(self):
         if self.item_class is None:
@@ -700,4 +700,4 @@ class CustomExporterItemTest(unittest.TestCase):
 
 
 class CustomExporterDataclassTest(CustomExporterItemTest):
-    item_class = TestDataClass
+    item_class = MyDataClass

tests/test_extension_periodic_log.py

@@ -51,7 +51,7 @@ stats_dump_2 = {
 }
 
 
-class TestExtPeriodicLog(PeriodicLog):
+class CustomPeriodicLog(PeriodicLog):
     def set_a(self):
         self.stats._stats = stats_dump_1
 
@@ -62,7 +62,7 @@ class TestExtPeriodicLog(PeriodicLog):
 def extension(settings=None):
     crawler = Crawler(MetaSpider, settings=settings)
     crawler._apply_settings()
-    return TestExtPeriodicLog.from_crawler(crawler)
+    return CustomPeriodicLog.from_crawler(crawler)
 
 
 class TestPeriodicLog(unittest.TestCase):

tests/test_extension_throttle.py

@@ -13,15 +13,12 @@ from scrapy.settings.default_settings import (
     DOWNLOAD_DELAY,
 )
 from scrapy.utils.misc import build_from_crawler
+from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler as _get_crawler
 
 UNSET = object()
 
 
-class TestSpider(Spider):
-    name = "test"
-
-
 def get_crawler(settings=None, spidercls=None):
     settings = settings or {}
     settings["AUTOTHROTTLE_ENABLED"] = True
@@ -30,11 +27,11 @@ def get_crawler(settings=None, spidercls=None):
 
 @pytest.mark.parametrize(
     ("value", "expected"),
-    (
+    [
         (UNSET, False),
         (False, False),
         (True, True),
-    ),
+    ],
 )
 def test_enabled(value, expected):
     settings = {}
@@ -50,10 +47,10 @@ def test_enabled(value, expected):
 
 @pytest.mark.parametrize(
     "value",
-    (
+    [
         0.0,
         -1.0,
-    ),
+    ],
 )
 def test_target_concurrency_invalid(value):
     settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": value}
@@ -64,13 +61,13 @@ def test_target_concurrency_invalid(value):
 
 @pytest.mark.parametrize(
     ("spider", "setting", "expected"),
-    (
+    [
         (UNSET, UNSET, DOWNLOAD_DELAY),
         (1.0, UNSET, 1.0),
         (UNSET, 1.0, 1.0),
         (1.0, 2.0, 1.0),
         (3.0, 2.0, 3.0),
-    ),
+    ],
 )
 def test_mindelay_definition(spider, setting, expected):
     settings = {}
@@ -91,10 +88,10 @@ def test_mindelay_definition(spider, setting, expected):
 
 @pytest.mark.parametrize(
     ("value", "expected"),
-    (
+    [
         (UNSET, AUTOTHROTTLE_MAX_DELAY),
         (1.0, 1.0),
-    ),
+    ],
 )
 def test_maxdelay_definition(value, expected):
     settings = {}
@@ -102,13 +99,13 @@ def test_maxdelay_definition(value, expected):
         settings["AUTOTHROTTLE_MAX_DELAY"] = value
     crawler = get_crawler(settings)
     at = build_from_crawler(AutoThrottle, crawler)
-    at._spider_opened(TestSpider())
+    at._spider_opened(DefaultSpider())
     assert at.maxdelay == expected
 
 
 @pytest.mark.parametrize(
     ("min_spider", "min_setting", "start_setting", "expected"),
-    (
+    [
         (UNSET, UNSET, UNSET, AUTOTHROTTLE_START_DELAY),
         (AUTOTHROTTLE_START_DELAY - 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY),
         (AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0),
@@ -134,7 +131,7 @@ def test_maxdelay_definition(value, expected):
             AUTOTHROTTLE_START_DELAY + 2.0,
             AUTOTHROTTLE_START_DELAY + 2.0,
         ),
-    ),
+    ],
 )
 def test_startdelay_definition(min_spider, min_setting, start_setting, expected):
     settings = {}
@@ -158,7 +155,7 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected)
 
 @pytest.mark.parametrize(
     ("meta", "slot"),
-    (
+    [
         ({}, None),
         ({"download_latency": 1.0}, None),
         ({"download_slot": "foo"}, None),
@@ -172,12 +169,12 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected)
             },
             "foo",
         ),
-    ),
+    ],
 )
 def test_skipped(meta, slot):
     crawler = get_crawler()
     at = build_from_crawler(AutoThrottle, crawler)
-    spider = TestSpider()
+    spider = DefaultSpider()
     at._spider_opened(spider)
     request = Request("https://example.com", meta=meta)
 
@@ -193,7 +190,7 @@ def test_skipped(meta, slot):
 
 @pytest.mark.parametrize(
     ("download_latency", "target_concurrency", "slot_delay", "expected"),
-    (
+    [
         (2.0, 2.0, 1.0, 1.0),
         (1.0, 2.0, 1.0, 0.75),
         (4.0, 2.0, 1.0, 2.0),
@@ -201,13 +198,13 @@ def test_skipped(meta, slot):
         (2.0, 4.0, 1.0, 0.75),
         (2.0, 2.0, 0.5, 1.0),
         (2.0, 2.0, 2.0, 1.5),
-    ),
+    ],
 )
 def test_adjustment(download_latency, target_concurrency, slot_delay, expected):
     settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency}
     crawler = get_crawler(settings)
     at = build_from_crawler(AutoThrottle, crawler)
-    spider = TestSpider()
+    spider = DefaultSpider()
     at._spider_opened(spider)
     meta = {"download_latency": download_latency, "download_slot": "foo"}
     request = Request("https://example.com", meta=meta)
@@ -227,11 +224,11 @@ def test_adjustment(download_latency, target_concurrency, slot_delay, expected):
 
 @pytest.mark.parametrize(
     ("mindelay", "maxdelay", "expected"),
-    (
+    [
         (0.5, 2.0, 1.0),
         (0.25, 0.5, 0.5),
         (2.0, 4.0, 2.0),
-    ),
+    ],
 )
 def test_adjustment_limits(mindelay, maxdelay, expected):
     download_latency, target_concurrency, slot_delay = (2.0, 2.0, 1.0)
@@ -243,7 +240,7 @@ def test_adjustment_limits(mindelay, maxdelay, expected):
     }
     crawler = get_crawler(settings)
     at = build_from_crawler(AutoThrottle, crawler)
-    spider = TestSpider()
+    spider = DefaultSpider()
     at._spider_opened(spider)
     meta = {"download_latency": download_latency, "download_slot": "foo"}
     request = Request("https://example.com", meta=meta)
@@ -263,11 +260,11 @@ def test_adjustment_limits(mindelay, maxdelay, expected):
 
 @pytest.mark.parametrize(
     ("download_latency", "target_concurrency", "slot_delay", "expected"),
-    (
+    [
         (2.0, 2.0, 1.0, 1.0),
         (1.0, 2.0, 1.0, 1.0),  # Instead of 0.75
         (4.0, 2.0, 1.0, 2.0),
-    ),
+    ],
 )
 def test_adjustment_bad_response(
     download_latency, target_concurrency, slot_delay, expected
@@ -275,7 +272,7 @@ def test_adjustment_bad_response(
     settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency}
     crawler = get_crawler(settings)
     at = build_from_crawler(AutoThrottle, crawler)
-    spider = TestSpider()
+    spider = DefaultSpider()
     at._spider_opened(spider)
     meta = {"download_latency": download_latency, "download_slot": "foo"}
     request = Request("https://example.com", meta=meta)
@@ -297,7 +294,7 @@ def test_debug(caplog):
     settings = {"AUTOTHROTTLE_DEBUG": True}
     crawler = get_crawler(settings)
     at = build_from_crawler(AutoThrottle, crawler)
-    spider = TestSpider()
+    spider = DefaultSpider()
     at._spider_opened(spider)
     meta = {"download_latency": 1.0, "download_slot": "foo"}
     request = Request("https://example.com", meta=meta)
@@ -327,7 +324,7 @@ def test_debug(caplog):
 def test_debug_disabled(caplog):
     crawler = get_crawler()
     at = build_from_crawler(AutoThrottle, crawler)
-    spider = TestSpider()
+    spider = DefaultSpider()
     at._spider_opened(spider)
     meta = {"download_latency": 1.0, "download_slot": "foo"}
     request = Request("https://example.com", meta=meta)

tests/test_http2_client_protocol.py

@@ -258,7 +258,8 @@ class Https2ClientProtocolTestCase(TestCase):
         :param path: Should have / at the starting compulsorily if not empty
         :return: Complete url
         """
-        assert len(path) > 0 and (path[0] == "/" or path[0] == "&")
+        assert len(path) > 0
+        assert path[0] == "/" or path[0] == "&"
         return f"{self.scheme}://{self.hostname}:{self.port_number}{path}"
 
     def make_request(self, request: Request) -> Deferred:

tests/test_http_response.py

@@ -2,8 +2,8 @@ import codecs
 import unittest
 from unittest import mock
 
+import pytest
 from packaging.version import Version as parse_version
-from pytest import mark
 from w3lib import __version__ as w3lib_version
 from w3lib.encoding import resolve_encoding
 
@@ -218,7 +218,7 @@ class BaseResponseTest(unittest.TestCase):
         r = self.response_class("http://example.com")
         self.assertRaises(ValueError, r.follow, None)
 
-    @mark.xfail(
+    @pytest.mark.xfail(
         parse_version(w3lib_version) < parse_version("2.1.1"),
         reason="https://github.com/scrapy/w3lib/pull/207",
         strict=True,
@@ -226,7 +226,7 @@ class BaseResponseTest(unittest.TestCase):
     def test_follow_whitespace_url(self):
         self._assert_followed_url("foo ", "http://example.com/foo")
 
-    @mark.xfail(
+    @pytest.mark.xfail(
         parse_version(w3lib_version) < parse_version("2.1.1"),
         reason="https://github.com/scrapy/w3lib/pull/207",
         strict=True,
@@ -473,10 +473,8 @@ class TextResponseTest(BaseResponseTest):
         self._assert_response_encoding(r5, "utf-8")
         self._assert_response_encoding(r8, "utf-8")
         self._assert_response_encoding(r9, "cp1252")
-        assert (
+        assert r4._body_inferred_encoding() is not None
-            r4._body_inferred_encoding() is not None
+        assert r4._body_inferred_encoding() != "ascii"
-            and r4._body_inferred_encoding() != "ascii"
-        )
         self._assert_response_values(r1, "utf-8", "\xa3")
         self._assert_response_values(r2, "utf-8", "\xa3")
         self._assert_response_values(r3, "iso-8859-1", "\xa3")

tests/test_linkextractors.py

@@ -4,8 +4,8 @@ import pickle
 import re
 import unittest
 
+import pytest
 from packaging.version import Version
-from pytest import mark
 from w3lib import __version__ as w3lib_version
 
 from scrapy.http import HtmlResponse, XmlResponse
@@ -930,7 +930,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
             ],
         )
 
-    @mark.skipif(
+    @pytest.mark.skipif(
         Version(w3lib_version) < Version("2.0.0"),
         reason=(
             "Before w3lib 2.0.0, w3lib.url.safe_url_string would not complain "

tests/test_loader.py

@@ -18,12 +18,12 @@ class NameItem(Item):
     name = Field()
 
 
-class TestItem(NameItem):
+class SummaryItem(NameItem):
     url = Field()
     summary = Field()
 
 
-class TestNestedItem(Item):
+class NestedItem(Item):
     name = Field()
     name_div = Field()
     name_value = Field()
@@ -38,20 +38,20 @@ class AttrsNameItem:
 
 
 @dataclasses.dataclass
-class TestDataClass:
+class NameDataClass:
     name: list = dataclasses.field(default_factory=list)
 
 
 # test item loaders
 class NameItemLoader(ItemLoader):
-    default_item_class = TestItem
+    default_item_class = SummaryItem
 
 
 class NestedItemLoader(ItemLoader):
-    default_item_class = TestNestedItem
+    default_item_class = NestedItem
 
 
-class TestItemLoader(NameItemLoader):
+class ProcessorItemLoader(NameItemLoader):
     name_in = MapCompose(lambda v: v.title())
 
 
@@ -68,11 +68,11 @@ def processor_with_args(value, other=None, loader_context=None):
 
 class BasicItemLoaderTest(unittest.TestCase):
     def test_add_value_on_unknown_field(self):
-        il = TestItemLoader()
+        il = ProcessorItemLoader()
         self.assertRaises(KeyError, il.add_value, "wrong_field", ["lala", "lolo"])
 
     def test_load_item_using_default_loader(self):
-        i = TestItem()
+        i = SummaryItem()
         i["summary"] = "lala"
         il = ItemLoader(item=i)
         il.add_value("name", "marta")
@@ -82,7 +82,7 @@ class BasicItemLoaderTest(unittest.TestCase):
         self.assertEqual(item["name"], ["marta"])
 
     def test_load_item_using_custom_loader(self):
-        il = TestItemLoader()
+        il = ProcessorItemLoader()
         il.add_value("name", "marta")
         item = il.load_item()
         self.assertEqual(item["name"], ["Marta"])
@@ -194,7 +194,7 @@ class InitializationFromAttrsItemTest(InitializationTestMixin, unittest.TestCase
 
 
 class InitializationFromDataClassTest(InitializationTestMixin, unittest.TestCase):
-    item_class = TestDataClass
+    item_class = NameDataClass
 
 
 class BaseNoInputReprocessingLoader(ItemLoader):
@@ -289,11 +289,11 @@ class SelectortemLoaderTest(unittest.TestCase):
     )
 
     def test_init_method(self):
-        l = TestItemLoader()
+        l = ProcessorItemLoader()
         self.assertEqual(l.selector, None)
 
     def test_init_method_errors(self):
-        l = TestItemLoader()
+        l = ProcessorItemLoader()
         self.assertRaises(RuntimeError, l.add_xpath, "url", "//a/@href")
         self.assertRaises(RuntimeError, l.replace_xpath, "url", "//a/@href")
         self.assertRaises(RuntimeError, l.get_xpath, "//a/@href")
@@ -303,7 +303,7 @@ class SelectortemLoaderTest(unittest.TestCase):
 
     def test_init_method_with_selector(self):
         sel = Selector(text="<html><body><div>marta</div></body></html>")
-        l = TestItemLoader(selector=sel)
+        l = ProcessorItemLoader(selector=sel)
         self.assertIs(l.selector, sel)
 
         l.add_xpath("name", "//div/text()")
@@ -311,7 +311,7 @@ class SelectortemLoaderTest(unittest.TestCase):
 
     def test_init_method_with_selector_css(self):
         sel = Selector(text="<html><body><div>marta</div></body></html>")
-        l = TestItemLoader(selector=sel)
+        l = ProcessorItemLoader(selector=sel)
         self.assertIs(l.selector, sel)
 
         l.add_css("name", "div::text")
@@ -320,18 +320,18 @@ class SelectortemLoaderTest(unittest.TestCase):
     def test_init_method_with_base_response(self):
         """Selector should be None after initialization"""
         response = Response("https://scrapy.org")
-        l = TestItemLoader(response=response)
+        l = ProcessorItemLoader(response=response)
         self.assertIs(l.selector, None)
 
     def test_init_method_with_response(self):
-        l = TestItemLoader(response=self.response)
+        l = ProcessorItemLoader(response=self.response)
         self.assertTrue(l.selector)
 
         l.add_xpath("name", "//div/text()")
         self.assertEqual(l.get_output_value("name"), ["Marta"])
 
     def test_init_method_with_response_css(self):
-        l = TestItemLoader(response=self.response)
+        l = ProcessorItemLoader(response=self.response)
         self.assertTrue(l.selector)
 
         l.add_css("name", "div::text")
@@ -350,12 +350,12 @@ class SelectortemLoaderTest(unittest.TestCase):
         )
 
     def test_add_xpath_re(self):
-        l = TestItemLoader(response=self.response)
+        l = ProcessorItemLoader(response=self.response)
         l.add_xpath("name", "//div/text()", re="ma")
         self.assertEqual(l.get_output_value("name"), ["Ma"])
 
     def test_replace_xpath(self):
-        l = TestItemLoader(response=self.response)
+        l = ProcessorItemLoader(response=self.response)
         self.assertTrue(l.selector)
         l.add_xpath("name", "//div/text()")
         self.assertEqual(l.get_output_value("name"), ["Marta"])
@@ -366,7 +366,7 @@ class SelectortemLoaderTest(unittest.TestCase):
         self.assertEqual(l.get_output_value("name"), ["Paragraph", "Marta"])
 
     def test_get_xpath(self):
-        l = TestItemLoader(response=self.response)
+        l = ProcessorItemLoader(response=self.response)
         self.assertEqual(l.get_xpath("//p/text()"), ["paragraph"])
         self.assertEqual(l.get_xpath("//p/text()", TakeFirst()), "paragraph")
         self.assertEqual(l.get_xpath("//p/text()", TakeFirst(), re="pa"), "pa")
@@ -376,14 +376,14 @@ class SelectortemLoaderTest(unittest.TestCase):
         )
 
     def test_replace_xpath_multi_fields(self):
-        l = TestItemLoader(response=self.response)
+        l = ProcessorItemLoader(response=self.response)
         l.add_xpath(None, "//div/text()", TakeFirst(), lambda x: {"name": x})
         self.assertEqual(l.get_output_value("name"), ["Marta"])
         l.replace_xpath(None, "//p/text()", TakeFirst(), lambda x: {"name": x})
         self.assertEqual(l.get_output_value("name"), ["Paragraph"])
 
     def test_replace_xpath_re(self):
-        l = TestItemLoader(response=self.response)
+        l = ProcessorItemLoader(response=self.response)
         self.assertTrue(l.selector)
         l.add_xpath("name", "//div/text()")
         self.assertEqual(l.get_output_value("name"), ["Marta"])
@@ -391,7 +391,7 @@ class SelectortemLoaderTest(unittest.TestCase):
         self.assertEqual(l.get_output_value("name"), ["Ma"])
 
     def test_add_css_re(self):
-        l = TestItemLoader(response=self.response)
+        l = ProcessorItemLoader(response=self.response)
         l.add_css("name", "div::text", re="ma")
         self.assertEqual(l.get_output_value("name"), ["Ma"])
 
@@ -399,7 +399,7 @@ class SelectortemLoaderTest(unittest.TestCase):
         self.assertEqual(l.get_output_value("url"), ["www.scrapy.org"])
 
     def test_replace_css(self):
-        l = TestItemLoader(response=self.response)
+        l = ProcessorItemLoader(response=self.response)
         self.assertTrue(l.selector)
         l.add_css("name", "div::text")
         self.assertEqual(l.get_output_value("name"), ["Marta"])
@@ -415,7 +415,7 @@ class SelectortemLoaderTest(unittest.TestCase):
         self.assertEqual(l.get_output_value("url"), ["/images/logo.png"])
 
     def test_get_css(self):
-        l = TestItemLoader(response=self.response)
+        l = ProcessorItemLoader(response=self.response)
         self.assertEqual(l.get_css("p::text"), ["paragraph"])
         self.assertEqual(l.get_css("p::text", TakeFirst()), "paragraph")
         self.assertEqual(l.get_css("p::text", TakeFirst(), re="pa"), "pa")
@@ -427,7 +427,7 @@ class SelectortemLoaderTest(unittest.TestCase):
         )
 
     def test_replace_css_multi_fields(self):
-        l = TestItemLoader(response=self.response)
+        l = ProcessorItemLoader(response=self.response)
         l.add_css(None, "div::text", TakeFirst(), lambda x: {"name": x})
         self.assertEqual(l.get_output_value("name"), ["Marta"])
         l.replace_css(None, "p::text", TakeFirst(), lambda x: {"name": x})
@@ -439,7 +439,7 @@ class SelectortemLoaderTest(unittest.TestCase):
         self.assertEqual(l.get_output_value("url"), ["/images/logo.png"])
 
     def test_replace_css_re(self):
-        l = TestItemLoader(response=self.response)
+        l = ProcessorItemLoader(response=self.response)
         self.assertTrue(l.selector)
         l.add_css("url", "a::attr(href)")
         self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"])

tests/test_loader_deprecated.py

@@ -24,17 +24,17 @@ class NameItem(Item):
     name = Field()
 
 
-class TestItem(NameItem):
+class SummaryItem(NameItem):
     url = Field()
     summary = Field()
 
 
 # test item loaders
 class NameItemLoader(ItemLoader):
-    default_item_class = TestItem
+    default_item_class = SummaryItem
 
 
-class TestItemLoader(NameItemLoader):
+class ProcessorItemLoader(NameItemLoader):
     name_in = MapCompose(lambda v: v.title())
 
 
@@ -51,7 +51,7 @@ def processor_with_args(value, other=None, loader_context=None):
 
 class BasicItemLoaderTest(unittest.TestCase):
     def test_load_item_using_default_loader(self):
-        i = TestItem()
+        i = SummaryItem()
         i["summary"] = "lala"
         il = ItemLoader(item=i)
         il.add_value("name", "marta")
@@ -61,7 +61,7 @@ class BasicItemLoaderTest(unittest.TestCase):
         self.assertEqual(item["name"], ["marta"])
 
     def test_load_item_using_custom_loader(self):
-        il = TestItemLoader()
+        il = ProcessorItemLoader()
         il.add_value("name", "marta")
         item = il.load_item()
         self.assertEqual(item["name"], ["Marta"])
@@ -125,7 +125,7 @@ class BasicItemLoaderTest(unittest.TestCase):
         )
 
     def test_add_value(self):
-        il = TestItemLoader()
+        il = ProcessorItemLoader()
         il.add_value("name", "marta")
         self.assertEqual(il.get_collected_values("name"), ["Marta"])
         self.assertEqual(il.get_output_value("name"), ["Marta"])
@@ -146,7 +146,7 @@ class BasicItemLoaderTest(unittest.TestCase):
         self.assertEqual(il.get_collected_values("name"), [0])
 
     def test_replace_value(self):
-        il = TestItemLoader()
+        il = ProcessorItemLoader()
         il.replace_value("name", "marta")
         self.assertEqual(il.get_collected_values("name"), ["Marta"])
         self.assertEqual(il.get_output_value("name"), ["Marta"])
@@ -229,7 +229,7 @@ class BasicItemLoaderTest(unittest.TestCase):
         self.assertEqual(il.get_output_value("name"), ["mart"])
 
     def test_input_processor_inheritance(self):
-        class ChildItemLoader(TestItemLoader):
+        class ChildItemLoader(ProcessorItemLoader):
             url_in = MapCompose(lambda v: v.lower())
 
         il = ChildItemLoader()
@@ -265,8 +265,8 @@ class BasicItemLoaderTest(unittest.TestCase):
         self.assertEqual(il.get_output_value("name"), ["marta"])
 
     def test_extend_custom_input_processors(self):
-        class ChildItemLoader(TestItemLoader):
+        class ChildItemLoader(ProcessorItemLoader):
-            name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
+            name_in = MapCompose(ProcessorItemLoader.name_in, str.swapcase)
 
         il = ChildItemLoader()
         il.add_value("name", "marta")
@@ -283,11 +283,11 @@ class BasicItemLoaderTest(unittest.TestCase):
         self.assertEqual(il.get_output_value("name"), ["MART"])
 
     def test_output_processor_using_function(self):
-        il = TestItemLoader()
+        il = ProcessorItemLoader()
         il.add_value("name", ["mar", "ta"])
         self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"])
 
-        class TakeFirstItemLoader(TestItemLoader):
+        class TakeFirstItemLoader(ProcessorItemLoader):
             name_out = " ".join
 
         il = TakeFirstItemLoader()
@@ -296,7 +296,7 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_output_processor_error(self):
         class TestItemLoader(ItemLoader):
-            default_item_class = TestItem
+            default_item_class = SummaryItem
             name_out = MapCompose(float)
 
         il = TestItemLoader()
@@ -319,18 +319,18 @@ class BasicItemLoaderTest(unittest.TestCase):
         assert expected_exc_str in s, s
 
     def test_output_processor_using_classes(self):
-        il = TestItemLoader()
+        il = ProcessorItemLoader()
         il.add_value("name", ["mar", "ta"])
         self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"])
 
-        class TakeFirstItemLoader(TestItemLoader):
+        class TakeFirstItemLoader(ProcessorItemLoader):
             name_out = Join()
 
         il = TakeFirstItemLoader()
         il.add_value("name", ["mar", "ta"])
         self.assertEqual(il.get_output_value("name"), "Mar Ta")
 
-        class TakeFirstItemLoader2(TestItemLoader):
+        class TakeFirstItemLoader2(ProcessorItemLoader):
             name_out = Join("<br>")
 
         il = TakeFirstItemLoader2()
@@ -338,11 +338,11 @@ class BasicItemLoaderTest(unittest.TestCase):
         self.assertEqual(il.get_output_value("name"), "Mar<br>Ta")
 
     def test_default_output_processor(self):
-        il = TestItemLoader()
+        il = ProcessorItemLoader()
         il.add_value("name", ["mar", "ta"])
         self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"])
 
-        class LalaItemLoader(TestItemLoader):
+        class LalaItemLoader(ProcessorItemLoader):
             default_output_processor = Identity()
 
         il = LalaItemLoader()
@@ -350,7 +350,7 @@ class BasicItemLoaderTest(unittest.TestCase):
         self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"])
 
     def test_loader_context_on_declaration(self):
-        class ChildItemLoader(TestItemLoader):
+        class ChildItemLoader(ProcessorItemLoader):
             url_in = MapCompose(processor_with_args, key="val")
 
         il = ChildItemLoader()
@@ -360,7 +360,7 @@ class BasicItemLoaderTest(unittest.TestCase):
         self.assertEqual(il.get_output_value("url"), ["val"])
 
     def test_loader_context_on_instantiation(self):
-        class ChildItemLoader(TestItemLoader):
+        class ChildItemLoader(ProcessorItemLoader):
             url_in = MapCompose(processor_with_args)
 
         il = ChildItemLoader(key="val")
@@ -370,7 +370,7 @@ class BasicItemLoaderTest(unittest.TestCase):
         self.assertEqual(il.get_output_value("url"), ["val"])
 
     def test_loader_context_on_assign(self):
-        class ChildItemLoader(TestItemLoader):
+        class ChildItemLoader(ProcessorItemLoader):
             url_in = MapCompose(processor_with_args)
 
         il = ChildItemLoader()
@@ -384,10 +384,10 @@ class BasicItemLoaderTest(unittest.TestCase):
         def processor(value, loader_context):
             return loader_context["item"]["name"]
 
-        class ChildItemLoader(TestItemLoader):
+        class ChildItemLoader(ProcessorItemLoader):
             url_in = MapCompose(processor)
 
-        it = TestItem(name="marta")
+        it = SummaryItem(name="marta")
         il = ChildItemLoader(item=it)
         il.add_value("url", "text")
         self.assertEqual(il.get_output_value("url"), ["marta"])

tests/test_middleware.py

@@ -40,7 +40,7 @@ class MOff:
         raise NotConfigured("foo")
 
 
-class TestMiddlewareManager(MiddlewareManager):
+class MyMiddlewareManager(MiddlewareManager):
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
         return [M1, MOff, M3]
@@ -54,7 +54,7 @@ class TestMiddlewareManager(MiddlewareManager):
 class MiddlewareManagerTest(unittest.TestCase):
     def test_init(self):
         m1, m2, m3 = M1(), M2(), M3()
-        mwman = TestMiddlewareManager(m1, m2, m3)
+        mwman = MyMiddlewareManager(m1, m2, m3)
         self.assertEqual(
             list(mwman.methods["open_spider"]), [m1.open_spider, m2.open_spider]
         )
@@ -64,7 +64,7 @@ class MiddlewareManagerTest(unittest.TestCase):
         self.assertEqual(list(mwman.methods["process"]), [m1.process, m3.process])
 
     def test_methods(self):
-        mwman = TestMiddlewareManager(M1(), M2(), M3())
+        mwman = MyMiddlewareManager(M1(), M2(), M3())
         self.assertEqual(
             [x.__self__.__class__ for x in mwman.methods["open_spider"]], [M1, M2]
         )
@@ -82,6 +82,6 @@ class MiddlewareManagerTest(unittest.TestCase):
 
     def test_enabled_from_settings(self):
         crawler = get_crawler()
-        mwman = TestMiddlewareManager.from_crawler(crawler)
+        mwman = MyMiddlewareManager.from_crawler(crawler)
         classes = [x.__class__ for x in mwman.middlewares]
         self.assertEqual(classes, [M1, M3])

tests/test_pipelines.py

@@ -1,6 +1,6 @@
 import asyncio
 
-from pytest import mark
+import pytest
 from twisted.internet import defer
 from twisted.internet.defer import Deferred
 from twisted.trial import unittest
@@ -118,14 +118,14 @@ class PipelineTestCase(unittest.TestCase):
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertEqual(len(self.items), 1)
 
-    @mark.only_asyncio()
+    @pytest.mark.only_asyncio
     @defer.inlineCallbacks
     def test_asyncdef_asyncio_pipeline(self):
         crawler = self._create_crawler(AsyncDefAsyncioPipeline)
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertEqual(len(self.items), 1)
 
-    @mark.only_not_asyncio()
+    @pytest.mark.only_not_asyncio
     @defer.inlineCallbacks
     def test_asyncdef_not_asyncio_pipeline(self):
         crawler = self._create_crawler(AsyncDefNotAsyncioPipeline)

tests/test_request_dict.py

@@ -11,7 +11,7 @@ class CustomRequest(Request):
 
 class RequestSerializationTest(unittest.TestCase):
     def setUp(self):
-        self.spider = TestSpider()
+        self.spider = MethodsSpider()
 
     def test_basic(self):
         r = Request("http://www.example.com")
@@ -96,18 +96,22 @@ class RequestSerializationTest(unittest.TestCase):
     def test_private_reference_callback_serialization(self):
         r = Request(
             "http://www.example.com",
-            callback=self.spider._TestSpider__parse_item_reference,
+            callback=self.spider._MethodsSpider__parse_item_reference,
-            errback=self.spider._TestSpider__handle_error_reference,
+            errback=self.spider._MethodsSpider__handle_error_reference,
         )
         self._assert_serializes_ok(r, spider=self.spider)
         request_dict = r.to_dict(spider=self.spider)
-        self.assertEqual(request_dict["callback"], "_TestSpider__parse_item_reference")
+        self.assertEqual(
-        self.assertEqual(request_dict["errback"], "_TestSpider__handle_error_reference")
+            request_dict["callback"], "_MethodsSpider__parse_item_reference"
+        )
+        self.assertEqual(
+            request_dict["errback"], "_MethodsSpider__handle_error_reference"
+        )
 
     def test_private_callback_serialization(self):
         r = Request(
             "http://www.example.com",
-            callback=self.spider._TestSpider__parse_item_private,
+            callback=self.spider._MethodsSpider__parse_item_private,
             errback=self.spider.handle_error,
         )
         self._assert_serializes_ok(r, spider=self.spider)
@@ -115,7 +119,7 @@ class RequestSerializationTest(unittest.TestCase):
     def test_mixin_private_callback_serialization(self):
         r = Request(
             "http://www.example.com",
-            callback=self.spider._TestSpiderMixin__mixin_callback,
+            callback=self.spider._SpiderMixin__mixin_callback,
             errback=self.spider.handle_error,
         )
         self._assert_serializes_ok(r, spider=self.spider)
@@ -152,18 +156,18 @@ class RequestSerializationTest(unittest.TestCase):
 
     def test_callback_not_available(self):
         """Callback method is not available in the spider passed to from_dict"""
-        spider = TestSpiderDelegation()
+        spider = SpiderDelegation()
         r = Request("http://www.example.com", callback=spider.delegated_callback)
         d = r.to_dict(spider=spider)
         self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo"))
 
 
-class TestSpiderMixin:
+class SpiderMixin:
     def __mixin_callback(self, response):  # pylint: disable=unused-private-member
         pass
 
 
-class TestSpiderDelegation:
+class SpiderDelegation:
     def delegated_callback(self, response):
         pass
 
@@ -184,7 +188,7 @@ def private_handle_error(failure):
     pass
 
 
-class TestSpider(Spider, TestSpiderMixin):
+class MethodsSpider(Spider, SpiderMixin):
     name = "test"
     parse_item_reference = parse_item
     handle_error_reference = handle_error
@@ -193,7 +197,7 @@ class TestSpider(Spider, TestSpiderMixin):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
-        self.delegated_callback = TestSpiderDelegation().delegated_callback
+        self.delegated_callback = SpiderDelegation().delegated_callback
 
     def parse_item(self, response):
         pass

tests/test_scheduler_base.py

@@ -51,8 +51,8 @@ class SimpleScheduler(MinimalScheduler):
         return len(self.requests)
 
 
-class TestSpider(Spider):
+class PathsSpider(Spider):
-    name = "test"
+    name = "paths"
 
     def __init__(self, mockserver, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -155,7 +155,7 @@ class MinimalSchedulerCrawlTest(TwistedTestCase):
                 "SCHEDULER": self.scheduler_cls,
             }
             with LogCapture() as log:
-                crawler = get_crawler(TestSpider, settings)
+                crawler = get_crawler(PathsSpider, settings)
                 yield crawler.crawl(mockserver)
             for path in PATHS:
                 self.assertIn(f"{{'path': '{path}'}}", str(log))

tests/test_signals.py

@@ -1,4 +1,4 @@
-from pytest import mark
+import pytest
 from twisted.internet import defer
 from twisted.trial import unittest
 
@@ -37,7 +37,7 @@ class AsyncSignalTestCase(unittest.TestCase):
         item = await get_from_asyncio_queue(item)
         self.items.append(item)
 
-    @mark.only_asyncio()
+    @pytest.mark.only_asyncio
     @defer.inlineCallbacks
     def test_simple_pipeline(self):
         crawler = get_crawler(ItemSpider)

tests/test_squeues.py

@@ -15,7 +15,7 @@ from scrapy.squeues import (
 )
 
 
-class TestItem(Item):
+class MyItem(Item):
     name = Field()
 
 
@@ -23,8 +23,8 @@ def _test_procesor(x):
     return x + x
 
 
-class TestLoader(ItemLoader):
+class MyLoader(ItemLoader):
-    default_item_class = TestItem
+    default_item_class = MyItem
     name_out = staticmethod(_test_procesor)
 
 
@@ -80,19 +80,19 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
 
     def test_serialize_item(self):
         q = self.queue()
-        i = TestItem(name="foo")
+        i = MyItem(name="foo")
         q.push(i)
         i2 = q.pop()
-        assert isinstance(i2, TestItem)
+        assert isinstance(i2, MyItem)
         self.assertEqual(i, i2)
 
     def test_serialize_loader(self):
         q = self.queue()
-        loader = TestLoader()
+        loader = MyLoader()
         q.push(loader)
         loader2 = q.pop()
-        assert isinstance(loader2, TestLoader)
+        assert isinstance(loader2, MyLoader)
-        assert loader2.default_item_class is TestItem
+        assert loader2.default_item_class is MyItem
         self.assertEqual(loader2.name_out("x"), "xx")
 
     def test_serialize_request_recursive(self):
@@ -161,19 +161,19 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
 
     def test_serialize_item(self):
         q = self.queue()
-        i = TestItem(name="foo")
+        i = MyItem(name="foo")
         q.push(i)
         i2 = q.pop()
-        assert isinstance(i2, TestItem)
+        assert isinstance(i2, MyItem)
         self.assertEqual(i, i2)
 
     def test_serialize_loader(self):
         q = self.queue()
-        loader = TestLoader()
+        loader = MyLoader()
         q.push(loader)
         loader2 = q.pop()
-        assert isinstance(loader2, TestLoader)
+        assert isinstance(loader2, MyLoader)
-        assert loader2.default_item_class is TestItem
+        assert loader2.default_item_class is MyItem
         self.assertEqual(loader2.name_out("x"), "xx")
 
     def test_serialize_request_recursive(self):

tests/test_utils_asyncio.py

@@ -1,7 +1,7 @@
 import asyncio
 import warnings
 
-from pytest import mark
+import pytest
 from twisted.trial.unittest import TestCase
 
 from scrapy.utils.defer import deferred_f_from_coro_f
@@ -12,7 +12,7 @@ from scrapy.utils.reactor import (
 )
 
 
-@mark.usefixtures("reactor_pytest")
+@pytest.mark.usefixtures("reactor_pytest")
 class AsyncioTest(TestCase):
     def test_is_asyncio_reactor_installed(self):
         # the result should depend only on the pytest --reactor argument
@@ -30,7 +30,7 @@ class AsyncioTest(TestCase):
 
         assert original_reactor == reactor
 
-    @mark.only_asyncio()
+    @pytest.mark.only_asyncio
     @deferred_f_from_coro_f
     async def test_set_asyncio_event_loop(self):
         install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")

tests/test_utils_datatypes.py

@@ -16,8 +16,6 @@ from scrapy.utils.datatypes import (
 )
 from scrapy.utils.python import garbage_collect
 
-__doctests__ = ["scrapy.utils.datatypes"]
-
 
 class CaseInsensitiveDictMixin:
     def test_init_dict(self):

tests/test_utils_defer.py

@@ -1,6 +1,6 @@
 import random
 
-from pytest import mark
+import pytest
 from twisted.internet import defer, reactor
 from twisted.python.failure import Failure
 from twisted.trial import unittest
@@ -150,7 +150,7 @@ class AsyncDefTestsuiteTest(unittest.TestCase):
     async def test_deferred_f_from_coro_f_generator(self):
         yield
 
-    @mark.xfail(reason="Checks that the test is actually executed", strict=True)
+    @pytest.mark.xfail(reason="Checks that the test is actually executed", strict=True)
     @deferred_f_from_coro_f
     async def test_deferred_f_from_coro_f_xfail(self):
         raise RuntimeError("This is expected to be raised")

tests/test_utils_log.py

@@ -119,7 +119,7 @@ class StreamLoggerTest(unittest.TestCase):
 
 @pytest.mark.parametrize(
     ("base_extra", "log_extra", "expected_extra"),
-    (
+    [
         (
             {"spider": "test"},
             {"extra": {"log_extra": "info"}},
@@ -135,7 +135,7 @@ class StreamLoggerTest(unittest.TestCase):
             {"extra": {"spider": "test2"}},
             {"extra": {"spider": "test"}},
         ),
-    ),
+    ],
 )
 def test_spider_logger_adapter_process(
     base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict

tests/test_utils_misc/__init__.py

@@ -17,8 +17,6 @@ from scrapy.utils.misc import (
     walk_modules,
 )
 
-__doctests__ = ["scrapy.utils.misc"]
-
 
 class UtilsMiscTestCase(unittest.TestCase):
     def test_load_object_class(self):

tests/test_utils_python.py

@@ -20,8 +20,6 @@ from scrapy.utils.python import (
     without_none_values,
 )
 
-__doctests__ = ["scrapy.utils.python"]
-
 
 class MutableChainTest(unittest.TestCase):
     def test_mutablechain(self):

tests/test_utils_response.py

@@ -15,8 +15,6 @@ from scrapy.utils.response import (
     response_status_message,
 )
 
-__doctests__ = ["scrapy.utils.response"]
-
 
 class ResponseUtilsTest(unittest.TestCase):
     dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response")
@@ -207,8 +205,8 @@ class ResponseUtilsTest(unittest.TestCase):
 
 
 @pytest.mark.parametrize(
-    "input_body,output_body",
+    ("input_body", "output_body"),
-    (
+    [
         (
             b"a<!--",
             b"a",
@@ -237,7 +235,7 @@ class ResponseUtilsTest(unittest.TestCase):
             b"a<!--b--><!--c-->d",
             b"ad",
         ),
-    ),
+    ],
 )
 def test_remove_html_comments(input_body, output_body):
     assert _remove_html_comments(input_body) == output_body, (

tests/test_utils_signal.py

@@ -1,7 +1,7 @@
 import asyncio
 
+import pytest
 from pydispatch import dispatcher
-from pytest import mark
 from testfixtures import LogCapture
 from twisted.internet import defer, reactor
 from twisted.python.failure import Failure
@@ -67,7 +67,7 @@ class SendCatchLogDeferredTest2(SendCatchLogDeferredTest):
         return d
 
 
-@mark.usefixtures("reactor_pytest")
+@pytest.mark.usefixtures("reactor_pytest")
 class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
     async def ok_handler(self, arg, handlers_called):
         handlers_called.add(self.ok_handler)
@@ -76,7 +76,7 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
         return "OK"
 
 
-@mark.only_asyncio()
+@pytest.mark.only_asyncio
 class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
     async def ok_handler(self, arg, handlers_called):
         handlers_called.add(self.ok_handler)

tests/test_utils_template.py

@@ -5,8 +5,6 @@ from tempfile import mkdtemp
 
 from scrapy.utils.template import render_templatefile
 
-__doctests__ = ["scrapy.utils.template"]
-
 
 class UtilsRenderTemplateFileTestCase(unittest.TestCase):
     def setUp(self):

tests/test_utils_url.py

@@ -17,8 +17,6 @@ from scrapy.utils.url import (  # type: ignore[attr-defined]
     url_is_from_spider,
 )
 
-__doctests__ = ["scrapy.utils.url"]
-
 
 class UrlUtilsTest(unittest.TestCase):
     def test_url_is_from_any_domain(self):