diff --git a/conftest.py b/conftest.py index e9765962a..a08ad9d05 100644 --- a/conftest.py +++ b/conftest.py @@ -41,7 +41,7 @@ if not H2_ENABLED: ) -@pytest.fixture() +@pytest.fixture def chdir(tmpdir): """Change to pytest-provided temporary directory""" tmpdir.chdir() diff --git a/pyproject.toml b/pyproject.toml index 1072730c0..ad62ea212 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -200,8 +200,7 @@ disable = [ [tool.pytest.ini_options] xfail_strict = true usefixtures = "chdir" -python_files = ["test_*.py", "__init__.py"] -python_classes = [] +python_files = ["test_*.py", "test_*/__init__.py"] addopts = [ "--assert=plain", "--ignore=docs/_ext", @@ -254,6 +253,8 @@ extend-select = [ "PIE", # pylint "PL", + # flake8-pytest-style + "PT", # flake8-use-pathlib "PTH", # flake8-pyi @@ -373,6 +374,12 @@ ignore = [ "B904", # Use capitalized environment variable "SIM112", + + # Temporarily silenced PT rules + # Use a regular `assert` instead of unittest-style `assertEqual` + "PT009", + # Use `pytest.raises` instead of unittest-style `assertRaises` + "PT027", ] [tool.ruff.lint.per-file-ignores] diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index a7b84baef..e89786103 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -15,9 +15,10 @@ from unittest import TestCase, mock from twisted.trial.unittest import SkipTest -from scrapy import Spider from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.boto import is_botocore_available +from scrapy.utils.deprecate import create_deprecated_class +from scrapy.utils.spider import DefaultSpider if TYPE_CHECKING: from collections.abc import Awaitable @@ -25,6 +26,7 @@ if TYPE_CHECKING: from twisted.internet.defer import Deferred from twisted.web.client import Response as TxResponse + from scrapy import Spider from scrapy.crawler import Crawler @@ -82,8 +84,7 @@ def get_ftp_content_and_delete( return b"".join(ftp_data) -class TestSpider(Spider): - name = "test" +TestSpider = create_deprecated_class("TestSpider", DefaultSpider) def get_crawler( @@ -101,7 +102,7 @@ def get_crawler( settings: dict[str, Any] = {} settings.update(settings_dict or {}) runner = CrawlerRunner(settings) - crawler = runner.create_crawler(spidercls or TestSpider) + crawler = runner.create_crawler(spidercls or DefaultSpider) crawler._apply_settings() return crawler diff --git a/tests/test_commands.py b/tests/test_commands.py index 872b54d04..1a0db1e03 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -19,7 +19,7 @@ from threading import Timer from typing import TYPE_CHECKING from unittest import mock, skipIf -from pytest import mark +import pytest from twisted.trial import unittest import scrapy @@ -822,7 +822,7 @@ class MySpider(scrapy.Spider): "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log ) - @mark.requires_uvloop + @pytest.mark.requires_uvloop def test_custom_asyncio_loop_enabled_true(self): log = self.get_log( self.debug_log_spider, diff --git a/tests/test_contracts.py b/tests/test_contracts.py index fb16140be..0f7d7b54c 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -21,7 +21,7 @@ from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -class TestItem(Item): +class DemoItem(Item): name = Field() url = Field() @@ -58,7 +58,7 @@ class CustomFormContract(Contract): return args -class TestSpider(Spider): +class DemoSpider(Spider): name = "demo_spider" def returns_request(self, response): @@ -80,7 +80,7 @@ class TestSpider(Spider): @url http://scrapy.org @returns items 1 1 """ - return TestItem(url=response.url) + return DemoItem(url=response.url) def returns_request_cb_kwargs(self, response, url): """method which returns request @@ -96,7 +96,7 @@ class TestSpider(Spider): @cb_kwargs {"name": "Scrapy"} @returns items 1 1 """ - return TestItem(name=name, url=response.url) + return DemoItem(name=name, url=response.url) def returns_item_cb_kwargs_error_unexpected_keyword(self, response): """method which returns item @@ -104,14 +104,14 @@ class TestSpider(Spider): @cb_kwargs {"arg": "value"} @returns items 1 1 """ - return TestItem(url=response.url) + return DemoItem(url=response.url) def returns_item_cb_kwargs_error_missing_argument(self, response, arg): """method which returns item @url http://scrapy.org @returns items 1 1 """ - return TestItem(url=response.url) + return DemoItem(url=response.url) def returns_dict_item(self, response): """method which returns item @@ -125,7 +125,7 @@ class TestSpider(Spider): @url http://scrapy.org @returns items 0 0 """ - return TestItem(url=response.url) + return DemoItem(url=response.url) def returns_dict_fail(self, response): """method which returns item @@ -140,7 +140,7 @@ class TestSpider(Spider): @returns items 1 1 @scrapes name url """ - return TestItem(name="test", url=response.url) + return DemoItem(name="test", url=response.url) def scrapes_dict_item_ok(self, response): """returns item with name and url @@ -156,7 +156,7 @@ class TestSpider(Spider): @returns items 1 1 @scrapes name url """ - return TestItem(url=response.url) + return DemoItem(url=response.url) def scrapes_dict_item_fail(self, response): """returns item with no name @@ -212,7 +212,7 @@ class TestSpider(Spider): @meta {"key": "example"} @returns items 1 1 """ - return TestItem(name="example", url=response.url) + return DemoItem(name="example", url=response.url) def returns_error_missing_meta(self, response): """method which depends of metadata be defined @@ -242,7 +242,7 @@ class CustomContractFailSpider(Spider): """ -class InheritsTestSpider(TestSpider): +class InheritsDemoSpider(DemoSpider): name = "inherits_demo_spider" @@ -274,7 +274,7 @@ class ContractsManagerTest(unittest.TestCase): self.assertTrue(self.results.errors) def test_contracts(self): - spider = TestSpider() + spider = DemoSpider() # extract contracts correctly contracts = self.conman.extract_contracts(spider.returns_request) @@ -293,7 +293,7 @@ class ContractsManagerTest(unittest.TestCase): self.assertEqual(request, None) def test_cb_kwargs(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() # extract contracts correctly @@ -356,7 +356,7 @@ class ContractsManagerTest(unittest.TestCase): self.should_error() def test_meta(self): - spider = TestSpider() + spider = DemoSpider() # extract contracts correctly contracts = self.conman.extract_contracts(spider.returns_request_meta) @@ -402,7 +402,7 @@ class ContractsManagerTest(unittest.TestCase): self.should_error() def test_returns(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() # returns_item @@ -431,7 +431,7 @@ class ContractsManagerTest(unittest.TestCase): self.should_fail() def test_returns_async(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() request = self.conman.from_method(spider.returns_request_async, self.results) @@ -439,7 +439,7 @@ class ContractsManagerTest(unittest.TestCase): self.should_error() def test_scrapes(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() # scrapes_item_ok @@ -472,7 +472,7 @@ class ContractsManagerTest(unittest.TestCase): assert message in self.results.failures[-1][-1] def test_regex(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() # invalid regex @@ -494,7 +494,7 @@ class ContractsManagerTest(unittest.TestCase): self.should_error() def test_errback(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() try: @@ -522,11 +522,11 @@ class ContractsManagerTest(unittest.TestCase): def parse_first(self, response): self.visited += 1 - return TestItem() + return DemoItem() def parse_second(self, response): self.visited += 1 - return TestItem() + return DemoItem() with MockServer() as mockserver: contract_doc = f"@url {mockserver.url('/status?n=200')}" @@ -540,13 +540,13 @@ class ContractsManagerTest(unittest.TestCase): self.assertEqual(crawler.spider.visited, 2) def test_form_contract(self): - spider = TestSpider() + spider = DemoSpider() request = self.conman.from_method(spider.custom_form, self.results) self.assertEqual(request.method, "POST") self.assertIsInstance(request, FormRequest) def test_inherited_contracts(self): - spider = InheritsTestSpider() + spider = InheritsDemoSpider() requests = self.conman.from_spider(spider, self.results) self.assertTrue(requests) @@ -571,7 +571,7 @@ class CustomContractPrePostProcess(unittest.TestCase): self.results = TextTestResult(stream=None, descriptions=False, verbosity=0) def test_pre_hook_keyboard_interrupt(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() contract = CustomFailContractPreProcess(spider.returns_request) conman = ContractsManager([contract]) @@ -590,7 +590,7 @@ class CustomContractPrePostProcess(unittest.TestCase): self.assertFalse(self.results.errors) def test_post_hook_keyboard_interrupt(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() contract = CustomFailContractPostProcess(spider.returns_request) conman = ContractsManager([contract]) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index cd2a559a8..3aca2bbce 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -6,7 +6,7 @@ from ipaddress import IPv4Address from socket import gethostbyname from urllib.parse import urlparse -from pytest import mark +import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.internet.ssl import Certificate @@ -536,7 +536,7 @@ class CrawlSpiderTestCase(TestCase): ) self.assertIn("Got response 200", str(log)) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncio_parse(self): crawler = get_crawler( @@ -551,7 +551,7 @@ class CrawlSpiderTestCase(TestCase): ) self.assertIn("Got response 200", str(log)) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncio_parse_items_list(self): log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider) @@ -559,7 +559,7 @@ class CrawlSpiderTestCase(TestCase): self.assertIn({"id": 1}, items) self.assertIn({"id": 2}, items) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncio_parse_items_single_element(self): items = [] @@ -576,7 +576,7 @@ class CrawlSpiderTestCase(TestCase): self.assertIn("Got response 200", str(log)) self.assertIn({"foo": 42}, items) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncgen_parse(self): log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider) @@ -584,7 +584,7 @@ class CrawlSpiderTestCase(TestCase): itemcount = stats.get_value("item_scraped_count") self.assertEqual(itemcount, 1) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncgen_parse_loop(self): log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider) @@ -594,7 +594,7 @@ class CrawlSpiderTestCase(TestCase): for i in range(10): self.assertIn({"foo": i}, items) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncgen_parse_exc(self): log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider) @@ -606,7 +606,7 @@ class CrawlSpiderTestCase(TestCase): for i in range(7): self.assertIn({"foo": i}, items) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncgen_parse_complex(self): _, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider) @@ -618,20 +618,20 @@ class CrawlSpiderTestCase(TestCase): for i in [10, 30, 122]: self.assertIn({"index2": i}, items) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncio_parse_reqs_list(self): log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider) for req_id in range(3): self.assertIn(f"Got response 200, req_id {req_id}", str(log)) - @mark.only_not_asyncio() + @pytest.mark.only_not_asyncio @defer.inlineCallbacks def test_async_def_deferred_direct(self): _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider) self.assertEqual(items, [{"code": 200}]) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_deferred_wrapped(self): log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider) @@ -659,7 +659,9 @@ class CrawlSpiderTestCase(TestCase): self.assertEqual(cert.getSubject().commonName, b"localhost") self.assertEqual(cert.getIssuer().commonName, b"localhost") - @mark.xfail(reason="Responses with no body return early and contain no certificate") + @pytest.mark.xfail( + reason="Responses with no body return early and contain no certificate" + ) @defer.inlineCallbacks def test_response_ssl_certificate_empty_response(self): crawler = get_crawler(SingleRequestSpider) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 6c3fe96b0..425188d32 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -8,9 +8,9 @@ import warnings from pathlib import Path from typing import Any +import pytest from packaging.version import parse as parse_version from pexpect.popen_spawn import PopenSpawn -from pytest import mark, raises from twisted.internet.defer import Deferred, inlineCallbacks from twisted.trial import unittest from w3lib import __version__ as w3lib_version @@ -77,14 +77,14 @@ class CrawlerTestCase(BaseCrawlerTest): self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED") def test_crawler_rejects_spider_objects(self): - with raises(ValueError): + with pytest.raises(ValueError, match="spidercls argument must be a class"): Crawler(DefaultSpider()) @inlineCallbacks def test_crawler_crawl_twice_unsupported(self): crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) yield crawler.crawl() - with raises(RuntimeError, match="more than once on the same instance"): + with pytest.raises(RuntimeError, match="more than once on the same instance"): yield crawler.crawl() def test_get_addon(self): @@ -203,7 +203,7 @@ class CrawlerTestCase(BaseCrawlerTest): raise crawler = get_raw_crawler(MySpider, BASE_SETTINGS) - with raises(RuntimeError): + with pytest.raises(RuntimeError): yield crawler.crawl() @inlineCallbacks @@ -282,7 +282,7 @@ class CrawlerTestCase(BaseCrawlerTest): raise crawler = get_raw_crawler(MySpider, BASE_SETTINGS) - with raises(RuntimeError): + with pytest.raises(RuntimeError): yield crawler.crawl() @inlineCallbacks @@ -361,7 +361,7 @@ class CrawlerTestCase(BaseCrawlerTest): raise crawler = get_raw_crawler(MySpider, BASE_SETTINGS) - with raises(RuntimeError): + with pytest.raises(RuntimeError): yield crawler.crawl() @inlineCallbacks @@ -440,7 +440,7 @@ class CrawlerTestCase(BaseCrawlerTest): raise crawler = get_raw_crawler(MySpider, BASE_SETTINGS) - with raises(RuntimeError): + with pytest.raises(RuntimeError): yield crawler.crawl() @@ -575,7 +575,7 @@ class NoRequestsSpider(scrapy.Spider): return [] -@mark.usefixtures("reactor_pytest") +@pytest.mark.usefixtures("reactor_pytest") class CrawlerRunnerHasSpider(unittest.TestCase): def _runner(self): return CrawlerRunner() @@ -744,7 +744,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log ) - @mark.skipif( + @pytest.mark.skipif( parse_version(w3lib_version) >= parse_version("2.0.0"), reason="w3lib 2.0.0 and later do not allow invalid domains.", ) @@ -781,7 +781,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "Using reactor: twisted.internet.selectreactor.SelectReactor", log ) - @mark.skipif( + @pytest.mark.skipif( platform.system() == "Windows", reason="PollReactor is not supported on Windows" ) def test_twisted_reactor_poll(self): @@ -820,7 +820,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): log, ) - @mark.requires_uvloop + @pytest.mark.requires_uvloop def test_custom_loop_asyncio(self): log = self.run_script("asyncio_custom_loop.py") self.assertIn("Spider closed (finished)", log) @@ -829,7 +829,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): ) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - @mark.requires_uvloop + @pytest.mark.requires_uvloop def test_custom_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") self.assertIn("Spider closed (finished)", log) @@ -839,7 +839,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) - @mark.requires_uvloop + @pytest.mark.requires_uvloop def test_asyncio_enabled_reactor_same_loop(self): log = self.run_script("asyncio_enabled_reactor_same_loop.py") self.assertIn("Spider closed (finished)", log) @@ -848,7 +848,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): ) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - @mark.requires_uvloop + @pytest.mark.requires_uvloop def test_asyncio_enabled_reactor_different_loop(self): log = self.run_script("asyncio_enabled_reactor_different_loop.py") self.assertNotIn("Spider closed (finished)", log) @@ -924,13 +924,13 @@ class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("DEBUG: Using asyncio event loop", log) -@mark.parametrize( - ["settings", "items"], - ( +@pytest.mark.parametrize( + ("settings", "items"), + [ ({}, default_settings.LOG_VERSIONS), ({"LOG_VERSIONS": ["itemadapter"]}, ["itemadapter"]), ({"LOG_VERSIONS": []}, None), - ), + ], ) def test_log_scrapy_info(settings, items, caplog): with caplog.at_level("INFO"): diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index c581e7596..42051042c 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -1,7 +1,7 @@ import asyncio from unittest import mock -from pytest import mark +import pytest from twisted.internet import defer from twisted.internet.defer import Deferred from twisted.python.failure import Failure @@ -220,7 +220,7 @@ class MiddlewareUsingDeferreds(ManagerTestCase): self.assertFalse(download_func.called) -@mark.usefixtures("reactor_pytest") +@pytest.mark.usefixtures("reactor_pytest") class MiddlewareUsingCoro(ManagerTestCase): """Middlewares using asyncio coroutines should work""" @@ -243,7 +243,7 @@ class MiddlewareUsingCoro(ManagerTestCase): self.assertIs(results[0], resp) self.assertFalse(download_func.called) - @mark.only_asyncio() + @pytest.mark.only_asyncio def test_asyncdef_asyncio(self): resp = Response("http://example.com/index.html") diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py index 63bd158f6..76fcece4f 100644 --- a/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -7,8 +7,6 @@ from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -__doctests__ = ["scrapy.downloadermiddlewares.ajaxcrawl"] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class AjaxCrawlMiddlewareTest(unittest.TestCase): diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 500af6536..581fc1974 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -7,18 +7,18 @@ from scrapy.http import Request from scrapy.spiders import Spider -class TestSpiderLegacy(Spider): +class LegacySpider(Spider): http_user = "foo" http_pass = "bar" -class TestSpider(Spider): +class DomainSpider(Spider): http_user = "foo" http_pass = "bar" http_auth_domain = "example.com" -class TestSpiderAny(Spider): +class AnyDomainSpider(Spider): http_user = "foo" http_pass = "bar" http_auth_domain = None @@ -26,7 +26,7 @@ class TestSpiderAny(Spider): class HttpAuthMiddlewareLegacyTest(unittest.TestCase): def setUp(self): - self.spider = TestSpiderLegacy("foo") + self.spider = LegacySpider("foo") def test_auth(self): with self.assertRaises(AttributeError): @@ -37,7 +37,7 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase): class HttpAuthMiddlewareTest(unittest.TestCase): def setUp(self): self.mw = HttpAuthMiddleware() - self.spider = TestSpider("foo") + self.spider = DomainSpider("foo") self.mw.spider_opened(self.spider) def tearDown(self): @@ -67,7 +67,7 @@ class HttpAuthMiddlewareTest(unittest.TestCase): class HttpAuthAnyMiddlewareTest(unittest.TestCase): def setUp(self): self.mw = HttpAuthMiddleware() - self.spider = TestSpiderAny("foo") + self.spider = AnyDomainSpider("foo") self.mw.spider_opened(self.spider) def tearDown(self): diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index a0886d9e9..74db93f8a 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -353,7 +353,8 @@ class RFC2616PolicyTest(DefaultStorageTest): resc = mw.storage.retrieve_response(self.spider, req0) if shouldcache: self.assertEqualResponse(resc, res1) - assert "cached" in res2.flags and res2.status != 304 + assert "cached" in res2.flags + assert res2.status != 304 else: self.assertFalse(resc) assert "cached" not in res2.flags @@ -376,7 +377,8 @@ class RFC2616PolicyTest(DefaultStorageTest): resc = mw.storage.retrieve_response(self.spider, req0) if shouldcache: self.assertEqualResponse(resc, res1) - assert "cached" in res2.flags and res2.status != 304 + assert "cached" in res2.flags + assert res2.status != 304 else: self.assertFalse(resc) assert "cached" not in res2.flags diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 97c276b48..f0826ef5b 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -131,7 +131,8 @@ class TestHttpProxyMiddleware(TestCase): mw = HttpProxyMiddleware() req = Request("http://noproxy.com", meta={"proxy": None}) assert mw.process_request(req, spider) is None - assert "proxy" in req.meta and req.meta["proxy"] is None + assert "proxy" in req.meta + assert req.meta["proxy"] is None def test_no_proxy(self): os.environ["http_proxy"] = "https://proxy.for.http:3128" diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index 23a1d06da..cace52a27 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -12,7 +12,7 @@ UNSET = object() @pytest.mark.parametrize( ("allowed_domain", "url", "allowed"), - ( + [ ("example.com", "http://example.com/1", True), ("example.com", "http://example.org/1", False), ("example.com", "http://sub.example.com/1", True), @@ -24,7 +24,7 @@ UNSET = object() ("example.com", "http://example.com.example", False), ("a.example", "http://nota.example", False), ("b.a.example", "http://notb.a.example", False), - ), + ], ) def test_process_request_domain_filtering(allowed_domain, url, allowed): crawler = get_crawler(Spider) @@ -41,12 +41,12 @@ def test_process_request_domain_filtering(allowed_domain, url, allowed): @pytest.mark.parametrize( ("value", "filtered"), - ( + [ (UNSET, True), (None, True), (False, True), (True, False), - ), + ], ) def test_process_request_dont_filter(value, filtered): crawler = get_crawler(Spider) @@ -66,7 +66,7 @@ def test_process_request_dont_filter(value, filtered): @pytest.mark.parametrize( ("allow_offsite", "dont_filter", "filtered"), - ( + [ (True, UNSET, False), (True, None, False), (True, False, False), @@ -75,7 +75,7 @@ def test_process_request_dont_filter(value, filtered): (False, None, True), (False, False, True), (False, True, False), - ), + ], ) def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered): crawler = get_crawler(Spider) @@ -97,11 +97,11 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered): @pytest.mark.parametrize( "value", - ( + [ UNSET, None, [], - ), + ], ) def test_process_request_no_allowed_domains(value): crawler = get_crawler(Spider) @@ -133,7 +133,7 @@ def test_process_request_invalid_domains(): @pytest.mark.parametrize( ("allowed_domain", "url", "allowed"), - ( + [ ("example.com", "http://example.com/1", True), ("example.com", "http://example.org/1", False), ("example.com", "http://sub.example.com/1", True), @@ -145,7 +145,7 @@ def test_process_request_invalid_domains(): ("example.com", "http://example.com.example", False), ("a.example", "http://nota.example", False), ("b.a.example", "http://notb.a.example", False), - ), + ], ) def test_request_scheduled_domain_filtering(allowed_domain, url, allowed): crawler = get_crawler(Spider) @@ -162,12 +162,12 @@ def test_request_scheduled_domain_filtering(allowed_domain, url, allowed): @pytest.mark.parametrize( ("value", "filtered"), - ( + [ (UNSET, True), (None, True), (False, True), (True, False), - ), + ], ) def test_request_scheduled_dont_filter(value, filtered): crawler = get_crawler(Spider) @@ -187,11 +187,11 @@ def test_request_scheduled_dont_filter(value, filtered): @pytest.mark.parametrize( "value", - ( + [ UNSET, None, [], - ), + ], ) def test_request_scheduled_no_allowed_domains(value): crawler = get_crawler(Spider) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index eb3cdfc11..f950906e9 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1278,7 +1278,7 @@ class MetaRefreshMiddlewareTest(Base.Test): @pytest.mark.parametrize( SCHEME_PARAMS, - ( + [ *REDIRECT_SCHEME_CASES, # data/file/ftp/s3/foo → * does not redirect *( @@ -1300,7 +1300,7 @@ class MetaRefreshMiddlewareTest(Base.Test): for scheme in NON_HTTP_SCHEMES for location in ("//example.com/b", "/b") ), - ), + ], ) def test_meta_refresh_schemes(url, location, target): crawler = get_crawler(Spider) diff --git a/tests/test_engine.py b/tests/test_engine.py index 91ce2c0de..e9470493f 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -42,7 +42,7 @@ from scrapy.utils.test import get_crawler from tests import get_testdata, tests_datadir -class TestItem(Item): +class MyItem(Item): name = Field() url = Field() price = Field() @@ -62,7 +62,7 @@ class DataClassItem: price: int = 0 -class TestSpider(Spider): +class MySpider(Spider): name = "scrapytest.org" allowed_domains = ["scrapytest.org", "localhost"] @@ -70,7 +70,7 @@ class TestSpider(Spider): name_re = re.compile(r"

(.*?)

", re.MULTILINE) price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE) - item_cls: type = TestItem + item_cls: type = MyItem def parse(self, response): xlink = LinkExtractor() @@ -91,24 +91,24 @@ class TestSpider(Spider): return adapter.item -class TestDupeFilterSpider(TestSpider): +class DupeFilterSpider(MySpider): def start_requests(self): return (Request(url) for url in self.start_urls) # no dont_filter=True -class DictItemsSpider(TestSpider): +class DictItemsSpider(MySpider): item_cls = dict -class AttrsItemsSpider(TestSpider): +class AttrsItemsSpider(MySpider): item_cls = AttrsItem -class DataClassItemsSpider(TestSpider): +class DataClassItemsSpider(MySpider): item_cls = DataClassItem -class ItemZeroDivisionErrorSpider(TestSpider): +class ItemZeroDivisionErrorSpider(MySpider): custom_settings = { "ITEM_PIPELINES": { "tests.pipelines.ProcessWithZeroDivisionErrorPipeline": 300, @@ -116,7 +116,7 @@ class ItemZeroDivisionErrorSpider(TestSpider): } -class ChangeCloseReasonSpider(TestSpider): +class ChangeCloseReasonSpider(MySpider): @classmethod def from_crawler(cls, crawler, *args, **kwargs): spider = cls(*args, **kwargs) @@ -388,7 +388,7 @@ class EngineTest(EngineTestBase): @defer.inlineCallbacks def test_crawler(self): for spider in ( - TestSpider, + MySpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider, @@ -404,7 +404,7 @@ class EngineTest(EngineTestBase): @defer.inlineCallbacks def test_crawler_dupefilter(self): - run = CrawlerRun(TestDupeFilterSpider) + run = CrawlerRun(DupeFilterSpider) yield run.run() self._assert_scheduled_requests(run, count=8) self._assert_dropped_requests(run) @@ -426,13 +426,13 @@ class EngineTest(EngineTestBase): @defer.inlineCallbacks def test_close_downloader(self): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + e = ExecutionEngine(get_crawler(MySpider), lambda _: None) yield e.close() @defer.inlineCallbacks def test_start_already_running_exception(self): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - yield e.open_spider(TestSpider(), []) + e = ExecutionEngine(get_crawler(MySpider), lambda _: None) + yield e.open_spider(MySpider(), []) e.start() try: yield self.assertFailure(e.start(), RuntimeError).addBoth( @@ -486,7 +486,7 @@ def test_request_scheduled_signal(caplog): if "drop" in request.url: raise IgnoreRequest - spider = TestSpider() + spider = MySpider() crawler = get_crawler(spider.__class__) engine = ExecutionEngine(crawler, lambda _: None) engine.downloader._slot_gc_loop.stop() diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index 8bf225ab1..5dd04c310 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -8,7 +8,7 @@ from tests.test_engine import ( DataClassItemsSpider, DictItemsSpider, EngineTestBase, - TestSpider, + MySpider, ) @@ -22,7 +22,7 @@ class BytesReceivedEngineTest(EngineTestBase): @defer.inlineCallbacks def test_crawler(self): for spider in ( - TestSpider, + MySpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider, diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index 4efb6b7a8..06929d1e4 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -8,7 +8,7 @@ from tests.test_engine import ( DataClassItemsSpider, DictItemsSpider, EngineTestBase, - TestSpider, + MySpider, ) @@ -22,7 +22,7 @@ class HeadersReceivedEngineTest(EngineTestBase): @defer.inlineCallbacks def test_crawler(self): for spider in ( - TestSpider, + MySpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider, diff --git a/tests/test_exporters.py b/tests/test_exporters.py index c2cab9b2a..eb8d309b6 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -31,7 +31,7 @@ def custom_serializer(value): return str(int(value) + 2) -class TestItem(Item): +class MyItem(Item): name = Field() age = Field() @@ -42,7 +42,7 @@ class CustomFieldItem(Item): @dataclasses.dataclass -class TestDataClass: +class MyDataClass: name: str age: int @@ -54,7 +54,7 @@ class CustomFieldDataclass: class BaseItemExporterTest(unittest.TestCase): - item_class: type = TestItem + item_class: type = MyItem custom_field_item_class: type = CustomFieldItem def setUp(self): @@ -138,7 +138,7 @@ class BaseItemExporterTest(unittest.TestCase): class BaseItemExporterDataclassTest(BaseItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -207,7 +207,7 @@ class PythonItemExporterTest(BaseItemExporterTest): class PythonItemExporterDataclassTest(PythonItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -222,7 +222,7 @@ class PprintItemExporterTest(BaseItemExporterTest): class PprintItemExporterDataclassTest(PprintItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -259,7 +259,7 @@ class PickleItemExporterTest(BaseItemExporterTest): class PickleItemExporterDataclassTest(PickleItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -286,7 +286,7 @@ class MarshalItemExporterTest(BaseItemExporterTest): class MarshalItemExporterDataclassTest(MarshalItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -406,7 +406,7 @@ class CsvItemExporterTest(BaseItemExporterTest): class CsvItemExporterDataclassTest(CsvItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -517,7 +517,7 @@ class XmlItemExporterTest(BaseItemExporterTest): class XmlItemExporterDataclassTest(XmlItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -563,7 +563,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -595,11 +595,11 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.assertTwoItemsExported(ItemAdapter(self.i).asdict()) def test_two_items_with_failure_between(self): - i1 = TestItem(name="Joseph\xa3", age="22") - i2 = TestItem( + i1 = MyItem(name="Joseph\xa3", age="22") + i2 = MyItem( name="Maria", age=1j ) # Invalid datetimes didn't consistently fail between Python versions - i3 = TestItem(name="Jesus", age="44") + i3 = MyItem(name="Jesus", age="44") self.ie.start_exporting() self.ie.export_item(i1) self.assertRaises(TypeError, self.ie.export_item, i2) @@ -652,9 +652,9 @@ class JsonItemExporterToBytesTest(BaseItemExporterTest): return JsonItemExporter(self.output, **kwargs) def test_two_items_with_failure_between(self): - i1 = TestItem(name="Joseph", age="22") - i2 = TestItem(name="\u263a", age="11") - i3 = TestItem(name="Jesus", age="44") + i1 = MyItem(name="Joseph", age="22") + i2 = MyItem(name="\u263a", age="11") + i3 = MyItem(name="Jesus", age="44") self.ie.start_exporting() self.ie.export_item(i1) self.assertRaises(UnicodeEncodeError, self.ie.export_item, i2) @@ -665,12 +665,12 @@ class JsonItemExporterToBytesTest(BaseItemExporterTest): class JsonItemExporterDataclassTest(JsonItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass class CustomExporterItemTest(unittest.TestCase): - item_class: type = TestItem + item_class: type = MyItem def setUp(self): if self.item_class is None: @@ -700,4 +700,4 @@ class CustomExporterItemTest(unittest.TestCase): class CustomExporterDataclassTest(CustomExporterItemTest): - item_class = TestDataClass + item_class = MyDataClass diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 15129e31f..ca5ffdc26 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -51,7 +51,7 @@ stats_dump_2 = { } -class TestExtPeriodicLog(PeriodicLog): +class CustomPeriodicLog(PeriodicLog): def set_a(self): self.stats._stats = stats_dump_1 @@ -62,7 +62,7 @@ class TestExtPeriodicLog(PeriodicLog): def extension(settings=None): crawler = Crawler(MetaSpider, settings=settings) crawler._apply_settings() - return TestExtPeriodicLog.from_crawler(crawler) + return CustomPeriodicLog.from_crawler(crawler) class TestPeriodicLog(unittest.TestCase): diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py index f2c9dc063..4874f284a 100644 --- a/tests/test_extension_throttle.py +++ b/tests/test_extension_throttle.py @@ -13,15 +13,12 @@ from scrapy.settings.default_settings import ( DOWNLOAD_DELAY, ) from scrapy.utils.misc import build_from_crawler +from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler as _get_crawler UNSET = object() -class TestSpider(Spider): - name = "test" - - def get_crawler(settings=None, spidercls=None): settings = settings or {} settings["AUTOTHROTTLE_ENABLED"] = True @@ -30,11 +27,11 @@ def get_crawler(settings=None, spidercls=None): @pytest.mark.parametrize( ("value", "expected"), - ( + [ (UNSET, False), (False, False), (True, True), - ), + ], ) def test_enabled(value, expected): settings = {} @@ -50,10 +47,10 @@ def test_enabled(value, expected): @pytest.mark.parametrize( "value", - ( + [ 0.0, -1.0, - ), + ], ) def test_target_concurrency_invalid(value): settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": value} @@ -64,13 +61,13 @@ def test_target_concurrency_invalid(value): @pytest.mark.parametrize( ("spider", "setting", "expected"), - ( + [ (UNSET, UNSET, DOWNLOAD_DELAY), (1.0, UNSET, 1.0), (UNSET, 1.0, 1.0), (1.0, 2.0, 1.0), (3.0, 2.0, 3.0), - ), + ], ) def test_mindelay_definition(spider, setting, expected): settings = {} @@ -91,10 +88,10 @@ def test_mindelay_definition(spider, setting, expected): @pytest.mark.parametrize( ("value", "expected"), - ( + [ (UNSET, AUTOTHROTTLE_MAX_DELAY), (1.0, 1.0), - ), + ], ) def test_maxdelay_definition(value, expected): settings = {} @@ -102,13 +99,13 @@ def test_maxdelay_definition(value, expected): settings["AUTOTHROTTLE_MAX_DELAY"] = value crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) - at._spider_opened(TestSpider()) + at._spider_opened(DefaultSpider()) assert at.maxdelay == expected @pytest.mark.parametrize( ("min_spider", "min_setting", "start_setting", "expected"), - ( + [ (UNSET, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), (AUTOTHROTTLE_START_DELAY - 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), (AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), @@ -134,7 +131,7 @@ def test_maxdelay_definition(value, expected): AUTOTHROTTLE_START_DELAY + 2.0, AUTOTHROTTLE_START_DELAY + 2.0, ), - ), + ], ) def test_startdelay_definition(min_spider, min_setting, start_setting, expected): settings = {} @@ -158,7 +155,7 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) @pytest.mark.parametrize( ("meta", "slot"), - ( + [ ({}, None), ({"download_latency": 1.0}, None), ({"download_slot": "foo"}, None), @@ -172,12 +169,12 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) }, "foo", ), - ), + ], ) def test_skipped(meta, slot): crawler = get_crawler() at = build_from_crawler(AutoThrottle, crawler) - spider = TestSpider() + spider = DefaultSpider() at._spider_opened(spider) request = Request("https://example.com", meta=meta) @@ -193,7 +190,7 @@ def test_skipped(meta, slot): @pytest.mark.parametrize( ("download_latency", "target_concurrency", "slot_delay", "expected"), - ( + [ (2.0, 2.0, 1.0, 1.0), (1.0, 2.0, 1.0, 0.75), (4.0, 2.0, 1.0, 2.0), @@ -201,13 +198,13 @@ def test_skipped(meta, slot): (2.0, 4.0, 1.0, 0.75), (2.0, 2.0, 0.5, 1.0), (2.0, 2.0, 2.0, 1.5), - ), + ], ) def test_adjustment(download_latency, target_concurrency, slot_delay, expected): settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) - spider = TestSpider() + spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": download_latency, "download_slot": "foo"} request = Request("https://example.com", meta=meta) @@ -227,11 +224,11 @@ def test_adjustment(download_latency, target_concurrency, slot_delay, expected): @pytest.mark.parametrize( ("mindelay", "maxdelay", "expected"), - ( + [ (0.5, 2.0, 1.0), (0.25, 0.5, 0.5), (2.0, 4.0, 2.0), - ), + ], ) def test_adjustment_limits(mindelay, maxdelay, expected): download_latency, target_concurrency, slot_delay = (2.0, 2.0, 1.0) @@ -243,7 +240,7 @@ def test_adjustment_limits(mindelay, maxdelay, expected): } crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) - spider = TestSpider() + spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": download_latency, "download_slot": "foo"} request = Request("https://example.com", meta=meta) @@ -263,11 +260,11 @@ def test_adjustment_limits(mindelay, maxdelay, expected): @pytest.mark.parametrize( ("download_latency", "target_concurrency", "slot_delay", "expected"), - ( + [ (2.0, 2.0, 1.0, 1.0), (1.0, 2.0, 1.0, 1.0), # Instead of 0.75 (4.0, 2.0, 1.0, 2.0), - ), + ], ) def test_adjustment_bad_response( download_latency, target_concurrency, slot_delay, expected @@ -275,7 +272,7 @@ def test_adjustment_bad_response( settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) - spider = TestSpider() + spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": download_latency, "download_slot": "foo"} request = Request("https://example.com", meta=meta) @@ -297,7 +294,7 @@ def test_debug(caplog): settings = {"AUTOTHROTTLE_DEBUG": True} crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) - spider = TestSpider() + spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": 1.0, "download_slot": "foo"} request = Request("https://example.com", meta=meta) @@ -327,7 +324,7 @@ def test_debug(caplog): def test_debug_disabled(caplog): crawler = get_crawler() at = build_from_crawler(AutoThrottle, crawler) - spider = TestSpider() + spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": 1.0, "download_slot": "foo"} request = Request("https://example.com", meta=meta) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 1f998de1a..ddc772236 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -258,7 +258,8 @@ class Https2ClientProtocolTestCase(TestCase): :param path: Should have / at the starting compulsorily if not empty :return: Complete url """ - assert len(path) > 0 and (path[0] == "/" or path[0] == "&") + assert len(path) > 0 + assert path[0] == "/" or path[0] == "&" return f"{self.scheme}://{self.hostname}:{self.port_number}{path}" def make_request(self, request: Request) -> Deferred: diff --git a/tests/test_http_response.py b/tests/test_http_response.py index b157e9802..dde883451 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -2,8 +2,8 @@ import codecs import unittest from unittest import mock +import pytest from packaging.version import Version as parse_version -from pytest import mark from w3lib import __version__ as w3lib_version from w3lib.encoding import resolve_encoding @@ -218,7 +218,7 @@ class BaseResponseTest(unittest.TestCase): r = self.response_class("http://example.com") self.assertRaises(ValueError, r.follow, None) - @mark.xfail( + @pytest.mark.xfail( parse_version(w3lib_version) < parse_version("2.1.1"), reason="https://github.com/scrapy/w3lib/pull/207", strict=True, @@ -226,7 +226,7 @@ class BaseResponseTest(unittest.TestCase): def test_follow_whitespace_url(self): self._assert_followed_url("foo ", "http://example.com/foo") - @mark.xfail( + @pytest.mark.xfail( parse_version(w3lib_version) < parse_version("2.1.1"), reason="https://github.com/scrapy/w3lib/pull/207", strict=True, @@ -473,10 +473,8 @@ class TextResponseTest(BaseResponseTest): self._assert_response_encoding(r5, "utf-8") self._assert_response_encoding(r8, "utf-8") self._assert_response_encoding(r9, "cp1252") - assert ( - r4._body_inferred_encoding() is not None - and r4._body_inferred_encoding() != "ascii" - ) + assert r4._body_inferred_encoding() is not None + assert r4._body_inferred_encoding() != "ascii" self._assert_response_values(r1, "utf-8", "\xa3") self._assert_response_values(r2, "utf-8", "\xa3") self._assert_response_values(r3, "iso-8859-1", "\xa3") diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index a83cfb56c..e751e0a63 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -4,8 +4,8 @@ import pickle import re import unittest +import pytest from packaging.version import Version -from pytest import mark from w3lib import __version__ as w3lib_version from scrapy.http import HtmlResponse, XmlResponse @@ -930,7 +930,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): ], ) - @mark.skipif( + @pytest.mark.skipif( Version(w3lib_version) < Version("2.0.0"), reason=( "Before w3lib 2.0.0, w3lib.url.safe_url_string would not complain " diff --git a/tests/test_loader.py b/tests/test_loader.py index 824d7aecf..b52d5ea2e 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -18,12 +18,12 @@ class NameItem(Item): name = Field() -class TestItem(NameItem): +class SummaryItem(NameItem): url = Field() summary = Field() -class TestNestedItem(Item): +class NestedItem(Item): name = Field() name_div = Field() name_value = Field() @@ -38,20 +38,20 @@ class AttrsNameItem: @dataclasses.dataclass -class TestDataClass: +class NameDataClass: name: list = dataclasses.field(default_factory=list) # test item loaders class NameItemLoader(ItemLoader): - default_item_class = TestItem + default_item_class = SummaryItem class NestedItemLoader(ItemLoader): - default_item_class = TestNestedItem + default_item_class = NestedItem -class TestItemLoader(NameItemLoader): +class ProcessorItemLoader(NameItemLoader): name_in = MapCompose(lambda v: v.title()) @@ -68,11 +68,11 @@ def processor_with_args(value, other=None, loader_context=None): class BasicItemLoaderTest(unittest.TestCase): def test_add_value_on_unknown_field(self): - il = TestItemLoader() + il = ProcessorItemLoader() self.assertRaises(KeyError, il.add_value, "wrong_field", ["lala", "lolo"]) def test_load_item_using_default_loader(self): - i = TestItem() + i = SummaryItem() i["summary"] = "lala" il = ItemLoader(item=i) il.add_value("name", "marta") @@ -82,7 +82,7 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(item["name"], ["marta"]) def test_load_item_using_custom_loader(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.add_value("name", "marta") item = il.load_item() self.assertEqual(item["name"], ["Marta"]) @@ -194,7 +194,7 @@ class InitializationFromAttrsItemTest(InitializationTestMixin, unittest.TestCase class InitializationFromDataClassTest(InitializationTestMixin, unittest.TestCase): - item_class = TestDataClass + item_class = NameDataClass class BaseNoInputReprocessingLoader(ItemLoader): @@ -289,11 +289,11 @@ class SelectortemLoaderTest(unittest.TestCase): ) def test_init_method(self): - l = TestItemLoader() + l = ProcessorItemLoader() self.assertEqual(l.selector, None) def test_init_method_errors(self): - l = TestItemLoader() + l = ProcessorItemLoader() self.assertRaises(RuntimeError, l.add_xpath, "url", "//a/@href") self.assertRaises(RuntimeError, l.replace_xpath, "url", "//a/@href") self.assertRaises(RuntimeError, l.get_xpath, "//a/@href") @@ -303,7 +303,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_init_method_with_selector(self): sel = Selector(text="
marta
") - l = TestItemLoader(selector=sel) + l = ProcessorItemLoader(selector=sel) self.assertIs(l.selector, sel) l.add_xpath("name", "//div/text()") @@ -311,7 +311,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_init_method_with_selector_css(self): sel = Selector(text="
marta
") - l = TestItemLoader(selector=sel) + l = ProcessorItemLoader(selector=sel) self.assertIs(l.selector, sel) l.add_css("name", "div::text") @@ -320,18 +320,18 @@ class SelectortemLoaderTest(unittest.TestCase): def test_init_method_with_base_response(self): """Selector should be None after initialization""" response = Response("https://scrapy.org") - l = TestItemLoader(response=response) + l = ProcessorItemLoader(response=response) self.assertIs(l.selector, None) def test_init_method_with_response(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath("name", "//div/text()") self.assertEqual(l.get_output_value("name"), ["Marta"]) def test_init_method_with_response_css(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertTrue(l.selector) l.add_css("name", "div::text") @@ -350,12 +350,12 @@ class SelectortemLoaderTest(unittest.TestCase): ) def test_add_xpath_re(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) l.add_xpath("name", "//div/text()", re="ma") self.assertEqual(l.get_output_value("name"), ["Ma"]) def test_replace_xpath(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath("name", "//div/text()") self.assertEqual(l.get_output_value("name"), ["Marta"]) @@ -366,7 +366,7 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertEqual(l.get_output_value("name"), ["Paragraph", "Marta"]) def test_get_xpath(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertEqual(l.get_xpath("//p/text()"), ["paragraph"]) self.assertEqual(l.get_xpath("//p/text()", TakeFirst()), "paragraph") self.assertEqual(l.get_xpath("//p/text()", TakeFirst(), re="pa"), "pa") @@ -376,14 +376,14 @@ class SelectortemLoaderTest(unittest.TestCase): ) def test_replace_xpath_multi_fields(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) l.add_xpath(None, "//div/text()", TakeFirst(), lambda x: {"name": x}) self.assertEqual(l.get_output_value("name"), ["Marta"]) l.replace_xpath(None, "//p/text()", TakeFirst(), lambda x: {"name": x}) self.assertEqual(l.get_output_value("name"), ["Paragraph"]) def test_replace_xpath_re(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath("name", "//div/text()") self.assertEqual(l.get_output_value("name"), ["Marta"]) @@ -391,7 +391,7 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertEqual(l.get_output_value("name"), ["Ma"]) def test_add_css_re(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) l.add_css("name", "div::text", re="ma") self.assertEqual(l.get_output_value("name"), ["Ma"]) @@ -399,7 +399,7 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertEqual(l.get_output_value("url"), ["www.scrapy.org"]) def test_replace_css(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertTrue(l.selector) l.add_css("name", "div::text") self.assertEqual(l.get_output_value("name"), ["Marta"]) @@ -415,7 +415,7 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertEqual(l.get_output_value("url"), ["/images/logo.png"]) def test_get_css(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertEqual(l.get_css("p::text"), ["paragraph"]) self.assertEqual(l.get_css("p::text", TakeFirst()), "paragraph") self.assertEqual(l.get_css("p::text", TakeFirst(), re="pa"), "pa") @@ -427,7 +427,7 @@ class SelectortemLoaderTest(unittest.TestCase): ) def test_replace_css_multi_fields(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) l.add_css(None, "div::text", TakeFirst(), lambda x: {"name": x}) self.assertEqual(l.get_output_value("name"), ["Marta"]) l.replace_css(None, "p::text", TakeFirst(), lambda x: {"name": x}) @@ -439,7 +439,7 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertEqual(l.get_output_value("url"), ["/images/logo.png"]) def test_replace_css_re(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertTrue(l.selector) l.add_css("url", "a::attr(href)") self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 8d4bd6bc1..1e504f539 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -24,17 +24,17 @@ class NameItem(Item): name = Field() -class TestItem(NameItem): +class SummaryItem(NameItem): url = Field() summary = Field() # test item loaders class NameItemLoader(ItemLoader): - default_item_class = TestItem + default_item_class = SummaryItem -class TestItemLoader(NameItemLoader): +class ProcessorItemLoader(NameItemLoader): name_in = MapCompose(lambda v: v.title()) @@ -51,7 +51,7 @@ def processor_with_args(value, other=None, loader_context=None): class BasicItemLoaderTest(unittest.TestCase): def test_load_item_using_default_loader(self): - i = TestItem() + i = SummaryItem() i["summary"] = "lala" il = ItemLoader(item=i) il.add_value("name", "marta") @@ -61,7 +61,7 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(item["name"], ["marta"]) def test_load_item_using_custom_loader(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.add_value("name", "marta") item = il.load_item() self.assertEqual(item["name"], ["Marta"]) @@ -125,7 +125,7 @@ class BasicItemLoaderTest(unittest.TestCase): ) def test_add_value(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.add_value("name", "marta") self.assertEqual(il.get_collected_values("name"), ["Marta"]) self.assertEqual(il.get_output_value("name"), ["Marta"]) @@ -146,7 +146,7 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_collected_values("name"), [0]) def test_replace_value(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.replace_value("name", "marta") self.assertEqual(il.get_collected_values("name"), ["Marta"]) self.assertEqual(il.get_output_value("name"), ["Marta"]) @@ -229,7 +229,7 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_output_value("name"), ["mart"]) def test_input_processor_inheritance(self): - class ChildItemLoader(TestItemLoader): + class ChildItemLoader(ProcessorItemLoader): url_in = MapCompose(lambda v: v.lower()) il = ChildItemLoader() @@ -265,8 +265,8 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_output_value("name"), ["marta"]) def test_extend_custom_input_processors(self): - class ChildItemLoader(TestItemLoader): - name_in = MapCompose(TestItemLoader.name_in, str.swapcase) + class ChildItemLoader(ProcessorItemLoader): + name_in = MapCompose(ProcessorItemLoader.name_in, str.swapcase) il = ChildItemLoader() il.add_value("name", "marta") @@ -283,11 +283,11 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_output_value("name"), ["MART"]) def test_output_processor_using_function(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.add_value("name", ["mar", "ta"]) self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - class TakeFirstItemLoader(TestItemLoader): + class TakeFirstItemLoader(ProcessorItemLoader): name_out = " ".join il = TakeFirstItemLoader() @@ -296,7 +296,7 @@ class BasicItemLoaderTest(unittest.TestCase): def test_output_processor_error(self): class TestItemLoader(ItemLoader): - default_item_class = TestItem + default_item_class = SummaryItem name_out = MapCompose(float) il = TestItemLoader() @@ -319,18 +319,18 @@ class BasicItemLoaderTest(unittest.TestCase): assert expected_exc_str in s, s def test_output_processor_using_classes(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.add_value("name", ["mar", "ta"]) self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - class TakeFirstItemLoader(TestItemLoader): + class TakeFirstItemLoader(ProcessorItemLoader): name_out = Join() il = TakeFirstItemLoader() il.add_value("name", ["mar", "ta"]) self.assertEqual(il.get_output_value("name"), "Mar Ta") - class TakeFirstItemLoader2(TestItemLoader): + class TakeFirstItemLoader2(ProcessorItemLoader): name_out = Join("
") il = TakeFirstItemLoader2() @@ -338,11 +338,11 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_output_value("name"), "Mar
Ta") def test_default_output_processor(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.add_value("name", ["mar", "ta"]) self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - class LalaItemLoader(TestItemLoader): + class LalaItemLoader(ProcessorItemLoader): default_output_processor = Identity() il = LalaItemLoader() @@ -350,7 +350,7 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) def test_loader_context_on_declaration(self): - class ChildItemLoader(TestItemLoader): + class ChildItemLoader(ProcessorItemLoader): url_in = MapCompose(processor_with_args, key="val") il = ChildItemLoader() @@ -360,7 +360,7 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_output_value("url"), ["val"]) def test_loader_context_on_instantiation(self): - class ChildItemLoader(TestItemLoader): + class ChildItemLoader(ProcessorItemLoader): url_in = MapCompose(processor_with_args) il = ChildItemLoader(key="val") @@ -370,7 +370,7 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_output_value("url"), ["val"]) def test_loader_context_on_assign(self): - class ChildItemLoader(TestItemLoader): + class ChildItemLoader(ProcessorItemLoader): url_in = MapCompose(processor_with_args) il = ChildItemLoader() @@ -384,10 +384,10 @@ class BasicItemLoaderTest(unittest.TestCase): def processor(value, loader_context): return loader_context["item"]["name"] - class ChildItemLoader(TestItemLoader): + class ChildItemLoader(ProcessorItemLoader): url_in = MapCompose(processor) - it = TestItem(name="marta") + it = SummaryItem(name="marta") il = ChildItemLoader(item=it) il.add_value("url", "text") self.assertEqual(il.get_output_value("url"), ["marta"]) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 3a1cf19ad..0cc532570 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -40,7 +40,7 @@ class MOff: raise NotConfigured("foo") -class TestMiddlewareManager(MiddlewareManager): +class MyMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): return [M1, MOff, M3] @@ -54,7 +54,7 @@ class TestMiddlewareManager(MiddlewareManager): class MiddlewareManagerTest(unittest.TestCase): def test_init(self): m1, m2, m3 = M1(), M2(), M3() - mwman = TestMiddlewareManager(m1, m2, m3) + mwman = MyMiddlewareManager(m1, m2, m3) self.assertEqual( list(mwman.methods["open_spider"]), [m1.open_spider, m2.open_spider] ) @@ -64,7 +64,7 @@ class MiddlewareManagerTest(unittest.TestCase): self.assertEqual(list(mwman.methods["process"]), [m1.process, m3.process]) def test_methods(self): - mwman = TestMiddlewareManager(M1(), M2(), M3()) + mwman = MyMiddlewareManager(M1(), M2(), M3()) self.assertEqual( [x.__self__.__class__ for x in mwman.methods["open_spider"]], [M1, M2] ) @@ -82,6 +82,6 @@ class MiddlewareManagerTest(unittest.TestCase): def test_enabled_from_settings(self): crawler = get_crawler() - mwman = TestMiddlewareManager.from_crawler(crawler) + mwman = MyMiddlewareManager.from_crawler(crawler) classes = [x.__class__ for x in mwman.middlewares] self.assertEqual(classes, [M1, M3]) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 222b19e7f..0ae86235c 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -1,6 +1,6 @@ import asyncio -from pytest import mark +import pytest from twisted.internet import defer from twisted.internet.defer import Deferred from twisted.trial import unittest @@ -118,14 +118,14 @@ class PipelineTestCase(unittest.TestCase): yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(len(self.items), 1) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_asyncdef_asyncio_pipeline(self): crawler = self._create_crawler(AsyncDefAsyncioPipeline) yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(len(self.items), 1) - @mark.only_not_asyncio() + @pytest.mark.only_not_asyncio @defer.inlineCallbacks def test_asyncdef_not_asyncio_pipeline(self): crawler = self._create_crawler(AsyncDefNotAsyncioPipeline) diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index 854805cf7..85133038a 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -11,7 +11,7 @@ class CustomRequest(Request): class RequestSerializationTest(unittest.TestCase): def setUp(self): - self.spider = TestSpider() + self.spider = MethodsSpider() def test_basic(self): r = Request("http://www.example.com") @@ -96,18 +96,22 @@ class RequestSerializationTest(unittest.TestCase): def test_private_reference_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._TestSpider__parse_item_reference, - errback=self.spider._TestSpider__handle_error_reference, + callback=self.spider._MethodsSpider__parse_item_reference, + errback=self.spider._MethodsSpider__handle_error_reference, ) self._assert_serializes_ok(r, spider=self.spider) request_dict = r.to_dict(spider=self.spider) - self.assertEqual(request_dict["callback"], "_TestSpider__parse_item_reference") - self.assertEqual(request_dict["errback"], "_TestSpider__handle_error_reference") + self.assertEqual( + request_dict["callback"], "_MethodsSpider__parse_item_reference" + ) + self.assertEqual( + request_dict["errback"], "_MethodsSpider__handle_error_reference" + ) def test_private_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._TestSpider__parse_item_private, + callback=self.spider._MethodsSpider__parse_item_private, errback=self.spider.handle_error, ) self._assert_serializes_ok(r, spider=self.spider) @@ -115,7 +119,7 @@ class RequestSerializationTest(unittest.TestCase): def test_mixin_private_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._TestSpiderMixin__mixin_callback, + callback=self.spider._SpiderMixin__mixin_callback, errback=self.spider.handle_error, ) self._assert_serializes_ok(r, spider=self.spider) @@ -152,18 +156,18 @@ class RequestSerializationTest(unittest.TestCase): def test_callback_not_available(self): """Callback method is not available in the spider passed to from_dict""" - spider = TestSpiderDelegation() + spider = SpiderDelegation() r = Request("http://www.example.com", callback=spider.delegated_callback) d = r.to_dict(spider=spider) self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo")) -class TestSpiderMixin: +class SpiderMixin: def __mixin_callback(self, response): # pylint: disable=unused-private-member pass -class TestSpiderDelegation: +class SpiderDelegation: def delegated_callback(self, response): pass @@ -184,7 +188,7 @@ def private_handle_error(failure): pass -class TestSpider(Spider, TestSpiderMixin): +class MethodsSpider(Spider, SpiderMixin): name = "test" parse_item_reference = parse_item handle_error_reference = handle_error @@ -193,7 +197,7 @@ class TestSpider(Spider, TestSpiderMixin): def __init__(self, **kwargs): super().__init__(**kwargs) - self.delegated_callback = TestSpiderDelegation().delegated_callback + self.delegated_callback = SpiderDelegation().delegated_callback def parse_item(self, response): pass diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index b48a65e67..7c72805e2 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -51,8 +51,8 @@ class SimpleScheduler(MinimalScheduler): return len(self.requests) -class TestSpider(Spider): - name = "test" +class PathsSpider(Spider): + name = "paths" def __init__(self, mockserver, *args, **kwargs): super().__init__(*args, **kwargs) @@ -155,7 +155,7 @@ class MinimalSchedulerCrawlTest(TwistedTestCase): "SCHEDULER": self.scheduler_cls, } with LogCapture() as log: - crawler = get_crawler(TestSpider, settings) + crawler = get_crawler(PathsSpider, settings) yield crawler.crawl(mockserver) for path in PATHS: self.assertIn(f"{{'path': '{path}'}}", str(log)) diff --git a/tests/test_signals.py b/tests/test_signals.py index 1e693c094..a508eb41a 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -1,4 +1,4 @@ -from pytest import mark +import pytest from twisted.internet import defer from twisted.trial import unittest @@ -37,7 +37,7 @@ class AsyncSignalTestCase(unittest.TestCase): item = await get_from_asyncio_queue(item) self.items.append(item) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_simple_pipeline(self): crawler = get_crawler(ItemSpider) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 4ce7cc9a4..a2e7ae65d 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -15,7 +15,7 @@ from scrapy.squeues import ( ) -class TestItem(Item): +class MyItem(Item): name = Field() @@ -23,8 +23,8 @@ def _test_procesor(x): return x + x -class TestLoader(ItemLoader): - default_item_class = TestItem +class MyLoader(ItemLoader): + default_item_class = MyItem name_out = staticmethod(_test_procesor) @@ -80,19 +80,19 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): def test_serialize_item(self): q = self.queue() - i = TestItem(name="foo") + i = MyItem(name="foo") q.push(i) i2 = q.pop() - assert isinstance(i2, TestItem) + assert isinstance(i2, MyItem) self.assertEqual(i, i2) def test_serialize_loader(self): q = self.queue() - loader = TestLoader() + loader = MyLoader() q.push(loader) loader2 = q.pop() - assert isinstance(loader2, TestLoader) - assert loader2.default_item_class is TestItem + assert isinstance(loader2, MyLoader) + assert loader2.default_item_class is MyItem self.assertEqual(loader2.name_out("x"), "xx") def test_serialize_request_recursive(self): @@ -161,19 +161,19 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): def test_serialize_item(self): q = self.queue() - i = TestItem(name="foo") + i = MyItem(name="foo") q.push(i) i2 = q.pop() - assert isinstance(i2, TestItem) + assert isinstance(i2, MyItem) self.assertEqual(i, i2) def test_serialize_loader(self): q = self.queue() - loader = TestLoader() + loader = MyLoader() q.push(loader) loader2 = q.pop() - assert isinstance(loader2, TestLoader) - assert loader2.default_item_class is TestItem + assert isinstance(loader2, MyLoader) + assert loader2.default_item_class is MyItem self.assertEqual(loader2.name_out("x"), "xx") def test_serialize_request_recursive(self): diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index e00f69573..ecac0df9c 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,7 +1,7 @@ import asyncio import warnings -from pytest import mark +import pytest from twisted.trial.unittest import TestCase from scrapy.utils.defer import deferred_f_from_coro_f @@ -12,7 +12,7 @@ from scrapy.utils.reactor import ( ) -@mark.usefixtures("reactor_pytest") +@pytest.mark.usefixtures("reactor_pytest") class AsyncioTest(TestCase): def test_is_asyncio_reactor_installed(self): # the result should depend only on the pytest --reactor argument @@ -30,7 +30,7 @@ class AsyncioTest(TestCase): assert original_reactor == reactor - @mark.only_asyncio() + @pytest.mark.only_asyncio @deferred_f_from_coro_f async def test_set_asyncio_event_loop(self): install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index e80381671..fadbc6daa 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -16,8 +16,6 @@ from scrapy.utils.datatypes import ( ) from scrapy.utils.python import garbage_collect -__doctests__ = ["scrapy.utils.datatypes"] - class CaseInsensitiveDictMixin: def test_init_dict(self): diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index e4ab97e5d..3a1030fcf 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -1,6 +1,6 @@ import random -from pytest import mark +import pytest from twisted.internet import defer, reactor from twisted.python.failure import Failure from twisted.trial import unittest @@ -150,7 +150,7 @@ class AsyncDefTestsuiteTest(unittest.TestCase): async def test_deferred_f_from_coro_f_generator(self): yield - @mark.xfail(reason="Checks that the test is actually executed", strict=True) + @pytest.mark.xfail(reason="Checks that the test is actually executed", strict=True) @deferred_f_from_coro_f async def test_deferred_f_from_coro_f_xfail(self): raise RuntimeError("This is expected to be raised") diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 76820eabf..06e88bd10 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -119,7 +119,7 @@ class StreamLoggerTest(unittest.TestCase): @pytest.mark.parametrize( ("base_extra", "log_extra", "expected_extra"), - ( + [ ( {"spider": "test"}, {"extra": {"log_extra": "info"}}, @@ -135,7 +135,7 @@ class StreamLoggerTest(unittest.TestCase): {"extra": {"spider": "test2"}}, {"extra": {"spider": "test"}}, ), - ), + ], ) def test_spider_logger_adapter_process( base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 478c1e73a..e25bdfe3f 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -17,8 +17,6 @@ from scrapy.utils.misc import ( walk_modules, ) -__doctests__ = ["scrapy.utils.misc"] - class UtilsMiscTestCase(unittest.TestCase): def test_load_object_class(self): diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 83004cec4..a693d6b53 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -20,8 +20,6 @@ from scrapy.utils.python import ( without_none_values, ) -__doctests__ = ["scrapy.utils.python"] - class MutableChainTest(unittest.TestCase): def test_mutablechain(self): diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index db6866571..c6ba8cbbb 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -15,8 +15,6 @@ from scrapy.utils.response import ( response_status_message, ) -__doctests__ = ["scrapy.utils.response"] - class ResponseUtilsTest(unittest.TestCase): dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response") @@ -207,8 +205,8 @@ class ResponseUtilsTest(unittest.TestCase): @pytest.mark.parametrize( - "input_body,output_body", - ( + ("input_body", "output_body"), + [ ( b"ad", b"ad", ), - ), + ], ) def test_remove_html_comments(input_body, output_body): assert _remove_html_comments(input_body) == output_body, ( diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 60232f10b..858813e83 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -1,7 +1,7 @@ import asyncio +import pytest from pydispatch import dispatcher -from pytest import mark from testfixtures import LogCapture from twisted.internet import defer, reactor from twisted.python.failure import Failure @@ -67,7 +67,7 @@ class SendCatchLogDeferredTest2(SendCatchLogDeferredTest): return d -@mark.usefixtures("reactor_pytest") +@pytest.mark.usefixtures("reactor_pytest") class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): async def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) @@ -76,7 +76,7 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): return "OK" -@mark.only_asyncio() +@pytest.mark.only_asyncio class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): async def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 5fbbd74da..fc6c33200 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -5,8 +5,6 @@ from tempfile import mkdtemp from scrapy.utils.template import render_templatefile -__doctests__ = ["scrapy.utils.template"] - class UtilsRenderTemplateFileTestCase(unittest.TestCase): def setUp(self): diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 4b9a98d79..e99ef40c4 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -17,8 +17,6 @@ from scrapy.utils.url import ( # type: ignore[attr-defined] url_is_from_spider, ) -__doctests__ = ["scrapy.utils.url"] - class UrlUtilsTest(unittest.TestCase): def test_url_is_from_any_domain(self):