From 3ded1dfe31510f00e14a70811b7c01dae8b5a641 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Mar 2025 20:25:15 +0400 Subject: [PATCH] Converting tests to plain asserts, part 4. (#6702) --- tests/test_spider.py | 344 +++++++++----------- tests/test_spiderloader/__init__.py | 113 ++++--- tests/test_spidermiddleware.py | 105 +++--- tests/test_spidermiddleware_depth.py | 16 +- tests/test_spidermiddleware_httperror.py | 98 +++--- tests/test_spidermiddleware_offsite.py | 11 +- tests/test_spidermiddleware_output_chain.py | 112 +++---- tests/test_spidermiddleware_referer.py | 31 +- tests/test_spidermiddleware_urllength.py | 12 +- tests/test_spiderstate.py | 7 +- tests/test_squeues.py | 45 +-- tests/test_squeues_request.py | 148 ++++----- tests/test_stats.py | 72 ++-- 13 files changed, 529 insertions(+), 585 deletions(-) diff --git a/tests/test_spider.py b/tests/test_spider.py index af29872a8..05f1c59d0 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -31,7 +31,7 @@ from scrapy.utils.test import get_crawler from tests import get_testdata, tests_datadir -class SpiderTest(unittest.TestCase): +class TestSpider(unittest.TestCase): spider_class = Spider def setUp(self): @@ -42,19 +42,19 @@ class SpiderTest(unittest.TestCase): def test_base_spider(self): spider = self.spider_class("example.com") - self.assertEqual(spider.name, "example.com") - self.assertEqual(spider.start_urls, []) + assert spider.name == "example.com" + assert spider.start_urls == [] # pylint: disable=use-implicit-booleaness-not-comparison def test_start_requests(self): spider = self.spider_class("example.com") start_requests = spider.start_requests() - self.assertTrue(inspect.isgenerator(start_requests)) - self.assertEqual(list(start_requests), []) + assert inspect.isgenerator(start_requests) + assert not list(start_requests) def test_spider_args(self): """``__init__`` method arguments are assigned to spider attributes""" spider = self.spider_class("example.com", foo="bar") - self.assertEqual(spider.foo, "bar") + assert spider.foo == "bar" def test_spider_without_name(self): """``__init__`` method arguments are assigned to spider attributes""" @@ -67,10 +67,10 @@ class SpiderTest(unittest.TestCase): def test_from_crawler_crawler_and_settings_population(self): crawler = get_crawler() spider = self.spider_class.from_crawler(crawler, "example.com") - self.assertTrue(hasattr(spider, "crawler")) - self.assertIs(spider.crawler, crawler) - self.assertTrue(hasattr(spider, "settings")) - self.assertIs(spider.settings, crawler.settings) + assert hasattr(spider, "crawler") + assert spider.crawler is crawler + assert hasattr(spider, "settings") + assert spider.settings is crawler.settings def test_from_crawler_init_call(self): with mock.patch.object( @@ -92,7 +92,7 @@ class SpiderTest(unittest.TestCase): crawler.signals.send_catch_log( signal=signals.spider_closed, spider=spider, reason=None ) - self.assertTrue(spider.closed_called) + assert spider.closed_called def test_update_settings(self): spider_settings = {"TEST1": "spider", "TEST2": "spider"} @@ -101,9 +101,9 @@ class SpiderTest(unittest.TestCase): settings = Settings(project_settings, priority="project") self.spider_class.update_settings(settings) - self.assertEqual(settings.get("TEST1"), "spider") - self.assertEqual(settings.get("TEST2"), "spider") - self.assertEqual(settings.get("TEST3"), "project") + assert settings.get("TEST1") == "spider" + assert settings.get("TEST2") == "spider" + assert settings.get("TEST3") == "project" @inlineCallbacks def test_settings_in_from_crawler(self): @@ -121,11 +121,11 @@ class SpiderTest(unittest.TestCase): return spider crawler = Crawler(TestSpider, project_settings) - self.assertEqual(crawler.settings.get("TEST1"), "spider") - self.assertEqual(crawler.settings.get("TEST2"), "spider") - self.assertEqual(crawler.settings.get("TEST3"), "project") + assert crawler.settings.get("TEST1") == "spider" + assert crawler.settings.get("TEST2") == "spider" + assert crawler.settings.get("TEST3") == "project" yield crawler.crawl() - self.assertEqual(crawler.settings.get("TEST1"), "spider_instance") + assert crawler.settings.get("TEST1") == "spider_instance" def test_logger(self): spider = self.spider_class("example.com") @@ -134,8 +134,8 @@ class SpiderTest(unittest.TestCase): lc.check(("example.com", "INFO", "test log msg")) record = lc.records[0] - self.assertIn("spider", record.__dict__) - self.assertIs(record.spider, spider) + assert "spider" in record.__dict__ + assert record.spider is spider def test_log(self): spider = self.spider_class("example.com") @@ -144,11 +144,11 @@ class SpiderTest(unittest.TestCase): mock_logger.log.assert_called_once_with("INFO", "test log msg") -class InitSpiderTest(SpiderTest): +class TestInitSpider(TestSpider): spider_class = InitSpider -class XMLFeedSpiderTest(SpiderTest): +class TestXMLFeedSpider(TestSpider): spider_class = XMLFeedSpider def test_register_namespace(self): @@ -180,28 +180,24 @@ class XMLFeedSpiderTest(SpiderTest): for iterator in ("iternodes", "xml"): spider = _XMLSpider("example", iterator=iterator) output = list(spider._parse(response)) - self.assertEqual(len(output), 2, iterator) - self.assertEqual( - output, - [ - { - "loc": ["http://www.example.com/Special-Offers.html"], - "updated": ["2009-08-16"], - "custom": ["fuu"], - "other": ["bar"], - }, - { - "loc": [], - "updated": ["2009-08-16"], - "other": ["foo"], - "custom": [], - }, - ], - iterator, - ) + assert len(output) == 2, iterator + assert output == [ + { + "loc": ["http://www.example.com/Special-Offers.html"], + "updated": ["2009-08-16"], + "custom": ["fuu"], + "other": ["bar"], + }, + { + "loc": [], + "updated": ["2009-08-16"], + "other": ["foo"], + "custom": [], + }, + ], iterator -class CSVFeedSpiderTest(SpiderTest): +class TestCSVFeedSpider(TestSpider): spider_class = CSVFeedSpider def test_parse_rows(self): @@ -222,7 +218,7 @@ class CSVFeedSpiderTest(SpiderTest): assert len(rows) == 4 -class CrawlSpiderTest(SpiderTest): +class TestCrawlSpider(TestSpider): test_body = b"""Page title<title> <body> <p><a href="item/12.html">Item 12</a></p> @@ -247,16 +243,13 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] def test_process_links(self): response = HtmlResponse( @@ -273,16 +266,13 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] def test_process_links_filter(self): response = HtmlResponse( @@ -302,15 +292,12 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 2) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - ], - ) + assert len(output) == 2 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + ] def test_process_links_generator(self): response = HtmlResponse( @@ -327,16 +314,13 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] def test_process_request(self): response = HtmlResponse( @@ -355,16 +339,13 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.com/somepage/item/12.html", - "http://example.com/about.html", - "http://example.com/nofollow.html", - ], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.com/somepage/item/12.html", + "http://example.com/about.html", + "http://example.com/nofollow.html", + ] def test_process_request_with_response(self): response = HtmlResponse( @@ -386,20 +367,18 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) - self.assertEqual( - [r.meta["response_class"] for r in output], - ["HtmlResponse", "HtmlResponse", "HtmlResponse"], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + assert [r.meta["response_class"] for r in output] == [ + "HtmlResponse", + "HtmlResponse", + "HtmlResponse", + ] def test_process_request_instance_method(self): response = HtmlResponse( @@ -416,16 +395,13 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"), - safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"), - safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"), - ], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"), + safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"), + safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"), + ] def test_process_request_instance_method_with_response(self): response = HtmlResponse( @@ -448,32 +424,30 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) - self.assertEqual( - [r.meta["response_class"] for r in output], - ["HtmlResponse", "HtmlResponse", "HtmlResponse"], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + assert [r.meta["response_class"] for r in output] == [ + "HtmlResponse", + "HtmlResponse", + "HtmlResponse", + ] def test_follow_links_attribute_population(self): crawler = get_crawler() spider = self.spider_class.from_crawler(crawler, "example.com") - self.assertTrue(hasattr(spider, "_follow_links")) - self.assertTrue(spider._follow_links) + assert hasattr(spider, "_follow_links") + assert spider._follow_links settings_dict = {"CRAWLSPIDER_FOLLOW_LINKS": False} crawler = get_crawler(settings_dict=settings_dict) spider = self.spider_class.from_crawler(crawler, "example.com") - self.assertTrue(hasattr(spider, "_follow_links")) - self.assertFalse(spider._follow_links) + assert hasattr(spider, "_follow_links") + assert not spider._follow_links def test_start_url(self): spider = self.spider_class("example.com") @@ -483,7 +457,7 @@ class CrawlSpiderTest(SpiderTest): list(spider.start_requests()) -class SitemapSpiderTest(SpiderTest): +class TestSitemapSpider(TestSpider): spider_class = SitemapSpider BODY = b"SITEMAP" @@ -496,7 +470,7 @@ class SitemapSpiderTest(SpiderTest): def assertSitemapBody(self, response, body): crawler = get_crawler() spider = self.spider_class.from_crawler(crawler, "example.com") - self.assertEqual(spider._get_sitemap_body(response), body) + assert spider._get_sitemap_body(response) == body def test_get_sitemap_body(self): r = XmlResponse(url="http://www.example.com/", body=self.BODY) @@ -543,15 +517,12 @@ Sitemap: /sitemap-relative-url.xml r = TextResponse(url="http://www.example.com/robots.txt", body=robots) spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - [ - "http://example.com/sitemap.xml", - "http://example.com/sitemap-product-index.xml", - "http://example.com/sitemap-uppercase.xml", - "http://www.example.com/sitemap-relative-url.xml", - ], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://example.com/sitemap.xml", + "http://example.com/sitemap-product-index.xml", + "http://example.com/sitemap-uppercase.xml", + "http://www.example.com/sitemap-relative-url.xml", + ] def test_alternate_url_locs(self): sitemap = b"""<?xml version="1.0" encoding="UTF-8"?> @@ -570,21 +541,17 @@ Sitemap: /sitemap-relative-url.xml </urlset>""" r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/english/"], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/" + ] spider.sitemap_alternate_links = True - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - [ - "http://www.example.com/english/", - "http://www.example.com/deutsch/", - "http://www.example.com/schweiz-deutsch/", - "http://www.example.com/italiano/", - ], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/", + "http://www.example.com/deutsch/", + "http://www.example.com/schweiz-deutsch/", + "http://www.example.com/italiano/", + ] def test_sitemap_filter(self): sitemap = b"""<?xml version="1.0" encoding="UTF-8"?> @@ -611,16 +578,15 @@ Sitemap: /sitemap-relative-url.xml r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/english/", "http://www.example.com/portuguese/"], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/", + "http://www.example.com/portuguese/", + ] spider = FilteredSitemapSpider("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/english/"], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/" + ] def test_sitemap_filter_with_alternate_links(self): sitemap = b"""<?xml version="1.0" encoding="UTF-8"?> @@ -649,19 +615,15 @@ Sitemap: /sitemap-relative-url.xml r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - [ - "http://www.example.com/english/article_1/", - "http://www.example.com/english/article_2/", - ], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/article_1/", + "http://www.example.com/english/article_2/", + ] spider = FilteredSitemapSpider("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/deutsch/article_1/"], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/deutsch/article_1/" + ] def test_sitemapindex_filter(self): sitemap = b"""<?xml version="1.0" encoding="UTF-8"?> @@ -689,19 +651,15 @@ Sitemap: /sitemap-relative-url.xml r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - [ - "http://www.example.com/sitemap1.xml", - "http://www.example.com/sitemap2.xml", - ], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/sitemap1.xml", + "http://www.example.com/sitemap2.xml", + ] spider = FilteredSitemapSpider("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/sitemap2.xml"], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/sitemap2.xml" + ] def test_compression_bomb_setting(self): settings = {"DOWNLOAD_MAXSIZE": 10_000_000} @@ -711,7 +669,7 @@ Sitemap: /sitemap-relative-url.xml body = body_path.read_bytes() request = Request(url="https://example.com") response = Response(url="https://example.com", body=body, request=request) - self.assertIsNone(spider._get_sitemap_body(response)) + assert spider._get_sitemap_body(response) is None def test_compression_bomb_spider_attr(self): class DownloadMaxSizeSpider(self.spider_class): @@ -723,7 +681,7 @@ Sitemap: /sitemap-relative-url.xml body = body_path.read_bytes() request = Request(url="https://example.com") response = Response(url="https://example.com", body=body, request=request) - self.assertIsNone(spider._get_sitemap_body(response)) + assert spider._get_sitemap_body(response) is None def test_compression_bomb_request_meta(self): crawler = get_crawler() @@ -734,7 +692,7 @@ Sitemap: /sitemap-relative-url.xml url="https://example.com", meta={"download_maxsize": 10_000_000} ) response = Response(url="https://example.com", body=body, request=request) - self.assertIsNone(spider._get_sitemap_body(response)) + assert spider._get_sitemap_body(response) is None def test_download_warnsize_setting(self): settings = {"DOWNLOAD_WARNSIZE": 10_000_000} @@ -814,13 +772,13 @@ Sitemap: /sitemap-relative-url.xml ) -class DeprecationTest(unittest.TestCase): +class TestDeprecation: def test_crawl_spider(self): assert issubclass(CrawlSpider, Spider) assert isinstance(CrawlSpider(name="foo"), Spider) -class NoParseMethodSpiderTest(unittest.TestCase): +class TestNoParseMethodSpider: spider_class = Spider def test_undefined_parse_method(self): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index b103e9ed0..476487a04 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -8,7 +8,6 @@ from tempfile import mkdtemp from unittest import mock import pytest -from twisted.trial import unittest from zope.interface.verify import verifyObject # ugly hack to avoid cyclic imports of scrapy.spiders when running this test @@ -28,8 +27,8 @@ def _copytree(source: Path, target: Path): shutil.copytree(source, target) -class SpiderLoaderTest(unittest.TestCase): - def setUp(self): +class TestSpiderLoader: + def setup_method(self): orig_spiders_dir = module_dir / "test_spiders" self.tmpdir = Path(tempfile.mkdtemp()) self.spiders_dir = self.tmpdir / "test_spiders_xxx" @@ -38,7 +37,7 @@ class SpiderLoaderTest(unittest.TestCase): settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) self.spider_loader = SpiderLoader.from_settings(settings) - def tearDown(self): + def teardown_method(self): del self.spider_loader del sys.modules["test_spiders_xxx"] sys.path.remove(str(self.tmpdir)) @@ -47,37 +46,35 @@ class SpiderLoaderTest(unittest.TestCase): verifyObject(ISpiderLoader, self.spider_loader) def test_list(self): - self.assertEqual( - set(self.spider_loader.list()), {"spider1", "spider2", "spider3", "spider4"} - ) + assert set(self.spider_loader.list()) == { + "spider1", + "spider2", + "spider3", + "spider4", + } def test_load(self): spider1 = self.spider_loader.load("spider1") - self.assertEqual(spider1.__name__, "Spider1") + assert spider1.__name__ == "Spider1" def test_find_by_request(self): - self.assertEqual( - self.spider_loader.find_by_request(Request("http://scrapy1.org/test")), - ["spider1"], - ) - self.assertEqual( - self.spider_loader.find_by_request(Request("http://scrapy2.org/test")), - ["spider2"], - ) - self.assertEqual( - set(self.spider_loader.find_by_request(Request("http://scrapy3.org/test"))), - {"spider1", "spider2"}, - ) - self.assertEqual( - self.spider_loader.find_by_request(Request("http://scrapy999.org/test")), [] - ) - self.assertEqual( - self.spider_loader.find_by_request(Request("http://spider3.com")), [] - ) - self.assertEqual( - self.spider_loader.find_by_request(Request("http://spider3.com/onlythis")), - ["spider3"], + assert self.spider_loader.find_by_request( + Request("http://scrapy1.org/test") + ) == ["spider1"] + assert self.spider_loader.find_by_request( + Request("http://scrapy2.org/test") + ) == ["spider2"] + assert set( + self.spider_loader.find_by_request(Request("http://scrapy3.org/test")) + ) == {"spider1", "spider2"} + assert ( + self.spider_loader.find_by_request(Request("http://scrapy999.org/test")) + == [] ) + assert self.spider_loader.find_by_request(Request("http://spider3.com")) == [] + assert self.spider_loader.find_by_request( + Request("http://spider3.com/onlythis") + ) == ["spider3"] def test_load_spider_module(self): module = "tests.test_spiderloader.test_spiders.spider1" @@ -113,9 +110,9 @@ class SpiderLoaderTest(unittest.TestCase): runner = CrawlerRunner({"ADDONS": {SpiderModuleAddon: 1}}) crawler = runner.create_crawler("spider_from_addon") - self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) - self.assertEqual(crawler.spidercls.name, "spider_from_addon") - self.assertTrue(len(crawler.settings["SPIDER_MODULES"]) == 1) + assert issubclass(crawler.spidercls, scrapy.Spider) + assert crawler.spidercls.name == "spider_from_addon" + assert len(crawler.settings["SPIDER_MODULES"]) == 1 def test_crawler_runner_loading(self): module = "tests.test_spiderloader.test_spiders.spider1" @@ -129,8 +126,8 @@ class SpiderLoaderTest(unittest.TestCase): runner.create_crawler("spider2") crawler = runner.create_crawler("spider1") - self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) - self.assertEqual(crawler.spidercls.name, "spider1") + assert issubclass(crawler.spidercls, scrapy.Spider) + assert crawler.spidercls.name == "spider1" def test_bad_spider_modules_exception(self): module = "tests.test_spiderloader.test_spiders.doesnotexist" @@ -150,10 +147,10 @@ class SpiderLoaderTest(unittest.TestCase): # at least until all six versions we can import (including botocore.vendored.six) # are updated to 1.16.0+ w.pop(0) - self.assertIn("Could not load spiders from module", str(w[0].message)) + assert "Could not load spiders from module" in str(w[0].message) spiders = spider_loader.list() - self.assertEqual(spiders, []) + assert not spiders def test_syntax_error_exception(self): module = "tests.test_spiderloader.test_spiders.spider1" @@ -179,14 +176,14 @@ class SpiderLoaderTest(unittest.TestCase): # at least until all six versions we can import (including botocore.vendored.six) # are updated to 1.16.0+ w.pop(0) - self.assertIn("Could not load spiders from module", str(w[0].message)) + assert "Could not load spiders from module" in str(w[0].message) spiders = spider_loader.list() - self.assertEqual(spiders, []) + assert not spiders -class DuplicateSpiderNameLoaderTest(unittest.TestCase): - def setUp(self): +class TestDuplicateSpiderNameLoader: + def setup_method(self): orig_spiders_dir = module_dir / "test_spiders" self.tmpdir = Path(mkdtemp()) self.spiders_dir = self.tmpdir / "test_spiders_xxx" @@ -194,7 +191,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): sys.path.append(str(self.tmpdir)) self.settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) - def tearDown(self): + def teardown_method(self): del sys.modules["test_spiders_xxx"] sys.path.remove(str(self.tmpdir)) @@ -208,18 +205,18 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) - self.assertEqual(len(w), 1) + assert len(w) == 1 msg = str(w[0].message) - self.assertIn("several spiders with the same name", msg) - self.assertIn("'spider3'", msg) - self.assertTrue(msg.count("'spider3'") == 2) + assert "several spiders with the same name" in msg + assert "'spider3'" in msg + assert msg.count("'spider3'") == 2 - self.assertNotIn("'spider1'", msg) - self.assertNotIn("'spider2'", msg) - self.assertNotIn("'spider4'", msg) + assert "'spider1'" not in msg + assert "'spider2'" not in msg + assert "'spider4'" not in msg spiders = set(spider_loader.list()) - self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"}) + assert spiders == {"spider1", "spider2", "spider3", "spider4"} def test_multiple_dupename_warning(self): # copy 2 spider modules so as to have duplicate spider name @@ -236,17 +233,17 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) - self.assertEqual(len(w), 1) + assert len(w) == 1 msg = str(w[0].message) - self.assertIn("several spiders with the same name", msg) - self.assertIn("'spider1'", msg) - self.assertTrue(msg.count("'spider1'") == 2) + assert "several spiders with the same name" in msg + assert "'spider1'" in msg + assert msg.count("'spider1'") == 2 - self.assertIn("'spider2'", msg) - self.assertTrue(msg.count("'spider2'") == 2) + assert "'spider2'" in msg + assert msg.count("'spider2'") == 2 - self.assertNotIn("'spider3'", msg) - self.assertNotIn("'spider4'", msg) + assert "'spider3'" not in msg + assert "'spider4'" not in msg spiders = set(spider_loader.list()) - self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"}) + assert spiders == {"spider1", "spider2", "spider3", "spider4"} diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index a9f3876bb..ddc9b5206 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -18,7 +18,7 @@ from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.test import get_crawler -class SpiderMiddlewareTestCase(TestCase): +class TestSpiderMiddleware(TestCase): def setUp(self): self.request = Request("http://example.com/index.html") self.response = Response(self.request.url, request=self.request) @@ -41,7 +41,7 @@ class SpiderMiddlewareTestCase(TestCase): return results[0] -class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase): +class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_input method""" def test_invalid_process_spider_input(self): @@ -51,11 +51,11 @@ class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase): self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware()) result = self._scrape_response() - self.assertIsInstance(result, Failure) - self.assertIsInstance(result.value, _InvalidOutput) + assert isinstance(result, Failure) + assert isinstance(result.value, _InvalidOutput) -class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase): +class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_output method""" def test_invalid_process_spider_output(self): @@ -65,11 +65,11 @@ class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase): self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware()) result = self._scrape_response() - self.assertIsInstance(result, Failure) - self.assertIsInstance(result.value, _InvalidOutput) + assert isinstance(result, Failure) + assert isinstance(result.value, _InvalidOutput) -class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase): +class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_exception method""" def test_invalid_process_spider_exception(self): @@ -84,11 +84,11 @@ class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase): self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) result = self._scrape_response() - self.assertIsInstance(result, Failure) - self.assertIsInstance(result.value, _InvalidOutput) + assert isinstance(result, Failure) + assert isinstance(result.value, _InvalidOutput) -class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): +class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): """Re raise the exception by returning None""" def test_process_spider_exception_return_none(self): @@ -103,11 +103,11 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) result = self._scrape_response() - self.assertIsInstance(result, Failure) - self.assertIsInstance(result.value, ZeroDivisionError) + assert isinstance(result, Failure) + assert isinstance(result.value, ZeroDivisionError) -class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): +class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): """Helpers for testing sync, async and mixed middlewares. Should work for process_spider_output and, when it's supported, process_start_requests. @@ -148,14 +148,13 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): result = yield self._get_middleware_result( *mw_classes, start_index=start_index ) - self.assertIsInstance(result, Iterable) + assert isinstance(result, Iterable) result_list = list(result) - self.assertEqual(len(result_list), self.RESULT_COUNT) - self.assertIsInstance(result_list[0], self.ITEM_TYPE) - self.assertEqual("downgraded to a non-async" in str(log), downgrade) - self.assertEqual( - "doesn't support asynchronous spider output" in str(log), - ProcessSpiderOutputSimpleMiddleware in mw_classes, + assert len(result_list) == self.RESULT_COUNT + assert isinstance(result_list[0], self.ITEM_TYPE) + assert ("downgraded to a non-async" in str(log)) == downgrade + assert ("doesn't support asynchronous spider output" in str(log)) == ( + ProcessSpiderOutputSimpleMiddleware in mw_classes ) @defer.inlineCallbacks @@ -166,11 +165,11 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): result = yield self._get_middleware_result( *mw_classes, start_index=start_index ) - self.assertIsInstance(result, AsyncIterator) + assert isinstance(result, AsyncIterator) result_list = yield deferred_from_coro(collect_asyncgen(result)) - self.assertEqual(len(result_list), self.RESULT_COUNT) - self.assertIsInstance(result_list[0], self.ITEM_TYPE) - self.assertEqual("downgraded to a non-async" in str(log), downgrade) + assert len(result_list) == self.RESULT_COUNT + assert isinstance(result_list[0], self.ITEM_TYPE) + assert ("downgraded to a non-async" in str(log)) == downgrade class ProcessSpiderOutputSimpleMiddleware: @@ -212,7 +211,7 @@ class ProcessSpiderExceptionAsyncIterableMiddleware: yield {"foo": 3} -class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase): +class TestProcessSpiderOutputSimple(TestBaseAsyncSpiderMiddleware): """process_spider_output tests for simple callbacks""" ITEM_TYPE = dict @@ -257,7 +256,7 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase): return self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN) -class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple): +class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple): """process_spider_output tests for async generator callbacks""" async def _scrape_func(self, *args, **kwargs): @@ -297,7 +296,7 @@ class ProcessSpiderOutputCoroutineMiddleware: return result -class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): +class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware): @defer.inlineCallbacks def test_non_iterable(self): with pytest.raises( @@ -324,7 +323,7 @@ class ProcessStartRequestsSimpleMiddleware: yield from start_requests -class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase): +class TestProcessStartRequestsSimple(TestBaseAsyncSpiderMiddleware): """process_start_requests tests for simple start_requests""" ITEM_TYPE = (Request, dict) @@ -373,67 +372,65 @@ class UniversalMiddlewareBothAsync: yield -class UniversalMiddlewareManagerTest(TestCase): - def setUp(self): +class TestUniversalMiddlewareManager: + def setup_method(self): self.mwman = SpiderMiddlewareManager() def test_simple_mw(self): mw = ProcessSpiderOutputSimpleMiddleware() self.mwman._add_middleware(mw) - self.assertEqual( - self.mwman.methods["process_spider_output"][0], mw.process_spider_output + assert ( + self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable ) def test_async_mw(self): mw = ProcessSpiderOutputAsyncGenMiddleware() self.mwman._add_middleware(mw) - self.assertEqual( - self.mwman.methods["process_spider_output"][0], mw.process_spider_output + assert ( + self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable ) def test_universal_mw(self): mw = ProcessSpiderOutputUniversalMiddleware() self.mwman._add_middleware(mw) - self.assertEqual( - self.mwman.methods["process_spider_output"][0], - (mw.process_spider_output, mw.process_spider_output_async), + assert self.mwman.methods["process_spider_output"][0] == ( + mw.process_spider_output, + mw.process_spider_output_async, ) def test_universal_mw_no_sync(self): with LogCapture() as log: self.mwman._add_middleware(UniversalMiddlewareNoSync()) - self.assertIn( + assert ( "UniversalMiddlewareNoSync has process_spider_output_async" - " without process_spider_output", - str(log), + " without process_spider_output" in str(log) ) - self.assertEqual(self.mwman.methods["process_spider_output"][0], None) + assert self.mwman.methods["process_spider_output"][0] is None def test_universal_mw_both_sync(self): mw = UniversalMiddlewareBothSync() with LogCapture() as log: self.mwman._add_middleware(mw) - self.assertIn( + assert ( "UniversalMiddlewareBothSync.process_spider_output_async " - "is not an async generator function", - str(log), + "is not an async generator function" in str(log) ) - self.assertEqual( - self.mwman.methods["process_spider_output"][0], mw.process_spider_output + assert ( + self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable ) def test_universal_mw_both_async(self): with LogCapture() as log: self.mwman._add_middleware(UniversalMiddlewareBothAsync()) - self.assertIn( + assert ( "UniversalMiddlewareBothAsync.process_spider_output " - "is an async generator function while process_spider_output_async exists", - str(log), + "is an async generator function while process_spider_output_async exists" + in str(log) ) - self.assertEqual(self.mwman.methods["process_spider_output"][0], None) + assert self.mwman.methods["process_spider_output"][0] is None -class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase): +class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware): ITEM_TYPE = dict MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware @@ -474,7 +471,7 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase): return self._test_simple_base(self.MW_UNIVERSAL) -class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest): +class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple): async def _scrape_func(self, *args, **kwargs): for item in super()._scrape_func(): yield item @@ -503,7 +500,7 @@ class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest): return self._test_asyncgen_base(self.MW_UNIVERSAL) -class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase): +class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): ITEM_TYPE = dict MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index e359d9cfc..dfcc141c3 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -1,5 +1,3 @@ -from unittest import TestCase - from scrapy.http import Request, Response from scrapy.spidermiddlewares.depth import DepthMiddleware from scrapy.spiders import Spider @@ -7,8 +5,8 @@ from scrapy.statscollectors import StatsCollector from scrapy.utils.test import get_crawler -class TestDepthMiddleware(TestCase): - def setUp(self): +class TestDepthMiddleware: + def setup_method(self): crawler = get_crawler(Spider) self.spider = crawler._create_spider("scrapytest.org") @@ -24,18 +22,18 @@ class TestDepthMiddleware(TestCase): result = [Request("http://scrapytest.org")] out = list(self.mw.process_spider_output(resp, result, self.spider)) - self.assertEqual(out, result) + assert out == result rdc = self.stats.get_value("request_depth_count/1", spider=self.spider) - self.assertEqual(rdc, 1) + assert rdc == 1 req.meta["depth"] = 1 out2 = list(self.mw.process_spider_output(resp, result, self.spider)) - self.assertEqual(out2, []) + assert not out2 rdm = self.stats.get_value("request_depth_max", spider=self.spider) - self.assertEqual(rdm, 1) + assert rdm == 1 - def tearDown(self): + def teardown_method(self): self.stats.close_spider(self.spider, "") diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index f9eb93d6b..e306579fa 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,10 +1,9 @@ import logging -from unittest import TestCase import pytest from testfixtures import LogCapture from twisted.internet import defer -from twisted.trial.unittest import TestCase as TrialTestCase +from twisted.trial.unittest import TestCase from scrapy.http import Request, Response from scrapy.settings import Settings @@ -59,8 +58,8 @@ def _responses(request, status_codes): return responses -class TestHttpErrorMiddleware(TestCase): - def setUp(self): +class TestHttpErrorMiddleware: + def setup_method(self): crawler = get_crawler(Spider) self.spider = Spider.from_crawler(crawler, name="foo") self.mw = HttpErrorMiddleware(Settings({})) @@ -68,19 +67,20 @@ class TestHttpErrorMiddleware(TestCase): self.res200, self.res404 = _responses(self.req, [200, 404]) def test_process_spider_input(self): - self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) + assert self.mw.process_spider_input(self.res200, self.spider) is None with pytest.raises(HttpError): self.mw.process_spider_input(self.res404, self.spider) def test_process_spider_exception(self): - self.assertEqual( - [], + assert ( self.mw.process_spider_exception( self.res404, HttpError(self.res404), self.spider - ), + ) + == [] ) - self.assertIsNone( + assert ( self.mw.process_spider_exception(self.res404, Exception(), self.spider) + is None ) def test_handle_httpstatus_list(self): @@ -88,26 +88,26 @@ class TestHttpErrorMiddleware(TestCase): res.request = Request( "http://scrapytest.org", meta={"handle_httpstatus_list": [404]} ) - self.assertIsNone(self.mw.process_spider_input(res, self.spider)) + assert self.mw.process_spider_input(res, self.spider) is None self.spider.handle_httpstatus_list = [404] - self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) + assert self.mw.process_spider_input(self.res404, self.spider) is None -class TestHttpErrorMiddlewareSettings(TestCase): +class TestHttpErrorMiddlewareSettings: """Similar test, but with settings""" - def setUp(self): + def setup_method(self): self.spider = Spider("foo") self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)})) self.req = Request("http://scrapytest.org") self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) + assert self.mw.process_spider_input(self.res200, self.spider) is None with pytest.raises(HttpError): self.mw.process_spider_input(self.res404, self.spider) - self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider)) + assert self.mw.process_spider_input(self.res402, self.spider) is None def test_meta_overrides_settings(self): request = Request( @@ -118,27 +118,27 @@ class TestHttpErrorMiddlewareSettings(TestCase): res402 = self.res402.copy() res402.request = request - self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) + assert self.mw.process_spider_input(res404, self.spider) is None with pytest.raises(HttpError): self.mw.process_spider_input(res402, self.spider) def test_spider_override_settings(self): self.spider.handle_httpstatus_list = [404] - self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) + assert self.mw.process_spider_input(self.res404, self.spider) is None with pytest.raises(HttpError): self.mw.process_spider_input(self.res402, self.spider) -class TestHttpErrorMiddlewareHandleAll(TestCase): - def setUp(self): +class TestHttpErrorMiddlewareHandleAll: + def setup_method(self): self.spider = Spider("foo") self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True})) self.req = Request("http://scrapytest.org") self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) - self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) + assert self.mw.process_spider_input(self.res200, self.spider) is None + assert self.mw.process_spider_input(self.res404, self.spider) is None def test_meta_overrides_settings(self): request = Request( @@ -149,7 +149,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): res402 = self.res402.copy() res402.request = request - self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) + assert self.mw.process_spider_input(res404, self.spider) is None with pytest.raises(HttpError): self.mw.process_spider_input(res402, self.spider) @@ -169,10 +169,10 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): with pytest.raises(HttpError): mw.process_spider_input(res404, self.spider) - self.assertIsNone(mw.process_spider_input(res402, self.spider)) + assert mw.process_spider_input(res402, self.spider) is None -class TestHttpErrorMiddlewareIntegrational(TrialTestCase): +class TestHttpErrorMiddlewareIntegrational(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -187,28 +187,28 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): crawler = get_crawler(_HttpErrorSpider) yield crawler.crawl(mockserver=self.mockserver) assert not crawler.spider.skipped, crawler.spider.skipped - self.assertEqual(crawler.spider.parsed, {"200"}) - self.assertEqual(crawler.spider.failed, {"404", "402", "500"}) + assert crawler.spider.parsed == {"200"} + assert crawler.spider.failed == {"404", "402", "500"} get_value = crawler.stats.get_value - self.assertEqual(get_value("httperror/response_ignored_count"), 3) - self.assertEqual(get_value("httperror/response_ignored_status_count/404"), 1) - self.assertEqual(get_value("httperror/response_ignored_status_count/402"), 1) - self.assertEqual(get_value("httperror/response_ignored_status_count/500"), 1) + assert get_value("httperror/response_ignored_count") == 3 + assert get_value("httperror/response_ignored_status_count/404") == 1 + assert get_value("httperror/response_ignored_status_count/402") == 1 + assert get_value("httperror/response_ignored_status_count/500") == 1 @defer.inlineCallbacks def test_logging(self): crawler = get_crawler(_HttpErrorSpider) with LogCapture() as log: yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402}) - self.assertEqual(crawler.spider.parsed, {"200", "402"}) - self.assertEqual(crawler.spider.skipped, {"402"}) - self.assertEqual(crawler.spider.failed, {"404", "500"}) + assert crawler.spider.parsed == {"200", "402"} + assert crawler.spider.skipped == {"402"} + assert crawler.spider.failed == {"404", "500"} - self.assertIn("Ignoring response <404", str(log)) - self.assertIn("Ignoring response <500", str(log)) - self.assertNotIn("Ignoring response <200", str(log)) - self.assertNotIn("Ignoring response <402", str(log)) + assert "Ignoring response <404" in str(log) + assert "Ignoring response <500" in str(log) + assert "Ignoring response <200" not in str(log) + assert "Ignoring response <402" not in str(log) @defer.inlineCallbacks def test_logging_level(self): @@ -216,22 +216,22 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): crawler = get_crawler(_HttpErrorSpider) with LogCapture(level=logging.INFO) as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(crawler.spider.parsed, {"200"}) - self.assertEqual(crawler.spider.failed, {"404", "402", "500"}) + assert crawler.spider.parsed == {"200"} + assert crawler.spider.failed == {"404", "402", "500"} - self.assertIn("Ignoring response <402", str(log)) - self.assertIn("Ignoring response <404", str(log)) - self.assertIn("Ignoring response <500", str(log)) - self.assertNotIn("Ignoring response <200", str(log)) + assert "Ignoring response <402" in str(log) + assert "Ignoring response <404" in str(log) + assert "Ignoring response <500" in str(log) + assert "Ignoring response <200" not in str(log) # with level WARNING, we shouldn't capture anything from HttpError crawler = get_crawler(_HttpErrorSpider) with LogCapture(level=logging.WARNING) as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(crawler.spider.parsed, {"200"}) - self.assertEqual(crawler.spider.failed, {"404", "402", "500"}) + assert crawler.spider.parsed == {"200"} + assert crawler.spider.failed == {"404", "402", "500"} - self.assertNotIn("Ignoring response <402", str(log)) - self.assertNotIn("Ignoring response <404", str(log)) - self.assertNotIn("Ignoring response <500", str(log)) - self.assertNotIn("Ignoring response <200", str(log)) + assert "Ignoring response <402" not in str(log) + assert "Ignoring response <404" not in str(log) + assert "Ignoring response <500" not in str(log) + assert "Ignoring response <200" not in str(log) diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 906928e01..f4563a0a4 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -1,5 +1,4 @@ import warnings -from unittest import TestCase from urllib.parse import urlparse from scrapy.http import Request, Response @@ -8,8 +7,8 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class TestOffsiteMiddleware(TestCase): - def setUp(self): +class TestOffsiteMiddleware: + def setup_method(self): crawler = get_crawler(Spider) self.spider = crawler._create_spider(**self._get_spiderargs()) self.mw = OffsiteMiddleware.from_crawler(crawler) @@ -46,7 +45,7 @@ class TestOffsiteMiddleware(TestCase): reqs = onsite_reqs + offsite_reqs out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEqual(out, onsite_reqs) + assert out == onsite_reqs class TestOffsiteMiddleware2(TestOffsiteMiddleware): @@ -57,7 +56,7 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware): res = Response("http://scrapytest.org") reqs = [Request("http://a.com/b.html"), Request("http://b.com/1")] out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEqual(out, reqs) + assert out == reqs class TestOffsiteMiddleware3(TestOffsiteMiddleware2): @@ -77,7 +76,7 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3): res = Response("http://scrapytest.org") reqs = [Request("http://scrapytest.org/1")] out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEqual(out, reqs) + assert out == reqs class TestOffsiteMiddleware5(TestOffsiteMiddleware4): diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index e51957497..6e26a85ea 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -324,9 +324,9 @@ class TestSpiderMiddleware(TestCase): was enqueued from the recovery middleware) """ log = yield self.crawl_log(RecoverySpider) - self.assertIn("Middleware: TabError exception caught", str(log)) - self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1) - self.assertIn("'item_scraped_count': 3", str(log)) + assert "Middleware: TabError exception caught" in str(log) + assert str(log).count("Middleware: TabError exception caught") == 1 + assert "'item_scraped_count': 3" in str(log) @defer.inlineCallbacks def test_recovery_asyncgen(self): @@ -334,9 +334,9 @@ class TestSpiderMiddleware(TestCase): Same as test_recovery but with an async callback. """ log = yield self.crawl_log(RecoveryAsyncGenSpider) - self.assertIn("Middleware: TabError exception caught", str(log)) - self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1) - self.assertIn("'item_scraped_count': 3", str(log)) + assert "Middleware: TabError exception caught" in str(log) + assert str(log).count("Middleware: TabError exception caught") == 1 + assert "'item_scraped_count': 3" in str(log) @defer.inlineCallbacks def test_process_spider_input_without_errback(self): @@ -345,8 +345,8 @@ class TestSpiderMiddleware(TestCase): process_spider_exception chain from the start if the Request has no errback """ log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithoutErrback) - self.assertIn("Middleware: will raise IndexError", str(log1)) - self.assertIn("Middleware: IndexError exception caught", str(log1)) + assert "Middleware: will raise IndexError" in str(log1) + assert "Middleware: IndexError exception caught" in str(log1) @defer.inlineCallbacks def test_process_spider_input_with_errback(self): @@ -355,12 +355,12 @@ class TestSpiderMiddleware(TestCase): process_spider_exception chain if the Request has an errback """ log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithErrback) - self.assertNotIn("Middleware: IndexError exception caught", str(log1)) - self.assertIn("Middleware: will raise IndexError", str(log1)) - self.assertIn("Got a Failure on the Request errback", str(log1)) - self.assertIn("{'from': 'errback'}", str(log1)) - self.assertNotIn("{'from': 'callback'}", str(log1)) - self.assertIn("'item_scraped_count': 1", str(log1)) + assert "Middleware: IndexError exception caught" not in str(log1) + assert "Middleware: will raise IndexError" in str(log1) + assert "Got a Failure on the Request errback" in str(log1) + assert "{'from': 'errback'}" in str(log1) + assert "{'from': 'callback'}" not in str(log1) + assert "'item_scraped_count': 1" in str(log1) @defer.inlineCallbacks def test_generator_callback(self): @@ -370,8 +370,8 @@ class TestSpiderMiddleware(TestCase): exception is raised should be processed normally. """ log2 = yield self.crawl_log(GeneratorCallbackSpider) - self.assertIn("Middleware: ImportError exception caught", str(log2)) - self.assertIn("'item_scraped_count': 2", str(log2)) + assert "Middleware: ImportError exception caught" in str(log2) + assert "'item_scraped_count': 2" in str(log2) @defer.inlineCallbacks def test_async_generator_callback(self): @@ -379,8 +379,8 @@ class TestSpiderMiddleware(TestCase): Same as test_generator_callback but with an async callback. """ log2 = yield self.crawl_log(AsyncGeneratorCallbackSpider) - self.assertIn("Middleware: ImportError exception caught", str(log2)) - self.assertIn("'item_scraped_count': 2", str(log2)) + assert "Middleware: ImportError exception caught" in str(log2) + assert "'item_scraped_count': 2" in str(log2) @defer.inlineCallbacks def test_generator_callback_right_after_callback(self): @@ -389,8 +389,8 @@ class TestSpiderMiddleware(TestCase): even if the middleware is placed right after the spider """ log21 = yield self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider) - self.assertIn("Middleware: ImportError exception caught", str(log21)) - self.assertIn("'item_scraped_count': 2", str(log21)) + assert "Middleware: ImportError exception caught" in str(log21) + assert "'item_scraped_count': 2" in str(log21) @defer.inlineCallbacks def test_not_a_generator_callback(self): @@ -399,8 +399,8 @@ class TestSpiderMiddleware(TestCase): be caught by the process_spider_exception chain. No items should be processed. """ log3 = yield self.crawl_log(NotGeneratorCallbackSpider) - self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3)) - self.assertNotIn("item_scraped_count", str(log3)) + assert "Middleware: ZeroDivisionError exception caught" in str(log3) + assert "item_scraped_count" not in str(log3) @defer.inlineCallbacks def test_not_a_generator_callback_right_after_callback(self): @@ -411,8 +411,8 @@ class TestSpiderMiddleware(TestCase): log31 = yield self.crawl_log( NotGeneratorCallbackSpiderMiddlewareRightAfterSpider ) - self.assertIn("Middleware: ZeroDivisionError exception caught", str(log31)) - self.assertNotIn("item_scraped_count", str(log31)) + assert "Middleware: ZeroDivisionError exception caught" in str(log31) + assert "item_scraped_count" not in str(log31) @defer.inlineCallbacks def test_generator_output_chain(self): @@ -425,22 +425,22 @@ class TestSpiderMiddleware(TestCase): process_spider_exception chain) """ log4 = yield self.crawl_log(GeneratorOutputChainSpider) - self.assertIn("'item_scraped_count': 2", str(log4)) - self.assertIn( - "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught", - str(log4), + assert "'item_scraped_count': 2" in str(log4) + assert ( + "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught" + in str(log4) ) - self.assertIn( - "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught", - str(log4), + assert ( + "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught" + in str(log4) ) - self.assertNotIn( - "GeneratorFailMiddleware.process_spider_exception: LookupError caught", - str(log4), + assert ( + "GeneratorFailMiddleware.process_spider_exception: LookupError caught" + not in str(log4) ) - self.assertNotIn( - "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught", - str(log4), + assert ( + "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught" + not in str(log4) ) item_from_callback = { "processed": [ @@ -457,9 +457,9 @@ class TestSpiderMiddleware(TestCase): "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output", ] } - self.assertIn(str(item_from_callback), str(log4)) - self.assertIn(str(item_recovered), str(log4)) - self.assertNotIn("parse-second-item", str(log4)) + assert str(item_from_callback) in str(log4) + assert str(item_recovered) in str(log4) + assert "parse-second-item" not in str(log4) @defer.inlineCallbacks def test_not_a_generator_output_chain(self): @@ -472,22 +472,22 @@ class TestSpiderMiddleware(TestCase): from the spider callback are lost) """ log5 = yield self.crawl_log(NotGeneratorOutputChainSpider) - self.assertIn("'item_scraped_count': 1", str(log5)) - self.assertIn( - "GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught", - str(log5), + assert "'item_scraped_count': 1" in str(log5) + assert ( + "GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught" + in str(log5) ) - self.assertIn( - "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught", - str(log5), + assert ( + "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught" + in str(log5) ) - self.assertNotIn( - "GeneratorFailMiddleware.process_spider_exception: ReferenceError caught", - str(log5), + assert ( + "GeneratorFailMiddleware.process_spider_exception: ReferenceError caught" + not in str(log5) ) - self.assertNotIn( - "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught", - str(log5), + assert ( + "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught" + not in str(log5) ) item_recovered = { "processed": [ @@ -495,6 +495,6 @@ class TestSpiderMiddleware(TestCase): "NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output", ] } - self.assertIn(str(item_recovered), str(log5)) - self.assertNotIn("parse-first-item", str(log5)) - self.assertNotIn("parse-second-item", str(log5)) + assert str(item_recovered) in str(log5) + assert "parse-first-item" not in str(log5) + assert "parse-second-item" not in str(log5) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 01a87c645..300a40c13 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -2,7 +2,6 @@ from __future__ import annotations import warnings from typing import Any -from unittest import TestCase from urllib.parse import urlparse import pytest @@ -35,7 +34,7 @@ from scrapy.spidermiddlewares.referer import ( from scrapy.spiders import Spider -class TestRefererMiddleware(TestCase): +class TestRefererMiddleware: req_meta: dict[str, Any] = {} resp_headers: dict[str, str] = {} settings: dict[str, Any] = {} @@ -43,7 +42,7 @@ class TestRefererMiddleware(TestCase): ("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"), ] - def setUp(self): + def setup_method(self): self.spider = Spider("foo") settings = Settings(self.settings) self.mw = RefererMiddleware(settings) @@ -59,7 +58,7 @@ class TestRefererMiddleware(TestCase): response = self.get_response(origin) request = self.get_request(target) out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEqual(out[0].headers.get("Referer"), referrer) + assert out[0].headers.get("Referer") == referrer class MixinDefault: @@ -773,7 +772,7 @@ class TestRequestMetaPrecedence003(MixinUnsafeUrl, TestRefererMiddleware): req_meta = {"referrer_policy": POLICY_UNSAFE_URL} -class TestRequestMetaSettingFallback(TestCase): +class TestRequestMetaSettingFallback: params = [ ( # When an unknown policy is referenced in Request.meta @@ -844,14 +843,14 @@ class TestRequestMetaSettingFallback(TestCase): with warnings.catch_warnings(record=True) as w: policy = mw.policy(response, request) - self.assertIsInstance(policy, policy_class) + assert isinstance(policy, policy_class) if check_warning: - self.assertEqual(len(w), 1) - self.assertEqual(w[0].category, RuntimeWarning, w[0].message) + assert len(w) == 1 + assert w[0].category is RuntimeWarning, w[0].message -class TestSettingsPolicyByName(TestCase): +class TestSettingsPolicyByName: def test_valid_name(self): for s, p in [ (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), @@ -866,7 +865,7 @@ class TestSettingsPolicyByName(TestCase): ]: settings = Settings({"REFERRER_POLICY": s}) mw = RefererMiddleware(settings) - self.assertEqual(mw.default_policy, p) + assert mw.default_policy == p def test_valid_name_casevariants(self): for s, p in [ @@ -882,7 +881,7 @@ class TestSettingsPolicyByName(TestCase): ]: settings = Settings({"REFERRER_POLICY": s.upper()}) mw = RefererMiddleware(settings) - self.assertEqual(mw.default_policy, p) + assert mw.default_policy == p def test_invalid_name(self): settings = Settings({"REFERRER_POLICY": "some-custom-unknown-policy"}) @@ -902,7 +901,7 @@ class TestSettingsPolicyByName(TestCase): } ) mw1 = RefererMiddleware(settings1) - self.assertEqual(mw1.default_policy, StrictOriginWhenCrossOriginPolicy) + assert mw1.default_policy == StrictOriginWhenCrossOriginPolicy # test parsing with space(s) after the comma settings2 = Settings( @@ -915,7 +914,7 @@ class TestSettingsPolicyByName(TestCase): } ) mw2 = RefererMiddleware(settings2) - self.assertEqual(mw2.default_policy, UnsafeUrlPolicy) + assert mw2.default_policy == UnsafeUrlPolicy def test_multiple_policy_tokens_all_invalid(self): settings = Settings( @@ -1003,7 +1002,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware): ), ] - def setUp(self): + def setup_method(self): self.spider = Spider("foo") settings = Settings(self.settings) self.referrermw = RefererMiddleware(settings) @@ -1023,7 +1022,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware): out = list( self.referrermw.process_spider_output(response, [request], self.spider) ) - self.assertEqual(out[0].headers.get("Referer"), init_referrer) + assert out[0].headers.get("Referer") == init_referrer for status, url in redirections: response = Response( @@ -1035,7 +1034,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware): self.referrermw.request_scheduled(request, self.spider) assert isinstance(request, Request) - self.assertEqual(request.headers.get("Referer"), final_referrer) + assert request.headers.get("Referer") == final_referrer class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect): diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 1a0f2e223..5cc3cdc6c 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -1,5 +1,3 @@ -from unittest import TestCase - from testfixtures import LogCapture from scrapy.http import Request, Response @@ -8,8 +6,8 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class TestUrlLengthMiddleware(TestCase): - def setUp(self): +class TestUrlLengthMiddleware: + def setup_method(self): self.maxlength = 25 crawler = get_crawler(Spider, {"URLLENGTH_LIMIT": self.maxlength}) self.spider = crawler._create_spider("foo") @@ -27,7 +25,7 @@ class TestUrlLengthMiddleware(TestCase): ) def test_middleware_works(self): - self.assertEqual(self.process_spider_output(), [self.short_url_req]) + assert self.process_spider_output() == [self.short_url_req] def test_logging(self): with LogCapture() as log: @@ -36,6 +34,6 @@ class TestUrlLengthMiddleware(TestCase): ric = self.stats.get_value( "urllength/request_ignored_count", spider=self.spider ) - self.assertEqual(ric, 1) + assert ric == 1 - self.assertIn(f"Ignoring link (url length > {self.maxlength})", str(log)) + assert f"Ignoring link (url length > {self.maxlength})" in str(log) diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index 72692afab..cd31891a0 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -3,7 +3,6 @@ from datetime import datetime, timezone from tempfile import mkdtemp import pytest -from twisted.trial import unittest from scrapy.exceptions import NotConfigured from scrapy.extensions.spiderstate import SpiderState @@ -11,7 +10,7 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class SpiderStateTest(unittest.TestCase): +class TestSpiderState: def test_store_load(self): jobdir = mkdtemp() try: @@ -27,7 +26,7 @@ class SpiderStateTest(unittest.TestCase): spider2 = Spider(name="default") ss2 = SpiderState(jobdir) ss2.spider_opened(spider2) - self.assertEqual(spider.state, {"one": 1, "dt": dt}) + assert spider.state == {"one": 1, "dt": dt} ss2.spider_closed(spider2) finally: shutil.rmtree(jobdir) @@ -38,7 +37,7 @@ class SpiderStateTest(unittest.TestCase): spider = Spider(name="default") ss = SpiderState() ss.spider_opened(spider) - self.assertEqual(spider.state, {}) + assert spider.state == {} ss.spider_closed(spider) def test_not_configured(self): diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 8556b75dd..6283b9ad6 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -50,9 +50,9 @@ class FifoDiskQueueTestMixin: q.push("a") q.push(123) q.push({"a": "dict"}) - self.assertEqual(q.pop(), "a") - self.assertEqual(q.pop(), 123) - self.assertEqual(q.pop(), {"a": "dict"}) + assert q.pop() == "a" + assert q.pop() == 123 + assert q.pop() == {"a": "dict"} test_nonserializable_object = nonserializable_object_test @@ -92,7 +92,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): q.push(i) i2 = q.pop() assert isinstance(i2, MyItem) - self.assertEqual(i, i2) + assert i == i2 def test_serialize_loader(self): q = self.queue() @@ -101,7 +101,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): loader2 = q.pop() assert isinstance(loader2, MyLoader) assert loader2.default_item_class is MyItem - self.assertEqual(loader2.name_out("x"), "xx") + assert loader2.name_out("x") == "xx" def test_serialize_request_recursive(self): q = self.queue() @@ -110,23 +110,26 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): q.push(r) r2 = q.pop() assert isinstance(r2, Request) - self.assertEqual(r.url, r2.url) + assert r.url == r2.url assert r2.meta["request"] is r2 def test_non_pickable_object(self): q = self.queue() - try: + with pytest.raises( + ValueError, + match="Can't (get|pickle) local object|Can't pickle .*: it's not found as", + ) as exc_info: q.push(lambda x: x) - except ValueError as exc: - if hasattr(sys, "pypy_version_info"): - self.assertIsInstance(exc.__context__, pickle.PicklingError) - else: - self.assertIsInstance(exc.__context__, AttributeError) + if hasattr(sys, "pypy_version_info"): + assert isinstance(exc_info.value.__context__, pickle.PicklingError) + else: + assert isinstance(exc_info.value.__context__, AttributeError) sel = Selector(text="<html><body><p>some text</p></body></html>") - try: + with pytest.raises( + ValueError, match="can't pickle Selector objects" + ) as exc_info: q.push(sel) - except ValueError as exc: - self.assertIsInstance(exc.__context__, TypeError) + assert isinstance(exc_info.value.__context__, TypeError) class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): @@ -151,9 +154,9 @@ class LifoDiskQueueTestMixin: q.push("a") q.push(123) q.push({"a": "dict"}) - self.assertEqual(q.pop(), {"a": "dict"}) - self.assertEqual(q.pop(), 123) - self.assertEqual(q.pop(), "a") + assert q.pop() == {"a": "dict"} + assert q.pop() == 123 + assert q.pop() == "a" test_nonserializable_object = nonserializable_object_test @@ -173,7 +176,7 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): q.push(i) i2 = q.pop() assert isinstance(i2, MyItem) - self.assertEqual(i, i2) + assert i == i2 def test_serialize_loader(self): q = self.queue() @@ -182,7 +185,7 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): loader2 = q.pop() assert isinstance(loader2, MyLoader) assert loader2.default_item_class is MyItem - self.assertEqual(loader2.name_out("x"), "xx") + assert loader2.name_out("x") == "xx" def test_serialize_request_recursive(self): q = self.queue() @@ -191,5 +194,5 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): q.push(r) r2 = q.pop() assert isinstance(r2, Request) - self.assertEqual(r.url, r2.url) + assert r.url == r2.url assert r2.meta["request"] is r2 diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index 88f6657d8..6c153f40e 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -22,14 +22,14 @@ from scrapy.squeues import ( from scrapy.utils.test import get_crawler -class BaseQueueTestCase(unittest.TestCase): - def setUp(self): +class TestBaseQueue: + def setup_method(self): self.tmpdir = tempfile.mkdtemp(prefix="scrapy-queue-tests-") self.qpath = self.tempfilename() self.qdir = tempfile.mkdtemp() self.crawler = get_crawler(Spider) - def tearDown(self): + def teardown_method(self): shutil.rmtree(self.tmpdir) def tempfilename(self): @@ -48,36 +48,36 @@ class RequestQueueTestMixin: if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues do not define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None req = Request("http://www.example.com") q.push(req) - self.assertEqual(len(q), 1) - self.assertEqual(q.peek().url, req.url) - self.assertEqual(q.pop().url, req.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 1 + assert q.peek().url == req.url + assert q.pop().url == req.url + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None q.close() def test_one_element_without_peek(self): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.pop() is None req = Request("http://www.example.com") q.push(req) - self.assertEqual(len(q), 1) + assert len(q) == 1 with pytest.raises( NotImplementedError, match="The underlying queue class does not implement 'peek'", ): q.peek() - self.assertEqual(q.pop().url, req.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert q.pop().url == req.url + assert len(q) == 0 + assert q.pop() is None q.close() @@ -86,35 +86,35 @@ class FifoQueueMixin(RequestQueueTestMixin): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues do not define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None req1 = Request("http://www.example.com/1") req2 = Request("http://www.example.com/2") req3 = Request("http://www.example.com/3") q.push(req1) q.push(req2) q.push(req3) - self.assertEqual(len(q), 3) - self.assertEqual(q.peek().url, req1.url) - self.assertEqual(q.pop().url, req1.url) - self.assertEqual(len(q), 2) - self.assertEqual(q.peek().url, req2.url) - self.assertEqual(q.pop().url, req2.url) - self.assertEqual(len(q), 1) - self.assertEqual(q.peek().url, req3.url) - self.assertEqual(q.pop().url, req3.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 3 + assert q.peek().url == req1.url + assert q.pop().url == req1.url + assert len(q) == 2 + assert q.peek().url == req2.url + assert q.pop().url == req2.url + assert len(q) == 1 + assert q.peek().url == req3.url + assert q.pop().url == req3.url + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None q.close() def test_fifo_without_peek(self): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues do not define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.pop() is None req1 = Request("http://www.example.com/1") req2 = Request("http://www.example.com/2") req3 = Request("http://www.example.com/3") @@ -126,14 +126,14 @@ class FifoQueueMixin(RequestQueueTestMixin): match="The underlying queue class does not implement 'peek'", ): q.peek() - self.assertEqual(len(q), 3) - self.assertEqual(q.pop().url, req1.url) - self.assertEqual(len(q), 2) - self.assertEqual(q.pop().url, req2.url) - self.assertEqual(len(q), 1) - self.assertEqual(q.pop().url, req3.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert len(q) == 3 + assert q.pop().url == req1.url + assert len(q) == 2 + assert q.pop().url == req2.url + assert len(q) == 1 + assert q.pop().url == req3.url + assert len(q) == 0 + assert q.pop() is None q.close() @@ -142,35 +142,35 @@ class LifoQueueMixin(RequestQueueTestMixin): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues do not define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None req1 = Request("http://www.example.com/1") req2 = Request("http://www.example.com/2") req3 = Request("http://www.example.com/3") q.push(req1) q.push(req2) q.push(req3) - self.assertEqual(len(q), 3) - self.assertEqual(q.peek().url, req3.url) - self.assertEqual(q.pop().url, req3.url) - self.assertEqual(len(q), 2) - self.assertEqual(q.peek().url, req2.url) - self.assertEqual(q.pop().url, req2.url) - self.assertEqual(len(q), 1) - self.assertEqual(q.peek().url, req1.url) - self.assertEqual(q.pop().url, req1.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 3 + assert q.peek().url == req3.url + assert q.pop().url == req3.url + assert len(q) == 2 + assert q.peek().url == req2.url + assert q.pop().url == req2.url + assert len(q) == 1 + assert q.peek().url == req1.url + assert q.pop().url == req1.url + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None q.close() def test_lifo_without_peek(self): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues do not define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.pop() is None req1 = Request("http://www.example.com/1") req2 = Request("http://www.example.com/2") req3 = Request("http://www.example.com/3") @@ -182,46 +182,46 @@ class LifoQueueMixin(RequestQueueTestMixin): match="The underlying queue class does not implement 'peek'", ): q.peek() - self.assertEqual(len(q), 3) - self.assertEqual(q.pop().url, req3.url) - self.assertEqual(len(q), 2) - self.assertEqual(q.pop().url, req2.url) - self.assertEqual(len(q), 1) - self.assertEqual(q.pop().url, req1.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert len(q) == 3 + assert q.pop().url == req3.url + assert len(q) == 2 + assert q.pop().url == req2.url + assert len(q) == 1 + assert q.pop().url == req1.url + assert len(q) == 0 + assert q.pop() is None q.close() -class PickleFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): +class TestPickleFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue): def queue(self): return PickleFifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/fifo") -class PickleLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): +class TestPickleLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue): def queue(self): return PickleLifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/lifo") -class MarshalFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): +class TestMarshalFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue): def queue(self): return MarshalFifoDiskQueue.from_crawler( crawler=self.crawler, key="marshal/fifo" ) -class MarshalLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): +class TestMarshalLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue): def queue(self): return MarshalLifoDiskQueue.from_crawler( crawler=self.crawler, key="marshal/lifo" ) -class FifoMemoryQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): +class TestFifoMemoryQueueRequest(FifoQueueMixin, TestBaseQueue): def queue(self): return FifoMemoryQueue.from_crawler(crawler=self.crawler) -class LifoMemoryQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): +class TestLifoMemoryQueueRequest(LifoQueueMixin, TestBaseQueue): def queue(self): return LifoMemoryQueue.from_crawler(crawler=self.crawler) diff --git a/tests/test_stats.py b/tests/test_stats.py index 3d4c7e88e..537614364 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -1,4 +1,3 @@ -import unittest from datetime import datetime from unittest import mock @@ -8,8 +7,8 @@ from scrapy.statscollectors import DummyStatsCollector, StatsCollector from scrapy.utils.test import get_crawler -class CoreStatsExtensionTest(unittest.TestCase): - def setUp(self): +class TestCoreStatsExtension: + def setup_method(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("foo") @@ -24,19 +23,16 @@ class CoreStatsExtensionTest(unittest.TestCase): ext.response_received(self.spider) ext.item_dropped({}, self.spider, ZeroDivisionError()) ext.spider_closed(self.spider, "finished") - self.assertEqual( - ext.stats._stats, - { - "start_time": fixed_datetime, - "finish_time": fixed_datetime, - "item_scraped_count": 1, - "response_received_count": 1, - "item_dropped_count": 1, - "item_dropped_reasons_count/ZeroDivisionError": 1, - "finish_reason": "finished", - "elapsed_time_seconds": 0.0, - }, - ) + assert ext.stats._stats == { + "start_time": fixed_datetime, + "finish_time": fixed_datetime, + "item_scraped_count": 1, + "response_received_count": 1, + "item_dropped_count": 1, + "item_dropped_reasons_count/ZeroDivisionError": 1, + "finish_reason": "finished", + "elapsed_time_seconds": 0.0, + } def test_core_stats_dummy_stats_collector(self): self.crawler.stats = DummyStatsCollector(self.crawler) @@ -46,51 +42,51 @@ class CoreStatsExtensionTest(unittest.TestCase): ext.response_received(self.spider) ext.item_dropped({}, self.spider, ZeroDivisionError()) ext.spider_closed(self.spider, "finished") - self.assertEqual(ext.stats._stats, {}) + assert ext.stats._stats == {} -class StatsCollectorTest(unittest.TestCase): - def setUp(self): +class TestStatsCollector: + def setup_method(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("foo") def test_collector(self): stats = StatsCollector(self.crawler) - self.assertEqual(stats.get_stats(), {}) - self.assertEqual(stats.get_value("anything"), None) - self.assertEqual(stats.get_value("anything", "default"), "default") + assert stats.get_stats() == {} + assert stats.get_value("anything") is None + assert stats.get_value("anything", "default") == "default" stats.set_value("test", "value") - self.assertEqual(stats.get_stats(), {"test": "value"}) + assert stats.get_stats() == {"test": "value"} stats.set_value("test2", 23) - self.assertEqual(stats.get_stats(), {"test": "value", "test2": 23}) - self.assertEqual(stats.get_value("test2"), 23) + assert stats.get_stats() == {"test": "value", "test2": 23} + assert stats.get_value("test2") == 23 stats.inc_value("test2") - self.assertEqual(stats.get_value("test2"), 24) + assert stats.get_value("test2") == 24 stats.inc_value("test2", 6) - self.assertEqual(stats.get_value("test2"), 30) + assert stats.get_value("test2") == 30 stats.max_value("test2", 6) - self.assertEqual(stats.get_value("test2"), 30) + assert stats.get_value("test2") == 30 stats.max_value("test2", 40) - self.assertEqual(stats.get_value("test2"), 40) + assert stats.get_value("test2") == 40 stats.max_value("test3", 1) - self.assertEqual(stats.get_value("test3"), 1) + assert stats.get_value("test3") == 1 stats.min_value("test2", 60) - self.assertEqual(stats.get_value("test2"), 40) + assert stats.get_value("test2") == 40 stats.min_value("test2", 35) - self.assertEqual(stats.get_value("test2"), 35) + assert stats.get_value("test2") == 35 stats.min_value("test4", 7) - self.assertEqual(stats.get_value("test4"), 7) + assert stats.get_value("test4") == 7 def test_dummy_collector(self): stats = DummyStatsCollector(self.crawler) - self.assertEqual(stats.get_stats(), {}) - self.assertEqual(stats.get_value("anything"), None) - self.assertEqual(stats.get_value("anything", "default"), "default") + assert stats.get_stats() == {} + assert stats.get_value("anything") is None + assert stats.get_value("anything", "default") == "default" stats.set_value("test", "value") stats.inc_value("v1") stats.max_value("v2", 100) stats.min_value("v3", 100) stats.open_spider("a") stats.set_value("test", "value", spider=self.spider) - self.assertEqual(stats.get_stats(), {}) - self.assertEqual(stats.get_stats("a"), {}) + assert stats.get_stats() == {} + assert stats.get_stats("a") == {}