From d2e5486d5a0ddfa9c202e39f5af98257a230d4f3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Mar 2025 13:20:42 +0500 Subject: [PATCH 1/4] Remove the Splash recommendation. --- docs/topics/dynamic-content.rst | 35 +++++++-------------------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 801f6d06d..65270433f 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -14,7 +14,7 @@ from it. If you fail to do that, and you can nonetheless access the desired data through the :ref:`DOM ` from your web browser, see -:ref:`topics-javascript-rendering`. +:ref:`topics-headless-browsing`. .. _topics-finding-data-source: @@ -97,7 +97,7 @@ it `. You can reproduce any request with Scrapy. However, some times reproducing all necessary requests may not seem efficient in developer time. If that is your case, and crawling speed is not a major concern for you, you can alternatively -consider :ref:`JavaScript pre-rendering `. +consider :ref:`using a headless browser `. If you get the expected response `sometimes`, but not always, the issue is probably not your request, but the target server. The target server might be @@ -220,9 +220,9 @@ data from it: >>> selector.css('var[name="data"]').get() 'value' -.. _topics-javascript-rendering: +.. _topics-headless-browsing: -Pre-rendering JavaScript +Using a headless browser ======================== On webpages that fetch data from additional requests, reproducing those @@ -232,29 +232,10 @@ network transfer. However, sometimes it can be really hard to reproduce certain requests. Or you may need something that no request can give you, such as a screenshot of a -webpage as seen in a web browser. +webpage as seen in a web browser. In this case using a `headless browser`_ will +help. -In these cases use the Splash_ JavaScript-rendering service, along with -`scrapy-splash`_ for seamless integration. - -Splash returns as HTML the :ref:`DOM ` of a webpage, so that -you can parse it with :ref:`selectors `. It provides great -flexibility through configuration_ or scripting_. - -If you need something beyond what Splash offers, such as interacting with the -DOM on-the-fly from Python code instead of using a previously-written script, -or handling multiple web browser windows, you might need to -:ref:`use a headless browser ` instead. - -.. _configuration: https://splash.readthedocs.io/en/stable/api.html -.. _scripting: https://splash.readthedocs.io/en/stable/scripting-tutorial.html - -.. _topics-headless-browsing: - -Using a headless browser -======================== - -A `headless browser`_ is a special web browser that provides an API for +A headless browser is a special web browser that provides an API for automation. By installing the :ref:`asyncio reactor `, it is possible to integrate ``asyncio``-based libraries which handle headless browsers. @@ -287,7 +268,6 @@ We recommend using `scrapy-playwright`_ for a better integration. .. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29 .. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets .. _JavaScript: https://en.wikipedia.org/wiki/JavaScript -.. _Splash: https://github.com/scrapinghub/splash .. _chompjs: https://github.com/Nykakin/chompjs .. _curl: https://curl.se/ .. _headless browser: https://en.wikipedia.org/wiki/Headless_browser @@ -297,7 +277,6 @@ We recommend using `scrapy-playwright`_ for a better integration. .. _pyppeteer: https://pyppeteer.github.io/pyppeteer/ .. _pytesseract: https://github.com/madmaze/pytesseract .. _scrapy-playwright: https://github.com/scrapy-plugins/scrapy-playwright -.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash .. _tabula-py: https://github.com/chezou/tabula-py .. _wget: https://www.gnu.org/software/wget/ .. _wgrep: https://github.com/stav/wgrep From 3ded1dfe31510f00e14a70811b7c01dae8b5a641 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Mar 2025 20:25:15 +0400 Subject: [PATCH 2/4] Converting tests to plain asserts, part 4. (#6702) --- tests/test_spider.py | 344 +++++++++----------- tests/test_spiderloader/__init__.py | 113 ++++--- tests/test_spidermiddleware.py | 105 +++--- tests/test_spidermiddleware_depth.py | 16 +- tests/test_spidermiddleware_httperror.py | 98 +++--- tests/test_spidermiddleware_offsite.py | 11 +- tests/test_spidermiddleware_output_chain.py | 112 +++---- tests/test_spidermiddleware_referer.py | 31 +- tests/test_spidermiddleware_urllength.py | 12 +- tests/test_spiderstate.py | 7 +- tests/test_squeues.py | 45 +-- tests/test_squeues_request.py | 148 ++++----- tests/test_stats.py | 72 ++-- 13 files changed, 529 insertions(+), 585 deletions(-) diff --git a/tests/test_spider.py b/tests/test_spider.py index af29872a8..05f1c59d0 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -31,7 +31,7 @@ from scrapy.utils.test import get_crawler from tests import get_testdata, tests_datadir -class SpiderTest(unittest.TestCase): +class TestSpider(unittest.TestCase): spider_class = Spider def setUp(self): @@ -42,19 +42,19 @@ class SpiderTest(unittest.TestCase): def test_base_spider(self): spider = self.spider_class("example.com") - self.assertEqual(spider.name, "example.com") - self.assertEqual(spider.start_urls, []) + assert spider.name == "example.com" + assert spider.start_urls == [] # pylint: disable=use-implicit-booleaness-not-comparison def test_start_requests(self): spider = self.spider_class("example.com") start_requests = spider.start_requests() - self.assertTrue(inspect.isgenerator(start_requests)) - self.assertEqual(list(start_requests), []) + assert inspect.isgenerator(start_requests) + assert not list(start_requests) def test_spider_args(self): """``__init__`` method arguments are assigned to spider attributes""" spider = self.spider_class("example.com", foo="bar") - self.assertEqual(spider.foo, "bar") + assert spider.foo == "bar" def test_spider_without_name(self): """``__init__`` method arguments are assigned to spider attributes""" @@ -67,10 +67,10 @@ class SpiderTest(unittest.TestCase): def test_from_crawler_crawler_and_settings_population(self): crawler = get_crawler() spider = self.spider_class.from_crawler(crawler, "example.com") - self.assertTrue(hasattr(spider, "crawler")) - self.assertIs(spider.crawler, crawler) - self.assertTrue(hasattr(spider, "settings")) - self.assertIs(spider.settings, crawler.settings) + assert hasattr(spider, "crawler") + assert spider.crawler is crawler + assert hasattr(spider, "settings") + assert spider.settings is crawler.settings def test_from_crawler_init_call(self): with mock.patch.object( @@ -92,7 +92,7 @@ class SpiderTest(unittest.TestCase): crawler.signals.send_catch_log( signal=signals.spider_closed, spider=spider, reason=None ) - self.assertTrue(spider.closed_called) + assert spider.closed_called def test_update_settings(self): spider_settings = {"TEST1": "spider", "TEST2": "spider"} @@ -101,9 +101,9 @@ class SpiderTest(unittest.TestCase): settings = Settings(project_settings, priority="project") self.spider_class.update_settings(settings) - self.assertEqual(settings.get("TEST1"), "spider") - self.assertEqual(settings.get("TEST2"), "spider") - self.assertEqual(settings.get("TEST3"), "project") + assert settings.get("TEST1") == "spider" + assert settings.get("TEST2") == "spider" + assert settings.get("TEST3") == "project" @inlineCallbacks def test_settings_in_from_crawler(self): @@ -121,11 +121,11 @@ class SpiderTest(unittest.TestCase): return spider crawler = Crawler(TestSpider, project_settings) - self.assertEqual(crawler.settings.get("TEST1"), "spider") - self.assertEqual(crawler.settings.get("TEST2"), "spider") - self.assertEqual(crawler.settings.get("TEST3"), "project") + assert crawler.settings.get("TEST1") == "spider" + assert crawler.settings.get("TEST2") == "spider" + assert crawler.settings.get("TEST3") == "project" yield crawler.crawl() - self.assertEqual(crawler.settings.get("TEST1"), "spider_instance") + assert crawler.settings.get("TEST1") == "spider_instance" def test_logger(self): spider = self.spider_class("example.com") @@ -134,8 +134,8 @@ class SpiderTest(unittest.TestCase): lc.check(("example.com", "INFO", "test log msg")) record = lc.records[0] - self.assertIn("spider", record.__dict__) - self.assertIs(record.spider, spider) + assert "spider" in record.__dict__ + assert record.spider is spider def test_log(self): spider = self.spider_class("example.com") @@ -144,11 +144,11 @@ class SpiderTest(unittest.TestCase): mock_logger.log.assert_called_once_with("INFO", "test log msg") -class InitSpiderTest(SpiderTest): +class TestInitSpider(TestSpider): spider_class = InitSpider -class XMLFeedSpiderTest(SpiderTest): +class TestXMLFeedSpider(TestSpider): spider_class = XMLFeedSpider def test_register_namespace(self): @@ -180,28 +180,24 @@ class XMLFeedSpiderTest(SpiderTest): for iterator in ("iternodes", "xml"): spider = _XMLSpider("example", iterator=iterator) output = list(spider._parse(response)) - self.assertEqual(len(output), 2, iterator) - self.assertEqual( - output, - [ - { - "loc": ["http://www.example.com/Special-Offers.html"], - "updated": ["2009-08-16"], - "custom": ["fuu"], - "other": ["bar"], - }, - { - "loc": [], - "updated": ["2009-08-16"], - "other": ["foo"], - "custom": [], - }, - ], - iterator, - ) + assert len(output) == 2, iterator + assert output == [ + { + "loc": ["http://www.example.com/Special-Offers.html"], + "updated": ["2009-08-16"], + "custom": ["fuu"], + "other": ["bar"], + }, + { + "loc": [], + "updated": ["2009-08-16"], + "other": ["foo"], + "custom": [], + }, + ], iterator -class CSVFeedSpiderTest(SpiderTest): +class TestCSVFeedSpider(TestSpider): spider_class = CSVFeedSpider def test_parse_rows(self): @@ -222,7 +218,7 @@ class CSVFeedSpiderTest(SpiderTest): assert len(rows) == 4 -class CrawlSpiderTest(SpiderTest): +class TestCrawlSpider(TestSpider): test_body = b"""Page title<title> <body> <p><a href="item/12.html">Item 12</a></p> @@ -247,16 +243,13 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] def test_process_links(self): response = HtmlResponse( @@ -273,16 +266,13 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] def test_process_links_filter(self): response = HtmlResponse( @@ -302,15 +292,12 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 2) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - ], - ) + assert len(output) == 2 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + ] def test_process_links_generator(self): response = HtmlResponse( @@ -327,16 +314,13 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] def test_process_request(self): response = HtmlResponse( @@ -355,16 +339,13 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.com/somepage/item/12.html", - "http://example.com/about.html", - "http://example.com/nofollow.html", - ], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.com/somepage/item/12.html", + "http://example.com/about.html", + "http://example.com/nofollow.html", + ] def test_process_request_with_response(self): response = HtmlResponse( @@ -386,20 +367,18 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) - self.assertEqual( - [r.meta["response_class"] for r in output], - ["HtmlResponse", "HtmlResponse", "HtmlResponse"], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + assert [r.meta["response_class"] for r in output] == [ + "HtmlResponse", + "HtmlResponse", + "HtmlResponse", + ] def test_process_request_instance_method(self): response = HtmlResponse( @@ -416,16 +395,13 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"), - safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"), - safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"), - ], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"), + safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"), + safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"), + ] def test_process_request_instance_method_with_response(self): response = HtmlResponse( @@ -448,32 +424,30 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) - self.assertEqual( - [r.meta["response_class"] for r in output], - ["HtmlResponse", "HtmlResponse", "HtmlResponse"], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + assert [r.meta["response_class"] for r in output] == [ + "HtmlResponse", + "HtmlResponse", + "HtmlResponse", + ] def test_follow_links_attribute_population(self): crawler = get_crawler() spider = self.spider_class.from_crawler(crawler, "example.com") - self.assertTrue(hasattr(spider, "_follow_links")) - self.assertTrue(spider._follow_links) + assert hasattr(spider, "_follow_links") + assert spider._follow_links settings_dict = {"CRAWLSPIDER_FOLLOW_LINKS": False} crawler = get_crawler(settings_dict=settings_dict) spider = self.spider_class.from_crawler(crawler, "example.com") - self.assertTrue(hasattr(spider, "_follow_links")) - self.assertFalse(spider._follow_links) + assert hasattr(spider, "_follow_links") + assert not spider._follow_links def test_start_url(self): spider = self.spider_class("example.com") @@ -483,7 +457,7 @@ class CrawlSpiderTest(SpiderTest): list(spider.start_requests()) -class SitemapSpiderTest(SpiderTest): +class TestSitemapSpider(TestSpider): spider_class = SitemapSpider BODY = b"SITEMAP" @@ -496,7 +470,7 @@ class SitemapSpiderTest(SpiderTest): def assertSitemapBody(self, response, body): crawler = get_crawler() spider = self.spider_class.from_crawler(crawler, "example.com") - self.assertEqual(spider._get_sitemap_body(response), body) + assert spider._get_sitemap_body(response) == body def test_get_sitemap_body(self): r = XmlResponse(url="http://www.example.com/", body=self.BODY) @@ -543,15 +517,12 @@ Sitemap: /sitemap-relative-url.xml r = TextResponse(url="http://www.example.com/robots.txt", body=robots) spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - [ - "http://example.com/sitemap.xml", - "http://example.com/sitemap-product-index.xml", - "http://example.com/sitemap-uppercase.xml", - "http://www.example.com/sitemap-relative-url.xml", - ], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://example.com/sitemap.xml", + "http://example.com/sitemap-product-index.xml", + "http://example.com/sitemap-uppercase.xml", + "http://www.example.com/sitemap-relative-url.xml", + ] def test_alternate_url_locs(self): sitemap = b"""<?xml version="1.0" encoding="UTF-8"?> @@ -570,21 +541,17 @@ Sitemap: /sitemap-relative-url.xml </urlset>""" r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/english/"], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/" + ] spider.sitemap_alternate_links = True - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - [ - "http://www.example.com/english/", - "http://www.example.com/deutsch/", - "http://www.example.com/schweiz-deutsch/", - "http://www.example.com/italiano/", - ], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/", + "http://www.example.com/deutsch/", + "http://www.example.com/schweiz-deutsch/", + "http://www.example.com/italiano/", + ] def test_sitemap_filter(self): sitemap = b"""<?xml version="1.0" encoding="UTF-8"?> @@ -611,16 +578,15 @@ Sitemap: /sitemap-relative-url.xml r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/english/", "http://www.example.com/portuguese/"], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/", + "http://www.example.com/portuguese/", + ] spider = FilteredSitemapSpider("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/english/"], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/" + ] def test_sitemap_filter_with_alternate_links(self): sitemap = b"""<?xml version="1.0" encoding="UTF-8"?> @@ -649,19 +615,15 @@ Sitemap: /sitemap-relative-url.xml r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - [ - "http://www.example.com/english/article_1/", - "http://www.example.com/english/article_2/", - ], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/article_1/", + "http://www.example.com/english/article_2/", + ] spider = FilteredSitemapSpider("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/deutsch/article_1/"], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/deutsch/article_1/" + ] def test_sitemapindex_filter(self): sitemap = b"""<?xml version="1.0" encoding="UTF-8"?> @@ -689,19 +651,15 @@ Sitemap: /sitemap-relative-url.xml r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - [ - "http://www.example.com/sitemap1.xml", - "http://www.example.com/sitemap2.xml", - ], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/sitemap1.xml", + "http://www.example.com/sitemap2.xml", + ] spider = FilteredSitemapSpider("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/sitemap2.xml"], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/sitemap2.xml" + ] def test_compression_bomb_setting(self): settings = {"DOWNLOAD_MAXSIZE": 10_000_000} @@ -711,7 +669,7 @@ Sitemap: /sitemap-relative-url.xml body = body_path.read_bytes() request = Request(url="https://example.com") response = Response(url="https://example.com", body=body, request=request) - self.assertIsNone(spider._get_sitemap_body(response)) + assert spider._get_sitemap_body(response) is None def test_compression_bomb_spider_attr(self): class DownloadMaxSizeSpider(self.spider_class): @@ -723,7 +681,7 @@ Sitemap: /sitemap-relative-url.xml body = body_path.read_bytes() request = Request(url="https://example.com") response = Response(url="https://example.com", body=body, request=request) - self.assertIsNone(spider._get_sitemap_body(response)) + assert spider._get_sitemap_body(response) is None def test_compression_bomb_request_meta(self): crawler = get_crawler() @@ -734,7 +692,7 @@ Sitemap: /sitemap-relative-url.xml url="https://example.com", meta={"download_maxsize": 10_000_000} ) response = Response(url="https://example.com", body=body, request=request) - self.assertIsNone(spider._get_sitemap_body(response)) + assert spider._get_sitemap_body(response) is None def test_download_warnsize_setting(self): settings = {"DOWNLOAD_WARNSIZE": 10_000_000} @@ -814,13 +772,13 @@ Sitemap: /sitemap-relative-url.xml ) -class DeprecationTest(unittest.TestCase): +class TestDeprecation: def test_crawl_spider(self): assert issubclass(CrawlSpider, Spider) assert isinstance(CrawlSpider(name="foo"), Spider) -class NoParseMethodSpiderTest(unittest.TestCase): +class TestNoParseMethodSpider: spider_class = Spider def test_undefined_parse_method(self): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index b103e9ed0..476487a04 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -8,7 +8,6 @@ from tempfile import mkdtemp from unittest import mock import pytest -from twisted.trial import unittest from zope.interface.verify import verifyObject # ugly hack to avoid cyclic imports of scrapy.spiders when running this test @@ -28,8 +27,8 @@ def _copytree(source: Path, target: Path): shutil.copytree(source, target) -class SpiderLoaderTest(unittest.TestCase): - def setUp(self): +class TestSpiderLoader: + def setup_method(self): orig_spiders_dir = module_dir / "test_spiders" self.tmpdir = Path(tempfile.mkdtemp()) self.spiders_dir = self.tmpdir / "test_spiders_xxx" @@ -38,7 +37,7 @@ class SpiderLoaderTest(unittest.TestCase): settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) self.spider_loader = SpiderLoader.from_settings(settings) - def tearDown(self): + def teardown_method(self): del self.spider_loader del sys.modules["test_spiders_xxx"] sys.path.remove(str(self.tmpdir)) @@ -47,37 +46,35 @@ class SpiderLoaderTest(unittest.TestCase): verifyObject(ISpiderLoader, self.spider_loader) def test_list(self): - self.assertEqual( - set(self.spider_loader.list()), {"spider1", "spider2", "spider3", "spider4"} - ) + assert set(self.spider_loader.list()) == { + "spider1", + "spider2", + "spider3", + "spider4", + } def test_load(self): spider1 = self.spider_loader.load("spider1") - self.assertEqual(spider1.__name__, "Spider1") + assert spider1.__name__ == "Spider1" def test_find_by_request(self): - self.assertEqual( - self.spider_loader.find_by_request(Request("http://scrapy1.org/test")), - ["spider1"], - ) - self.assertEqual( - self.spider_loader.find_by_request(Request("http://scrapy2.org/test")), - ["spider2"], - ) - self.assertEqual( - set(self.spider_loader.find_by_request(Request("http://scrapy3.org/test"))), - {"spider1", "spider2"}, - ) - self.assertEqual( - self.spider_loader.find_by_request(Request("http://scrapy999.org/test")), [] - ) - self.assertEqual( - self.spider_loader.find_by_request(Request("http://spider3.com")), [] - ) - self.assertEqual( - self.spider_loader.find_by_request(Request("http://spider3.com/onlythis")), - ["spider3"], + assert self.spider_loader.find_by_request( + Request("http://scrapy1.org/test") + ) == ["spider1"] + assert self.spider_loader.find_by_request( + Request("http://scrapy2.org/test") + ) == ["spider2"] + assert set( + self.spider_loader.find_by_request(Request("http://scrapy3.org/test")) + ) == {"spider1", "spider2"} + assert ( + self.spider_loader.find_by_request(Request("http://scrapy999.org/test")) + == [] ) + assert self.spider_loader.find_by_request(Request("http://spider3.com")) == [] + assert self.spider_loader.find_by_request( + Request("http://spider3.com/onlythis") + ) == ["spider3"] def test_load_spider_module(self): module = "tests.test_spiderloader.test_spiders.spider1" @@ -113,9 +110,9 @@ class SpiderLoaderTest(unittest.TestCase): runner = CrawlerRunner({"ADDONS": {SpiderModuleAddon: 1}}) crawler = runner.create_crawler("spider_from_addon") - self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) - self.assertEqual(crawler.spidercls.name, "spider_from_addon") - self.assertTrue(len(crawler.settings["SPIDER_MODULES"]) == 1) + assert issubclass(crawler.spidercls, scrapy.Spider) + assert crawler.spidercls.name == "spider_from_addon" + assert len(crawler.settings["SPIDER_MODULES"]) == 1 def test_crawler_runner_loading(self): module = "tests.test_spiderloader.test_spiders.spider1" @@ -129,8 +126,8 @@ class SpiderLoaderTest(unittest.TestCase): runner.create_crawler("spider2") crawler = runner.create_crawler("spider1") - self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) - self.assertEqual(crawler.spidercls.name, "spider1") + assert issubclass(crawler.spidercls, scrapy.Spider) + assert crawler.spidercls.name == "spider1" def test_bad_spider_modules_exception(self): module = "tests.test_spiderloader.test_spiders.doesnotexist" @@ -150,10 +147,10 @@ class SpiderLoaderTest(unittest.TestCase): # at least until all six versions we can import (including botocore.vendored.six) # are updated to 1.16.0+ w.pop(0) - self.assertIn("Could not load spiders from module", str(w[0].message)) + assert "Could not load spiders from module" in str(w[0].message) spiders = spider_loader.list() - self.assertEqual(spiders, []) + assert not spiders def test_syntax_error_exception(self): module = "tests.test_spiderloader.test_spiders.spider1" @@ -179,14 +176,14 @@ class SpiderLoaderTest(unittest.TestCase): # at least until all six versions we can import (including botocore.vendored.six) # are updated to 1.16.0+ w.pop(0) - self.assertIn("Could not load spiders from module", str(w[0].message)) + assert "Could not load spiders from module" in str(w[0].message) spiders = spider_loader.list() - self.assertEqual(spiders, []) + assert not spiders -class DuplicateSpiderNameLoaderTest(unittest.TestCase): - def setUp(self): +class TestDuplicateSpiderNameLoader: + def setup_method(self): orig_spiders_dir = module_dir / "test_spiders" self.tmpdir = Path(mkdtemp()) self.spiders_dir = self.tmpdir / "test_spiders_xxx" @@ -194,7 +191,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): sys.path.append(str(self.tmpdir)) self.settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) - def tearDown(self): + def teardown_method(self): del sys.modules["test_spiders_xxx"] sys.path.remove(str(self.tmpdir)) @@ -208,18 +205,18 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) - self.assertEqual(len(w), 1) + assert len(w) == 1 msg = str(w[0].message) - self.assertIn("several spiders with the same name", msg) - self.assertIn("'spider3'", msg) - self.assertTrue(msg.count("'spider3'") == 2) + assert "several spiders with the same name" in msg + assert "'spider3'" in msg + assert msg.count("'spider3'") == 2 - self.assertNotIn("'spider1'", msg) - self.assertNotIn("'spider2'", msg) - self.assertNotIn("'spider4'", msg) + assert "'spider1'" not in msg + assert "'spider2'" not in msg + assert "'spider4'" not in msg spiders = set(spider_loader.list()) - self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"}) + assert spiders == {"spider1", "spider2", "spider3", "spider4"} def test_multiple_dupename_warning(self): # copy 2 spider modules so as to have duplicate spider name @@ -236,17 +233,17 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) - self.assertEqual(len(w), 1) + assert len(w) == 1 msg = str(w[0].message) - self.assertIn("several spiders with the same name", msg) - self.assertIn("'spider1'", msg) - self.assertTrue(msg.count("'spider1'") == 2) + assert "several spiders with the same name" in msg + assert "'spider1'" in msg + assert msg.count("'spider1'") == 2 - self.assertIn("'spider2'", msg) - self.assertTrue(msg.count("'spider2'") == 2) + assert "'spider2'" in msg + assert msg.count("'spider2'") == 2 - self.assertNotIn("'spider3'", msg) - self.assertNotIn("'spider4'", msg) + assert "'spider3'" not in msg + assert "'spider4'" not in msg spiders = set(spider_loader.list()) - self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"}) + assert spiders == {"spider1", "spider2", "spider3", "spider4"} diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index a9f3876bb..ddc9b5206 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -18,7 +18,7 @@ from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.test import get_crawler -class SpiderMiddlewareTestCase(TestCase): +class TestSpiderMiddleware(TestCase): def setUp(self): self.request = Request("http://example.com/index.html") self.response = Response(self.request.url, request=self.request) @@ -41,7 +41,7 @@ class SpiderMiddlewareTestCase(TestCase): return results[0] -class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase): +class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_input method""" def test_invalid_process_spider_input(self): @@ -51,11 +51,11 @@ class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase): self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware()) result = self._scrape_response() - self.assertIsInstance(result, Failure) - self.assertIsInstance(result.value, _InvalidOutput) + assert isinstance(result, Failure) + assert isinstance(result.value, _InvalidOutput) -class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase): +class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_output method""" def test_invalid_process_spider_output(self): @@ -65,11 +65,11 @@ class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase): self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware()) result = self._scrape_response() - self.assertIsInstance(result, Failure) - self.assertIsInstance(result.value, _InvalidOutput) + assert isinstance(result, Failure) + assert isinstance(result.value, _InvalidOutput) -class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase): +class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_exception method""" def test_invalid_process_spider_exception(self): @@ -84,11 +84,11 @@ class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase): self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) result = self._scrape_response() - self.assertIsInstance(result, Failure) - self.assertIsInstance(result.value, _InvalidOutput) + assert isinstance(result, Failure) + assert isinstance(result.value, _InvalidOutput) -class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): +class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): """Re raise the exception by returning None""" def test_process_spider_exception_return_none(self): @@ -103,11 +103,11 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) result = self._scrape_response() - self.assertIsInstance(result, Failure) - self.assertIsInstance(result.value, ZeroDivisionError) + assert isinstance(result, Failure) + assert isinstance(result.value, ZeroDivisionError) -class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): +class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): """Helpers for testing sync, async and mixed middlewares. Should work for process_spider_output and, when it's supported, process_start_requests. @@ -148,14 +148,13 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): result = yield self._get_middleware_result( *mw_classes, start_index=start_index ) - self.assertIsInstance(result, Iterable) + assert isinstance(result, Iterable) result_list = list(result) - self.assertEqual(len(result_list), self.RESULT_COUNT) - self.assertIsInstance(result_list[0], self.ITEM_TYPE) - self.assertEqual("downgraded to a non-async" in str(log), downgrade) - self.assertEqual( - "doesn't support asynchronous spider output" in str(log), - ProcessSpiderOutputSimpleMiddleware in mw_classes, + assert len(result_list) == self.RESULT_COUNT + assert isinstance(result_list[0], self.ITEM_TYPE) + assert ("downgraded to a non-async" in str(log)) == downgrade + assert ("doesn't support asynchronous spider output" in str(log)) == ( + ProcessSpiderOutputSimpleMiddleware in mw_classes ) @defer.inlineCallbacks @@ -166,11 +165,11 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): result = yield self._get_middleware_result( *mw_classes, start_index=start_index ) - self.assertIsInstance(result, AsyncIterator) + assert isinstance(result, AsyncIterator) result_list = yield deferred_from_coro(collect_asyncgen(result)) - self.assertEqual(len(result_list), self.RESULT_COUNT) - self.assertIsInstance(result_list[0], self.ITEM_TYPE) - self.assertEqual("downgraded to a non-async" in str(log), downgrade) + assert len(result_list) == self.RESULT_COUNT + assert isinstance(result_list[0], self.ITEM_TYPE) + assert ("downgraded to a non-async" in str(log)) == downgrade class ProcessSpiderOutputSimpleMiddleware: @@ -212,7 +211,7 @@ class ProcessSpiderExceptionAsyncIterableMiddleware: yield {"foo": 3} -class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase): +class TestProcessSpiderOutputSimple(TestBaseAsyncSpiderMiddleware): """process_spider_output tests for simple callbacks""" ITEM_TYPE = dict @@ -257,7 +256,7 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase): return self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN) -class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple): +class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple): """process_spider_output tests for async generator callbacks""" async def _scrape_func(self, *args, **kwargs): @@ -297,7 +296,7 @@ class ProcessSpiderOutputCoroutineMiddleware: return result -class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): +class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware): @defer.inlineCallbacks def test_non_iterable(self): with pytest.raises( @@ -324,7 +323,7 @@ class ProcessStartRequestsSimpleMiddleware: yield from start_requests -class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase): +class TestProcessStartRequestsSimple(TestBaseAsyncSpiderMiddleware): """process_start_requests tests for simple start_requests""" ITEM_TYPE = (Request, dict) @@ -373,67 +372,65 @@ class UniversalMiddlewareBothAsync: yield -class UniversalMiddlewareManagerTest(TestCase): - def setUp(self): +class TestUniversalMiddlewareManager: + def setup_method(self): self.mwman = SpiderMiddlewareManager() def test_simple_mw(self): mw = ProcessSpiderOutputSimpleMiddleware() self.mwman._add_middleware(mw) - self.assertEqual( - self.mwman.methods["process_spider_output"][0], mw.process_spider_output + assert ( + self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable ) def test_async_mw(self): mw = ProcessSpiderOutputAsyncGenMiddleware() self.mwman._add_middleware(mw) - self.assertEqual( - self.mwman.methods["process_spider_output"][0], mw.process_spider_output + assert ( + self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable ) def test_universal_mw(self): mw = ProcessSpiderOutputUniversalMiddleware() self.mwman._add_middleware(mw) - self.assertEqual( - self.mwman.methods["process_spider_output"][0], - (mw.process_spider_output, mw.process_spider_output_async), + assert self.mwman.methods["process_spider_output"][0] == ( + mw.process_spider_output, + mw.process_spider_output_async, ) def test_universal_mw_no_sync(self): with LogCapture() as log: self.mwman._add_middleware(UniversalMiddlewareNoSync()) - self.assertIn( + assert ( "UniversalMiddlewareNoSync has process_spider_output_async" - " without process_spider_output", - str(log), + " without process_spider_output" in str(log) ) - self.assertEqual(self.mwman.methods["process_spider_output"][0], None) + assert self.mwman.methods["process_spider_output"][0] is None def test_universal_mw_both_sync(self): mw = UniversalMiddlewareBothSync() with LogCapture() as log: self.mwman._add_middleware(mw) - self.assertIn( + assert ( "UniversalMiddlewareBothSync.process_spider_output_async " - "is not an async generator function", - str(log), + "is not an async generator function" in str(log) ) - self.assertEqual( - self.mwman.methods["process_spider_output"][0], mw.process_spider_output + assert ( + self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable ) def test_universal_mw_both_async(self): with LogCapture() as log: self.mwman._add_middleware(UniversalMiddlewareBothAsync()) - self.assertIn( + assert ( "UniversalMiddlewareBothAsync.process_spider_output " - "is an async generator function while process_spider_output_async exists", - str(log), + "is an async generator function while process_spider_output_async exists" + in str(log) ) - self.assertEqual(self.mwman.methods["process_spider_output"][0], None) + assert self.mwman.methods["process_spider_output"][0] is None -class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase): +class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware): ITEM_TYPE = dict MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware @@ -474,7 +471,7 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase): return self._test_simple_base(self.MW_UNIVERSAL) -class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest): +class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple): async def _scrape_func(self, *args, **kwargs): for item in super()._scrape_func(): yield item @@ -503,7 +500,7 @@ class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest): return self._test_asyncgen_base(self.MW_UNIVERSAL) -class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase): +class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): ITEM_TYPE = dict MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index e359d9cfc..dfcc141c3 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -1,5 +1,3 @@ -from unittest import TestCase - from scrapy.http import Request, Response from scrapy.spidermiddlewares.depth import DepthMiddleware from scrapy.spiders import Spider @@ -7,8 +5,8 @@ from scrapy.statscollectors import StatsCollector from scrapy.utils.test import get_crawler -class TestDepthMiddleware(TestCase): - def setUp(self): +class TestDepthMiddleware: + def setup_method(self): crawler = get_crawler(Spider) self.spider = crawler._create_spider("scrapytest.org") @@ -24,18 +22,18 @@ class TestDepthMiddleware(TestCase): result = [Request("http://scrapytest.org")] out = list(self.mw.process_spider_output(resp, result, self.spider)) - self.assertEqual(out, result) + assert out == result rdc = self.stats.get_value("request_depth_count/1", spider=self.spider) - self.assertEqual(rdc, 1) + assert rdc == 1 req.meta["depth"] = 1 out2 = list(self.mw.process_spider_output(resp, result, self.spider)) - self.assertEqual(out2, []) + assert not out2 rdm = self.stats.get_value("request_depth_max", spider=self.spider) - self.assertEqual(rdm, 1) + assert rdm == 1 - def tearDown(self): + def teardown_method(self): self.stats.close_spider(self.spider, "") diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index f9eb93d6b..e306579fa 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,10 +1,9 @@ import logging -from unittest import TestCase import pytest from testfixtures import LogCapture from twisted.internet import defer -from twisted.trial.unittest import TestCase as TrialTestCase +from twisted.trial.unittest import TestCase from scrapy.http import Request, Response from scrapy.settings import Settings @@ -59,8 +58,8 @@ def _responses(request, status_codes): return responses -class TestHttpErrorMiddleware(TestCase): - def setUp(self): +class TestHttpErrorMiddleware: + def setup_method(self): crawler = get_crawler(Spider) self.spider = Spider.from_crawler(crawler, name="foo") self.mw = HttpErrorMiddleware(Settings({})) @@ -68,19 +67,20 @@ class TestHttpErrorMiddleware(TestCase): self.res200, self.res404 = _responses(self.req, [200, 404]) def test_process_spider_input(self): - self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) + assert self.mw.process_spider_input(self.res200, self.spider) is None with pytest.raises(HttpError): self.mw.process_spider_input(self.res404, self.spider) def test_process_spider_exception(self): - self.assertEqual( - [], + assert ( self.mw.process_spider_exception( self.res404, HttpError(self.res404), self.spider - ), + ) + == [] ) - self.assertIsNone( + assert ( self.mw.process_spider_exception(self.res404, Exception(), self.spider) + is None ) def test_handle_httpstatus_list(self): @@ -88,26 +88,26 @@ class TestHttpErrorMiddleware(TestCase): res.request = Request( "http://scrapytest.org", meta={"handle_httpstatus_list": [404]} ) - self.assertIsNone(self.mw.process_spider_input(res, self.spider)) + assert self.mw.process_spider_input(res, self.spider) is None self.spider.handle_httpstatus_list = [404] - self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) + assert self.mw.process_spider_input(self.res404, self.spider) is None -class TestHttpErrorMiddlewareSettings(TestCase): +class TestHttpErrorMiddlewareSettings: """Similar test, but with settings""" - def setUp(self): + def setup_method(self): self.spider = Spider("foo") self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)})) self.req = Request("http://scrapytest.org") self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) + assert self.mw.process_spider_input(self.res200, self.spider) is None with pytest.raises(HttpError): self.mw.process_spider_input(self.res404, self.spider) - self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider)) + assert self.mw.process_spider_input(self.res402, self.spider) is None def test_meta_overrides_settings(self): request = Request( @@ -118,27 +118,27 @@ class TestHttpErrorMiddlewareSettings(TestCase): res402 = self.res402.copy() res402.request = request - self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) + assert self.mw.process_spider_input(res404, self.spider) is None with pytest.raises(HttpError): self.mw.process_spider_input(res402, self.spider) def test_spider_override_settings(self): self.spider.handle_httpstatus_list = [404] - self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) + assert self.mw.process_spider_input(self.res404, self.spider) is None with pytest.raises(HttpError): self.mw.process_spider_input(self.res402, self.spider) -class TestHttpErrorMiddlewareHandleAll(TestCase): - def setUp(self): +class TestHttpErrorMiddlewareHandleAll: + def setup_method(self): self.spider = Spider("foo") self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True})) self.req = Request("http://scrapytest.org") self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) - self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) + assert self.mw.process_spider_input(self.res200, self.spider) is None + assert self.mw.process_spider_input(self.res404, self.spider) is None def test_meta_overrides_settings(self): request = Request( @@ -149,7 +149,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): res402 = self.res402.copy() res402.request = request - self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) + assert self.mw.process_spider_input(res404, self.spider) is None with pytest.raises(HttpError): self.mw.process_spider_input(res402, self.spider) @@ -169,10 +169,10 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): with pytest.raises(HttpError): mw.process_spider_input(res404, self.spider) - self.assertIsNone(mw.process_spider_input(res402, self.spider)) + assert mw.process_spider_input(res402, self.spider) is None -class TestHttpErrorMiddlewareIntegrational(TrialTestCase): +class TestHttpErrorMiddlewareIntegrational(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -187,28 +187,28 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): crawler = get_crawler(_HttpErrorSpider) yield crawler.crawl(mockserver=self.mockserver) assert not crawler.spider.skipped, crawler.spider.skipped - self.assertEqual(crawler.spider.parsed, {"200"}) - self.assertEqual(crawler.spider.failed, {"404", "402", "500"}) + assert crawler.spider.parsed == {"200"} + assert crawler.spider.failed == {"404", "402", "500"} get_value = crawler.stats.get_value - self.assertEqual(get_value("httperror/response_ignored_count"), 3) - self.assertEqual(get_value("httperror/response_ignored_status_count/404"), 1) - self.assertEqual(get_value("httperror/response_ignored_status_count/402"), 1) - self.assertEqual(get_value("httperror/response_ignored_status_count/500"), 1) + assert get_value("httperror/response_ignored_count") == 3 + assert get_value("httperror/response_ignored_status_count/404") == 1 + assert get_value("httperror/response_ignored_status_count/402") == 1 + assert get_value("httperror/response_ignored_status_count/500") == 1 @defer.inlineCallbacks def test_logging(self): crawler = get_crawler(_HttpErrorSpider) with LogCapture() as log: yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402}) - self.assertEqual(crawler.spider.parsed, {"200", "402"}) - self.assertEqual(crawler.spider.skipped, {"402"}) - self.assertEqual(crawler.spider.failed, {"404", "500"}) + assert crawler.spider.parsed == {"200", "402"} + assert crawler.spider.skipped == {"402"} + assert crawler.spider.failed == {"404", "500"} - self.assertIn("Ignoring response <404", str(log)) - self.assertIn("Ignoring response <500", str(log)) - self.assertNotIn("Ignoring response <200", str(log)) - self.assertNotIn("Ignoring response <402", str(log)) + assert "Ignoring response <404" in str(log) + assert "Ignoring response <500" in str(log) + assert "Ignoring response <200" not in str(log) + assert "Ignoring response <402" not in str(log) @defer.inlineCallbacks def test_logging_level(self): @@ -216,22 +216,22 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): crawler = get_crawler(_HttpErrorSpider) with LogCapture(level=logging.INFO) as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(crawler.spider.parsed, {"200"}) - self.assertEqual(crawler.spider.failed, {"404", "402", "500"}) + assert crawler.spider.parsed == {"200"} + assert crawler.spider.failed == {"404", "402", "500"} - self.assertIn("Ignoring response <402", str(log)) - self.assertIn("Ignoring response <404", str(log)) - self.assertIn("Ignoring response <500", str(log)) - self.assertNotIn("Ignoring response <200", str(log)) + assert "Ignoring response <402" in str(log) + assert "Ignoring response <404" in str(log) + assert "Ignoring response <500" in str(log) + assert "Ignoring response <200" not in str(log) # with level WARNING, we shouldn't capture anything from HttpError crawler = get_crawler(_HttpErrorSpider) with LogCapture(level=logging.WARNING) as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(crawler.spider.parsed, {"200"}) - self.assertEqual(crawler.spider.failed, {"404", "402", "500"}) + assert crawler.spider.parsed == {"200"} + assert crawler.spider.failed == {"404", "402", "500"} - self.assertNotIn("Ignoring response <402", str(log)) - self.assertNotIn("Ignoring response <404", str(log)) - self.assertNotIn("Ignoring response <500", str(log)) - self.assertNotIn("Ignoring response <200", str(log)) + assert "Ignoring response <402" not in str(log) + assert "Ignoring response <404" not in str(log) + assert "Ignoring response <500" not in str(log) + assert "Ignoring response <200" not in str(log) diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 906928e01..f4563a0a4 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -1,5 +1,4 @@ import warnings -from unittest import TestCase from urllib.parse import urlparse from scrapy.http import Request, Response @@ -8,8 +7,8 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class TestOffsiteMiddleware(TestCase): - def setUp(self): +class TestOffsiteMiddleware: + def setup_method(self): crawler = get_crawler(Spider) self.spider = crawler._create_spider(**self._get_spiderargs()) self.mw = OffsiteMiddleware.from_crawler(crawler) @@ -46,7 +45,7 @@ class TestOffsiteMiddleware(TestCase): reqs = onsite_reqs + offsite_reqs out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEqual(out, onsite_reqs) + assert out == onsite_reqs class TestOffsiteMiddleware2(TestOffsiteMiddleware): @@ -57,7 +56,7 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware): res = Response("http://scrapytest.org") reqs = [Request("http://a.com/b.html"), Request("http://b.com/1")] out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEqual(out, reqs) + assert out == reqs class TestOffsiteMiddleware3(TestOffsiteMiddleware2): @@ -77,7 +76,7 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3): res = Response("http://scrapytest.org") reqs = [Request("http://scrapytest.org/1")] out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEqual(out, reqs) + assert out == reqs class TestOffsiteMiddleware5(TestOffsiteMiddleware4): diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index e51957497..6e26a85ea 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -324,9 +324,9 @@ class TestSpiderMiddleware(TestCase): was enqueued from the recovery middleware) """ log = yield self.crawl_log(RecoverySpider) - self.assertIn("Middleware: TabError exception caught", str(log)) - self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1) - self.assertIn("'item_scraped_count': 3", str(log)) + assert "Middleware: TabError exception caught" in str(log) + assert str(log).count("Middleware: TabError exception caught") == 1 + assert "'item_scraped_count': 3" in str(log) @defer.inlineCallbacks def test_recovery_asyncgen(self): @@ -334,9 +334,9 @@ class TestSpiderMiddleware(TestCase): Same as test_recovery but with an async callback. """ log = yield self.crawl_log(RecoveryAsyncGenSpider) - self.assertIn("Middleware: TabError exception caught", str(log)) - self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1) - self.assertIn("'item_scraped_count': 3", str(log)) + assert "Middleware: TabError exception caught" in str(log) + assert str(log).count("Middleware: TabError exception caught") == 1 + assert "'item_scraped_count': 3" in str(log) @defer.inlineCallbacks def test_process_spider_input_without_errback(self): @@ -345,8 +345,8 @@ class TestSpiderMiddleware(TestCase): process_spider_exception chain from the start if the Request has no errback """ log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithoutErrback) - self.assertIn("Middleware: will raise IndexError", str(log1)) - self.assertIn("Middleware: IndexError exception caught", str(log1)) + assert "Middleware: will raise IndexError" in str(log1) + assert "Middleware: IndexError exception caught" in str(log1) @defer.inlineCallbacks def test_process_spider_input_with_errback(self): @@ -355,12 +355,12 @@ class TestSpiderMiddleware(TestCase): process_spider_exception chain if the Request has an errback """ log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithErrback) - self.assertNotIn("Middleware: IndexError exception caught", str(log1)) - self.assertIn("Middleware: will raise IndexError", str(log1)) - self.assertIn("Got a Failure on the Request errback", str(log1)) - self.assertIn("{'from': 'errback'}", str(log1)) - self.assertNotIn("{'from': 'callback'}", str(log1)) - self.assertIn("'item_scraped_count': 1", str(log1)) + assert "Middleware: IndexError exception caught" not in str(log1) + assert "Middleware: will raise IndexError" in str(log1) + assert "Got a Failure on the Request errback" in str(log1) + assert "{'from': 'errback'}" in str(log1) + assert "{'from': 'callback'}" not in str(log1) + assert "'item_scraped_count': 1" in str(log1) @defer.inlineCallbacks def test_generator_callback(self): @@ -370,8 +370,8 @@ class TestSpiderMiddleware(TestCase): exception is raised should be processed normally. """ log2 = yield self.crawl_log(GeneratorCallbackSpider) - self.assertIn("Middleware: ImportError exception caught", str(log2)) - self.assertIn("'item_scraped_count': 2", str(log2)) + assert "Middleware: ImportError exception caught" in str(log2) + assert "'item_scraped_count': 2" in str(log2) @defer.inlineCallbacks def test_async_generator_callback(self): @@ -379,8 +379,8 @@ class TestSpiderMiddleware(TestCase): Same as test_generator_callback but with an async callback. """ log2 = yield self.crawl_log(AsyncGeneratorCallbackSpider) - self.assertIn("Middleware: ImportError exception caught", str(log2)) - self.assertIn("'item_scraped_count': 2", str(log2)) + assert "Middleware: ImportError exception caught" in str(log2) + assert "'item_scraped_count': 2" in str(log2) @defer.inlineCallbacks def test_generator_callback_right_after_callback(self): @@ -389,8 +389,8 @@ class TestSpiderMiddleware(TestCase): even if the middleware is placed right after the spider """ log21 = yield self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider) - self.assertIn("Middleware: ImportError exception caught", str(log21)) - self.assertIn("'item_scraped_count': 2", str(log21)) + assert "Middleware: ImportError exception caught" in str(log21) + assert "'item_scraped_count': 2" in str(log21) @defer.inlineCallbacks def test_not_a_generator_callback(self): @@ -399,8 +399,8 @@ class TestSpiderMiddleware(TestCase): be caught by the process_spider_exception chain. No items should be processed. """ log3 = yield self.crawl_log(NotGeneratorCallbackSpider) - self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3)) - self.assertNotIn("item_scraped_count", str(log3)) + assert "Middleware: ZeroDivisionError exception caught" in str(log3) + assert "item_scraped_count" not in str(log3) @defer.inlineCallbacks def test_not_a_generator_callback_right_after_callback(self): @@ -411,8 +411,8 @@ class TestSpiderMiddleware(TestCase): log31 = yield self.crawl_log( NotGeneratorCallbackSpiderMiddlewareRightAfterSpider ) - self.assertIn("Middleware: ZeroDivisionError exception caught", str(log31)) - self.assertNotIn("item_scraped_count", str(log31)) + assert "Middleware: ZeroDivisionError exception caught" in str(log31) + assert "item_scraped_count" not in str(log31) @defer.inlineCallbacks def test_generator_output_chain(self): @@ -425,22 +425,22 @@ class TestSpiderMiddleware(TestCase): process_spider_exception chain) """ log4 = yield self.crawl_log(GeneratorOutputChainSpider) - self.assertIn("'item_scraped_count': 2", str(log4)) - self.assertIn( - "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught", - str(log4), + assert "'item_scraped_count': 2" in str(log4) + assert ( + "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught" + in str(log4) ) - self.assertIn( - "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught", - str(log4), + assert ( + "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught" + in str(log4) ) - self.assertNotIn( - "GeneratorFailMiddleware.process_spider_exception: LookupError caught", - str(log4), + assert ( + "GeneratorFailMiddleware.process_spider_exception: LookupError caught" + not in str(log4) ) - self.assertNotIn( - "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught", - str(log4), + assert ( + "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught" + not in str(log4) ) item_from_callback = { "processed": [ @@ -457,9 +457,9 @@ class TestSpiderMiddleware(TestCase): "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output", ] } - self.assertIn(str(item_from_callback), str(log4)) - self.assertIn(str(item_recovered), str(log4)) - self.assertNotIn("parse-second-item", str(log4)) + assert str(item_from_callback) in str(log4) + assert str(item_recovered) in str(log4) + assert "parse-second-item" not in str(log4) @defer.inlineCallbacks def test_not_a_generator_output_chain(self): @@ -472,22 +472,22 @@ class TestSpiderMiddleware(TestCase): from the spider callback are lost) """ log5 = yield self.crawl_log(NotGeneratorOutputChainSpider) - self.assertIn("'item_scraped_count': 1", str(log5)) - self.assertIn( - "GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught", - str(log5), + assert "'item_scraped_count': 1" in str(log5) + assert ( + "GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught" + in str(log5) ) - self.assertIn( - "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught", - str(log5), + assert ( + "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught" + in str(log5) ) - self.assertNotIn( - "GeneratorFailMiddleware.process_spider_exception: ReferenceError caught", - str(log5), + assert ( + "GeneratorFailMiddleware.process_spider_exception: ReferenceError caught" + not in str(log5) ) - self.assertNotIn( - "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught", - str(log5), + assert ( + "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught" + not in str(log5) ) item_recovered = { "processed": [ @@ -495,6 +495,6 @@ class TestSpiderMiddleware(TestCase): "NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output", ] } - self.assertIn(str(item_recovered), str(log5)) - self.assertNotIn("parse-first-item", str(log5)) - self.assertNotIn("parse-second-item", str(log5)) + assert str(item_recovered) in str(log5) + assert "parse-first-item" not in str(log5) + assert "parse-second-item" not in str(log5) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 01a87c645..300a40c13 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -2,7 +2,6 @@ from __future__ import annotations import warnings from typing import Any -from unittest import TestCase from urllib.parse import urlparse import pytest @@ -35,7 +34,7 @@ from scrapy.spidermiddlewares.referer import ( from scrapy.spiders import Spider -class TestRefererMiddleware(TestCase): +class TestRefererMiddleware: req_meta: dict[str, Any] = {} resp_headers: dict[str, str] = {} settings: dict[str, Any] = {} @@ -43,7 +42,7 @@ class TestRefererMiddleware(TestCase): ("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"), ] - def setUp(self): + def setup_method(self): self.spider = Spider("foo") settings = Settings(self.settings) self.mw = RefererMiddleware(settings) @@ -59,7 +58,7 @@ class TestRefererMiddleware(TestCase): response = self.get_response(origin) request = self.get_request(target) out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEqual(out[0].headers.get("Referer"), referrer) + assert out[0].headers.get("Referer") == referrer class MixinDefault: @@ -773,7 +772,7 @@ class TestRequestMetaPrecedence003(MixinUnsafeUrl, TestRefererMiddleware): req_meta = {"referrer_policy": POLICY_UNSAFE_URL} -class TestRequestMetaSettingFallback(TestCase): +class TestRequestMetaSettingFallback: params = [ ( # When an unknown policy is referenced in Request.meta @@ -844,14 +843,14 @@ class TestRequestMetaSettingFallback(TestCase): with warnings.catch_warnings(record=True) as w: policy = mw.policy(response, request) - self.assertIsInstance(policy, policy_class) + assert isinstance(policy, policy_class) if check_warning: - self.assertEqual(len(w), 1) - self.assertEqual(w[0].category, RuntimeWarning, w[0].message) + assert len(w) == 1 + assert w[0].category is RuntimeWarning, w[0].message -class TestSettingsPolicyByName(TestCase): +class TestSettingsPolicyByName: def test_valid_name(self): for s, p in [ (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), @@ -866,7 +865,7 @@ class TestSettingsPolicyByName(TestCase): ]: settings = Settings({"REFERRER_POLICY": s}) mw = RefererMiddleware(settings) - self.assertEqual(mw.default_policy, p) + assert mw.default_policy == p def test_valid_name_casevariants(self): for s, p in [ @@ -882,7 +881,7 @@ class TestSettingsPolicyByName(TestCase): ]: settings = Settings({"REFERRER_POLICY": s.upper()}) mw = RefererMiddleware(settings) - self.assertEqual(mw.default_policy, p) + assert mw.default_policy == p def test_invalid_name(self): settings = Settings({"REFERRER_POLICY": "some-custom-unknown-policy"}) @@ -902,7 +901,7 @@ class TestSettingsPolicyByName(TestCase): } ) mw1 = RefererMiddleware(settings1) - self.assertEqual(mw1.default_policy, StrictOriginWhenCrossOriginPolicy) + assert mw1.default_policy == StrictOriginWhenCrossOriginPolicy # test parsing with space(s) after the comma settings2 = Settings( @@ -915,7 +914,7 @@ class TestSettingsPolicyByName(TestCase): } ) mw2 = RefererMiddleware(settings2) - self.assertEqual(mw2.default_policy, UnsafeUrlPolicy) + assert mw2.default_policy == UnsafeUrlPolicy def test_multiple_policy_tokens_all_invalid(self): settings = Settings( @@ -1003,7 +1002,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware): ), ] - def setUp(self): + def setup_method(self): self.spider = Spider("foo") settings = Settings(self.settings) self.referrermw = RefererMiddleware(settings) @@ -1023,7 +1022,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware): out = list( self.referrermw.process_spider_output(response, [request], self.spider) ) - self.assertEqual(out[0].headers.get("Referer"), init_referrer) + assert out[0].headers.get("Referer") == init_referrer for status, url in redirections: response = Response( @@ -1035,7 +1034,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware): self.referrermw.request_scheduled(request, self.spider) assert isinstance(request, Request) - self.assertEqual(request.headers.get("Referer"), final_referrer) + assert request.headers.get("Referer") == final_referrer class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect): diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 1a0f2e223..5cc3cdc6c 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -1,5 +1,3 @@ -from unittest import TestCase - from testfixtures import LogCapture from scrapy.http import Request, Response @@ -8,8 +6,8 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class TestUrlLengthMiddleware(TestCase): - def setUp(self): +class TestUrlLengthMiddleware: + def setup_method(self): self.maxlength = 25 crawler = get_crawler(Spider, {"URLLENGTH_LIMIT": self.maxlength}) self.spider = crawler._create_spider("foo") @@ -27,7 +25,7 @@ class TestUrlLengthMiddleware(TestCase): ) def test_middleware_works(self): - self.assertEqual(self.process_spider_output(), [self.short_url_req]) + assert self.process_spider_output() == [self.short_url_req] def test_logging(self): with LogCapture() as log: @@ -36,6 +34,6 @@ class TestUrlLengthMiddleware(TestCase): ric = self.stats.get_value( "urllength/request_ignored_count", spider=self.spider ) - self.assertEqual(ric, 1) + assert ric == 1 - self.assertIn(f"Ignoring link (url length > {self.maxlength})", str(log)) + assert f"Ignoring link (url length > {self.maxlength})" in str(log) diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index 72692afab..cd31891a0 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -3,7 +3,6 @@ from datetime import datetime, timezone from tempfile import mkdtemp import pytest -from twisted.trial import unittest from scrapy.exceptions import NotConfigured from scrapy.extensions.spiderstate import SpiderState @@ -11,7 +10,7 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class SpiderStateTest(unittest.TestCase): +class TestSpiderState: def test_store_load(self): jobdir = mkdtemp() try: @@ -27,7 +26,7 @@ class SpiderStateTest(unittest.TestCase): spider2 = Spider(name="default") ss2 = SpiderState(jobdir) ss2.spider_opened(spider2) - self.assertEqual(spider.state, {"one": 1, "dt": dt}) + assert spider.state == {"one": 1, "dt": dt} ss2.spider_closed(spider2) finally: shutil.rmtree(jobdir) @@ -38,7 +37,7 @@ class SpiderStateTest(unittest.TestCase): spider = Spider(name="default") ss = SpiderState() ss.spider_opened(spider) - self.assertEqual(spider.state, {}) + assert spider.state == {} ss.spider_closed(spider) def test_not_configured(self): diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 8556b75dd..6283b9ad6 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -50,9 +50,9 @@ class FifoDiskQueueTestMixin: q.push("a") q.push(123) q.push({"a": "dict"}) - self.assertEqual(q.pop(), "a") - self.assertEqual(q.pop(), 123) - self.assertEqual(q.pop(), {"a": "dict"}) + assert q.pop() == "a" + assert q.pop() == 123 + assert q.pop() == {"a": "dict"} test_nonserializable_object = nonserializable_object_test @@ -92,7 +92,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): q.push(i) i2 = q.pop() assert isinstance(i2, MyItem) - self.assertEqual(i, i2) + assert i == i2 def test_serialize_loader(self): q = self.queue() @@ -101,7 +101,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): loader2 = q.pop() assert isinstance(loader2, MyLoader) assert loader2.default_item_class is MyItem - self.assertEqual(loader2.name_out("x"), "xx") + assert loader2.name_out("x") == "xx" def test_serialize_request_recursive(self): q = self.queue() @@ -110,23 +110,26 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): q.push(r) r2 = q.pop() assert isinstance(r2, Request) - self.assertEqual(r.url, r2.url) + assert r.url == r2.url assert r2.meta["request"] is r2 def test_non_pickable_object(self): q = self.queue() - try: + with pytest.raises( + ValueError, + match="Can't (get|pickle) local object|Can't pickle .*: it's not found as", + ) as exc_info: q.push(lambda x: x) - except ValueError as exc: - if hasattr(sys, "pypy_version_info"): - self.assertIsInstance(exc.__context__, pickle.PicklingError) - else: - self.assertIsInstance(exc.__context__, AttributeError) + if hasattr(sys, "pypy_version_info"): + assert isinstance(exc_info.value.__context__, pickle.PicklingError) + else: + assert isinstance(exc_info.value.__context__, AttributeError) sel = Selector(text="<html><body><p>some text</p></body></html>") - try: + with pytest.raises( + ValueError, match="can't pickle Selector objects" + ) as exc_info: q.push(sel) - except ValueError as exc: - self.assertIsInstance(exc.__context__, TypeError) + assert isinstance(exc_info.value.__context__, TypeError) class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): @@ -151,9 +154,9 @@ class LifoDiskQueueTestMixin: q.push("a") q.push(123) q.push({"a": "dict"}) - self.assertEqual(q.pop(), {"a": "dict"}) - self.assertEqual(q.pop(), 123) - self.assertEqual(q.pop(), "a") + assert q.pop() == {"a": "dict"} + assert q.pop() == 123 + assert q.pop() == "a" test_nonserializable_object = nonserializable_object_test @@ -173,7 +176,7 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): q.push(i) i2 = q.pop() assert isinstance(i2, MyItem) - self.assertEqual(i, i2) + assert i == i2 def test_serialize_loader(self): q = self.queue() @@ -182,7 +185,7 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): loader2 = q.pop() assert isinstance(loader2, MyLoader) assert loader2.default_item_class is MyItem - self.assertEqual(loader2.name_out("x"), "xx") + assert loader2.name_out("x") == "xx" def test_serialize_request_recursive(self): q = self.queue() @@ -191,5 +194,5 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): q.push(r) r2 = q.pop() assert isinstance(r2, Request) - self.assertEqual(r.url, r2.url) + assert r.url == r2.url assert r2.meta["request"] is r2 diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index 88f6657d8..6c153f40e 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -22,14 +22,14 @@ from scrapy.squeues import ( from scrapy.utils.test import get_crawler -class BaseQueueTestCase(unittest.TestCase): - def setUp(self): +class TestBaseQueue: + def setup_method(self): self.tmpdir = tempfile.mkdtemp(prefix="scrapy-queue-tests-") self.qpath = self.tempfilename() self.qdir = tempfile.mkdtemp() self.crawler = get_crawler(Spider) - def tearDown(self): + def teardown_method(self): shutil.rmtree(self.tmpdir) def tempfilename(self): @@ -48,36 +48,36 @@ class RequestQueueTestMixin: if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues do not define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None req = Request("http://www.example.com") q.push(req) - self.assertEqual(len(q), 1) - self.assertEqual(q.peek().url, req.url) - self.assertEqual(q.pop().url, req.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 1 + assert q.peek().url == req.url + assert q.pop().url == req.url + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None q.close() def test_one_element_without_peek(self): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.pop() is None req = Request("http://www.example.com") q.push(req) - self.assertEqual(len(q), 1) + assert len(q) == 1 with pytest.raises( NotImplementedError, match="The underlying queue class does not implement 'peek'", ): q.peek() - self.assertEqual(q.pop().url, req.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert q.pop().url == req.url + assert len(q) == 0 + assert q.pop() is None q.close() @@ -86,35 +86,35 @@ class FifoQueueMixin(RequestQueueTestMixin): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues do not define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None req1 = Request("http://www.example.com/1") req2 = Request("http://www.example.com/2") req3 = Request("http://www.example.com/3") q.push(req1) q.push(req2) q.push(req3) - self.assertEqual(len(q), 3) - self.assertEqual(q.peek().url, req1.url) - self.assertEqual(q.pop().url, req1.url) - self.assertEqual(len(q), 2) - self.assertEqual(q.peek().url, req2.url) - self.assertEqual(q.pop().url, req2.url) - self.assertEqual(len(q), 1) - self.assertEqual(q.peek().url, req3.url) - self.assertEqual(q.pop().url, req3.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 3 + assert q.peek().url == req1.url + assert q.pop().url == req1.url + assert len(q) == 2 + assert q.peek().url == req2.url + assert q.pop().url == req2.url + assert len(q) == 1 + assert q.peek().url == req3.url + assert q.pop().url == req3.url + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None q.close() def test_fifo_without_peek(self): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues do not define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.pop() is None req1 = Request("http://www.example.com/1") req2 = Request("http://www.example.com/2") req3 = Request("http://www.example.com/3") @@ -126,14 +126,14 @@ class FifoQueueMixin(RequestQueueTestMixin): match="The underlying queue class does not implement 'peek'", ): q.peek() - self.assertEqual(len(q), 3) - self.assertEqual(q.pop().url, req1.url) - self.assertEqual(len(q), 2) - self.assertEqual(q.pop().url, req2.url) - self.assertEqual(len(q), 1) - self.assertEqual(q.pop().url, req3.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert len(q) == 3 + assert q.pop().url == req1.url + assert len(q) == 2 + assert q.pop().url == req2.url + assert len(q) == 1 + assert q.pop().url == req3.url + assert len(q) == 0 + assert q.pop() is None q.close() @@ -142,35 +142,35 @@ class LifoQueueMixin(RequestQueueTestMixin): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues do not define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None req1 = Request("http://www.example.com/1") req2 = Request("http://www.example.com/2") req3 = Request("http://www.example.com/3") q.push(req1) q.push(req2) q.push(req3) - self.assertEqual(len(q), 3) - self.assertEqual(q.peek().url, req3.url) - self.assertEqual(q.pop().url, req3.url) - self.assertEqual(len(q), 2) - self.assertEqual(q.peek().url, req2.url) - self.assertEqual(q.pop().url, req2.url) - self.assertEqual(len(q), 1) - self.assertEqual(q.peek().url, req1.url) - self.assertEqual(q.pop().url, req1.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 3 + assert q.peek().url == req3.url + assert q.pop().url == req3.url + assert len(q) == 2 + assert q.peek().url == req2.url + assert q.pop().url == req2.url + assert len(q) == 1 + assert q.peek().url == req1.url + assert q.pop().url == req1.url + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None q.close() def test_lifo_without_peek(self): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues do not define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.pop() is None req1 = Request("http://www.example.com/1") req2 = Request("http://www.example.com/2") req3 = Request("http://www.example.com/3") @@ -182,46 +182,46 @@ class LifoQueueMixin(RequestQueueTestMixin): match="The underlying queue class does not implement 'peek'", ): q.peek() - self.assertEqual(len(q), 3) - self.assertEqual(q.pop().url, req3.url) - self.assertEqual(len(q), 2) - self.assertEqual(q.pop().url, req2.url) - self.assertEqual(len(q), 1) - self.assertEqual(q.pop().url, req1.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert len(q) == 3 + assert q.pop().url == req3.url + assert len(q) == 2 + assert q.pop().url == req2.url + assert len(q) == 1 + assert q.pop().url == req1.url + assert len(q) == 0 + assert q.pop() is None q.close() -class PickleFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): +class TestPickleFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue): def queue(self): return PickleFifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/fifo") -class PickleLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): +class TestPickleLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue): def queue(self): return PickleLifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/lifo") -class MarshalFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): +class TestMarshalFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue): def queue(self): return MarshalFifoDiskQueue.from_crawler( crawler=self.crawler, key="marshal/fifo" ) -class MarshalLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): +class TestMarshalLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue): def queue(self): return MarshalLifoDiskQueue.from_crawler( crawler=self.crawler, key="marshal/lifo" ) -class FifoMemoryQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): +class TestFifoMemoryQueueRequest(FifoQueueMixin, TestBaseQueue): def queue(self): return FifoMemoryQueue.from_crawler(crawler=self.crawler) -class LifoMemoryQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): +class TestLifoMemoryQueueRequest(LifoQueueMixin, TestBaseQueue): def queue(self): return LifoMemoryQueue.from_crawler(crawler=self.crawler) diff --git a/tests/test_stats.py b/tests/test_stats.py index 3d4c7e88e..537614364 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -1,4 +1,3 @@ -import unittest from datetime import datetime from unittest import mock @@ -8,8 +7,8 @@ from scrapy.statscollectors import DummyStatsCollector, StatsCollector from scrapy.utils.test import get_crawler -class CoreStatsExtensionTest(unittest.TestCase): - def setUp(self): +class TestCoreStatsExtension: + def setup_method(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("foo") @@ -24,19 +23,16 @@ class CoreStatsExtensionTest(unittest.TestCase): ext.response_received(self.spider) ext.item_dropped({}, self.spider, ZeroDivisionError()) ext.spider_closed(self.spider, "finished") - self.assertEqual( - ext.stats._stats, - { - "start_time": fixed_datetime, - "finish_time": fixed_datetime, - "item_scraped_count": 1, - "response_received_count": 1, - "item_dropped_count": 1, - "item_dropped_reasons_count/ZeroDivisionError": 1, - "finish_reason": "finished", - "elapsed_time_seconds": 0.0, - }, - ) + assert ext.stats._stats == { + "start_time": fixed_datetime, + "finish_time": fixed_datetime, + "item_scraped_count": 1, + "response_received_count": 1, + "item_dropped_count": 1, + "item_dropped_reasons_count/ZeroDivisionError": 1, + "finish_reason": "finished", + "elapsed_time_seconds": 0.0, + } def test_core_stats_dummy_stats_collector(self): self.crawler.stats = DummyStatsCollector(self.crawler) @@ -46,51 +42,51 @@ class CoreStatsExtensionTest(unittest.TestCase): ext.response_received(self.spider) ext.item_dropped({}, self.spider, ZeroDivisionError()) ext.spider_closed(self.spider, "finished") - self.assertEqual(ext.stats._stats, {}) + assert ext.stats._stats == {} -class StatsCollectorTest(unittest.TestCase): - def setUp(self): +class TestStatsCollector: + def setup_method(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("foo") def test_collector(self): stats = StatsCollector(self.crawler) - self.assertEqual(stats.get_stats(), {}) - self.assertEqual(stats.get_value("anything"), None) - self.assertEqual(stats.get_value("anything", "default"), "default") + assert stats.get_stats() == {} + assert stats.get_value("anything") is None + assert stats.get_value("anything", "default") == "default" stats.set_value("test", "value") - self.assertEqual(stats.get_stats(), {"test": "value"}) + assert stats.get_stats() == {"test": "value"} stats.set_value("test2", 23) - self.assertEqual(stats.get_stats(), {"test": "value", "test2": 23}) - self.assertEqual(stats.get_value("test2"), 23) + assert stats.get_stats() == {"test": "value", "test2": 23} + assert stats.get_value("test2") == 23 stats.inc_value("test2") - self.assertEqual(stats.get_value("test2"), 24) + assert stats.get_value("test2") == 24 stats.inc_value("test2", 6) - self.assertEqual(stats.get_value("test2"), 30) + assert stats.get_value("test2") == 30 stats.max_value("test2", 6) - self.assertEqual(stats.get_value("test2"), 30) + assert stats.get_value("test2") == 30 stats.max_value("test2", 40) - self.assertEqual(stats.get_value("test2"), 40) + assert stats.get_value("test2") == 40 stats.max_value("test3", 1) - self.assertEqual(stats.get_value("test3"), 1) + assert stats.get_value("test3") == 1 stats.min_value("test2", 60) - self.assertEqual(stats.get_value("test2"), 40) + assert stats.get_value("test2") == 40 stats.min_value("test2", 35) - self.assertEqual(stats.get_value("test2"), 35) + assert stats.get_value("test2") == 35 stats.min_value("test4", 7) - self.assertEqual(stats.get_value("test4"), 7) + assert stats.get_value("test4") == 7 def test_dummy_collector(self): stats = DummyStatsCollector(self.crawler) - self.assertEqual(stats.get_stats(), {}) - self.assertEqual(stats.get_value("anything"), None) - self.assertEqual(stats.get_value("anything", "default"), "default") + assert stats.get_stats() == {} + assert stats.get_value("anything") is None + assert stats.get_value("anything", "default") == "default" stats.set_value("test", "value") stats.inc_value("v1") stats.max_value("v2", 100) stats.min_value("v3", 100) stats.open_spider("a") stats.set_value("test", "value", spider=self.spider) - self.assertEqual(stats.get_stats(), {}) - self.assertEqual(stats.get_stats("a"), {}) + assert stats.get_stats() == {} + assert stats.get_stats("a") == {} From 40833afc86d45543a521b0f147387f7f770a8adc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 7 Mar 2025 22:33:41 +0400 Subject: [PATCH 3/4] Work around a queuelib test file close problem. (#6703) --- tests/test_squeues.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 6283b9ad6..0b6ed8e11 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -130,6 +130,9 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): ) as exc_info: q.push(sel) assert isinstance(exc_info.value.__context__, TypeError) + # This seems to help with https://github.com/scrapy/queuelib/issues/70. + # It will need to remain under a queuelib version check after that bug is fixed. + del exc_info class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): From 1469b2739ea566a57e0b5f8e6bb104fd19460d24 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sat, 8 Mar 2025 20:50:54 +0400 Subject: [PATCH 4/4] Drop tests/test_loader_deprecated.py. (#6704) --- tests/test_loader_deprecated.py | 750 -------------------------------- 1 file changed, 750 deletions(-) delete mode 100644 tests/test_loader_deprecated.py diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py deleted file mode 100644 index 0d7921b1d..000000000 --- a/tests/test_loader_deprecated.py +++ /dev/null @@ -1,750 +0,0 @@ -""" -These tests are kept as references from the ones that were ported to a itemloaders library. -Once we remove the references from scrapy, we can remove these tests. -""" - -import unittest -from functools import partial - -import pytest -from itemloaders.processors import ( - Compose, - Identity, - Join, - MapCompose, - SelectJmes, - TakeFirst, -) - -from scrapy.item import Field, Item -from scrapy.loader import ItemLoader - - -# test items -class NameItem(Item): - name = Field() - - -class SummaryItem(NameItem): - url = Field() - summary = Field() - - -# test item loaders -class NameItemLoader(ItemLoader): - default_item_class = SummaryItem - - -class ProcessorItemLoader(NameItemLoader): - name_in = MapCompose(lambda v: v.title()) - - -class DefaultedItemLoader(NameItemLoader): - default_input_processor = MapCompose(lambda v: v[:-1]) - - -# test processors -def processor_with_args(value, other=None, loader_context=None): - if "key" in loader_context: - return loader_context["key"] - return value - - -class BasicItemLoaderTest(unittest.TestCase): - def test_load_item_using_default_loader(self): - i = SummaryItem() - i["summary"] = "lala" - il = ItemLoader(item=i) - il.add_value("name", "marta") - item = il.load_item() - assert item is i - self.assertEqual(item["summary"], ["lala"]) - self.assertEqual(item["name"], ["marta"]) - - def test_load_item_using_custom_loader(self): - il = ProcessorItemLoader() - il.add_value("name", "marta") - item = il.load_item() - self.assertEqual(item["name"], ["Marta"]) - - def test_load_item_ignore_none_field_values(self): - def validate_sku(value): - # Let's assume a SKU is only digits. - return value if value.isdigit() else None - - class MyLoader(ItemLoader): - name_out = Compose(lambda vs: vs[0]) # take first which allows empty values - price_out = Compose(TakeFirst(), float) - sku_out = Compose(TakeFirst(), validate_sku) - - valid_fragment = "SKU: 1234" - invalid_fragment = "SKU: not available" - sku_re = "SKU: (.+)" - - il = MyLoader(item={}) - # Should not return "sku: None". - il.add_value("sku", [invalid_fragment], re=sku_re) - # Should not ignore empty values. - il.add_value("name", "") - il.add_value("price", ["0"]) - self.assertEqual( - il.load_item(), - { - "name": "", - "price": 0.0, - }, - ) - - il.replace_value("sku", [valid_fragment], re=sku_re) - self.assertEqual(il.load_item()["sku"], "1234") - - def test_self_referencing_loader(self): - class MyLoader(ItemLoader): - url_out = TakeFirst() - - def img_url_out(self, values): - return (self.get_output_value("url") or "") + values[0] - - il = MyLoader(item={}) - il.add_value("url", "http://example.com/") - il.add_value("img_url", "1234.png") - self.assertEqual( - il.load_item(), - { - "url": "http://example.com/", - "img_url": "http://example.com/1234.png", - }, - ) - - il = MyLoader(item={}) - il.add_value("img_url", "1234.png") - self.assertEqual( - il.load_item(), - { - "img_url": "1234.png", - }, - ) - - def test_add_value(self): - il = ProcessorItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_collected_values("name"), ["Marta"]) - self.assertEqual(il.get_output_value("name"), ["Marta"]) - il.add_value("name", "pepe") - self.assertEqual(il.get_collected_values("name"), ["Marta", "Pepe"]) - self.assertEqual(il.get_output_value("name"), ["Marta", "Pepe"]) - - # test add object value - il.add_value("summary", {"key": 1}) - self.assertEqual(il.get_collected_values("summary"), [{"key": 1}]) - - il.add_value(None, "Jim", lambda x: {"name": x}) - self.assertEqual(il.get_collected_values("name"), ["Marta", "Pepe", "Jim"]) - - def test_add_zero(self): - il = NameItemLoader() - il.add_value("name", 0) - self.assertEqual(il.get_collected_values("name"), [0]) - - def test_replace_value(self): - il = ProcessorItemLoader() - il.replace_value("name", "marta") - self.assertEqual(il.get_collected_values("name"), ["Marta"]) - self.assertEqual(il.get_output_value("name"), ["Marta"]) - il.replace_value("name", "pepe") - self.assertEqual(il.get_collected_values("name"), ["Pepe"]) - self.assertEqual(il.get_output_value("name"), ["Pepe"]) - - il.replace_value(None, "Jim", lambda x: {"name": x}) - self.assertEqual(il.get_collected_values("name"), ["Jim"]) - - def test_get_value(self): - il = NameItemLoader() - self.assertEqual("FOO", il.get_value(["foo", "bar"], TakeFirst(), str.upper)) - self.assertEqual( - ["foo", "bar"], il.get_value(["name:foo", "name:bar"], re="name:(.*)$") - ) - self.assertEqual( - "foo", il.get_value(["name:foo", "name:bar"], TakeFirst(), re="name:(.*)$") - ) - - il.add_value("name", ["name:foo", "name:bar"], TakeFirst(), re="name:(.*)$") - self.assertEqual(["foo"], il.get_collected_values("name")) - il.replace_value("name", "name:bar", re="name:(.*)$") - self.assertEqual(["bar"], il.get_collected_values("name")) - - def test_iter_on_input_processor_input(self): - class NameFirstItemLoader(NameItemLoader): - name_in = TakeFirst() - - il = NameFirstItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_collected_values("name"), ["marta"]) - il = NameFirstItemLoader() - il.add_value("name", ["marta", "jose"]) - self.assertEqual(il.get_collected_values("name"), ["marta"]) - - il = NameFirstItemLoader() - il.replace_value("name", "marta") - self.assertEqual(il.get_collected_values("name"), ["marta"]) - il = NameFirstItemLoader() - il.replace_value("name", ["marta", "jose"]) - self.assertEqual(il.get_collected_values("name"), ["marta"]) - - il = NameFirstItemLoader() - il.add_value("name", "marta") - il.add_value("name", ["jose", "pedro"]) - self.assertEqual(il.get_collected_values("name"), ["marta", "jose"]) - - def test_map_compose_filter(self): - def filter_world(x): - return None if x == "world" else x - - proc = MapCompose(filter_world, str.upper) - self.assertEqual( - proc(["hello", "world", "this", "is", "scrapy"]), - ["HELLO", "THIS", "IS", "SCRAPY"], - ) - - def test_map_compose_filter_multil(self): - class TestItemLoader(NameItemLoader): - name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1]) - - il = TestItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["Mart"]) - item = il.load_item() - self.assertEqual(item["name"], ["Mart"]) - - def test_default_input_processor(self): - il = DefaultedItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["mart"]) - - def test_inherited_default_input_processor(self): - class InheritDefaultedItemLoader(DefaultedItemLoader): - pass - - il = InheritDefaultedItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["mart"]) - - def test_input_processor_inheritance(self): - class ChildItemLoader(ProcessorItemLoader): - url_in = MapCompose(lambda v: v.lower()) - - il = ChildItemLoader() - il.add_value("url", "HTTP://scrapy.ORG") - self.assertEqual(il.get_output_value("url"), ["http://scrapy.org"]) - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["Marta"]) - - class ChildChildItemLoader(ChildItemLoader): - url_in = MapCompose(lambda v: v.upper()) - summary_in = MapCompose(lambda v: v) - - il = ChildChildItemLoader() - il.add_value("url", "http://scrapy.org") - self.assertEqual(il.get_output_value("url"), ["HTTP://SCRAPY.ORG"]) - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["Marta"]) - - def test_empty_map_compose(self): - class IdentityDefaultedItemLoader(DefaultedItemLoader): - name_in = MapCompose() - - il = IdentityDefaultedItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["marta"]) - - def test_identity_input_processor(self): - class IdentityDefaultedItemLoader(DefaultedItemLoader): - name_in = Identity() - - il = IdentityDefaultedItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["marta"]) - - def test_extend_custom_input_processors(self): - class ChildItemLoader(ProcessorItemLoader): - name_in = MapCompose(ProcessorItemLoader.name_in, str.swapcase) - - il = ChildItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["mARTA"]) - - def test_extend_default_input_processors(self): - class ChildDefaultedItemLoader(DefaultedItemLoader): - name_in = MapCompose( - DefaultedItemLoader.default_input_processor, str.swapcase - ) - - il = ChildDefaultedItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["MART"]) - - def test_output_processor_using_function(self): - il = ProcessorItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - - class TakeFirstItemLoader(ProcessorItemLoader): - name_out = " ".join - - il = TakeFirstItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), "Mar Ta") - - def test_output_processor_error(self): - class TestItemLoader(ItemLoader): - default_item_class = SummaryItem - name_out = MapCompose(float) - - il = TestItemLoader() - il.add_value("name", ["$10"]) - try: - float("$10") - except Exception as e: - expected_exc_str = str(e) - - exc = None - try: - il.load_item() - except Exception as e: - exc = e - assert isinstance(exc, ValueError) - s = str(exc) - assert "name" in s, s - assert "$10" in s, s - assert "ValueError" in s, s - assert expected_exc_str in s, s - - def test_output_processor_using_classes(self): - il = ProcessorItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - - class TakeFirstItemLoader(ProcessorItemLoader): - name_out = Join() - - il = TakeFirstItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), "Mar Ta") - - class TakeFirstItemLoader2(ProcessorItemLoader): - name_out = Join("<br>") - - il = TakeFirstItemLoader2() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), "Mar<br>Ta") - - def test_default_output_processor(self): - il = ProcessorItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - - class LalaItemLoader(ProcessorItemLoader): - default_output_processor = Identity() - - il = LalaItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - - def test_loader_context_on_declaration(self): - class ChildItemLoader(ProcessorItemLoader): - url_in = MapCompose(processor_with_args, key="val") - - il = ChildItemLoader() - il.add_value("url", "text") - self.assertEqual(il.get_output_value("url"), ["val"]) - il.replace_value("url", "text2") - self.assertEqual(il.get_output_value("url"), ["val"]) - - def test_loader_context_on_instantiation(self): - class ChildItemLoader(ProcessorItemLoader): - url_in = MapCompose(processor_with_args) - - il = ChildItemLoader(key="val") - il.add_value("url", "text") - self.assertEqual(il.get_output_value("url"), ["val"]) - il.replace_value("url", "text2") - self.assertEqual(il.get_output_value("url"), ["val"]) - - def test_loader_context_on_assign(self): - class ChildItemLoader(ProcessorItemLoader): - url_in = MapCompose(processor_with_args) - - il = ChildItemLoader() - il.context["key"] = "val" - il.add_value("url", "text") - self.assertEqual(il.get_output_value("url"), ["val"]) - il.replace_value("url", "text2") - self.assertEqual(il.get_output_value("url"), ["val"]) - - def test_item_passed_to_input_processor_functions(self): - def processor(value, loader_context): - return loader_context["item"]["name"] - - class ChildItemLoader(ProcessorItemLoader): - url_in = MapCompose(processor) - - it = SummaryItem(name="marta") - il = ChildItemLoader(item=it) - il.add_value("url", "text") - self.assertEqual(il.get_output_value("url"), ["marta"]) - il.replace_value("url", "text2") - self.assertEqual(il.get_output_value("url"), ["marta"]) - - def test_compose_processor(self): - class TestItemLoader(NameItemLoader): - name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1]) - - il = TestItemLoader() - il.add_value("name", ["marta", "other"]) - self.assertEqual(il.get_output_value("name"), "Mart") - item = il.load_item() - self.assertEqual(item["name"], "Mart") - - def test_partial_processor(self): - def join(values, sep=None, loader_context=None, ignored=None): - if sep is not None: - return sep.join(values) - if loader_context and "sep" in loader_context: - return loader_context["sep"].join(values) - return "".join(values) - - class TestItemLoader(NameItemLoader): - name_out = Compose(partial(join, sep="+")) - url_out = Compose(partial(join, loader_context={"sep": "."})) - summary_out = Compose(partial(join, ignored="foo")) - - il = TestItemLoader() - il.add_value("name", ["rabbit", "hole"]) - il.add_value("url", ["rabbit", "hole"]) - il.add_value("summary", ["rabbit", "hole"]) - item = il.load_item() - self.assertEqual(item["name"], "rabbit+hole") - self.assertEqual(item["url"], "rabbit.hole") - self.assertEqual(item["summary"], "rabbithole") - - def test_error_input_processor(self): - class TestItem(Item): - name = Field() - - class TestItemLoader(ItemLoader): - default_item_class = TestItem - name_in = MapCompose(float) - - il = TestItemLoader() - with pytest.raises( - ValueError, - match="Error with input processor MapCompose: .* " - "error='ValueError: Error in MapCompose .* " - "error='ValueError: could not convert", - ): - il.add_value("name", ["marta", "other"]) - - def test_error_output_processor(self): - class TestItem(Item): - name = Field() - - class TestItemLoader(ItemLoader): - default_item_class = TestItem - name_out = Compose(Join(), float) - - il = TestItemLoader() - il.add_value("name", "marta") - with pytest.raises( - ValueError, - match="Error with output processor: .* " - "error='ValueError: Error in Compose .* " - "error='ValueError: could not convert", - ): - il.load_item() - - def test_error_processor_as_argument(self): - class TestItem(Item): - name = Field() - - class TestItemLoader(ItemLoader): - default_item_class = TestItem - - il = TestItemLoader() - with pytest.raises( - ValueError, - match=r"Error with processor Compose .* " - r"error='ValueError: Error in Compose .* " - r"error='TypeError: float\(\) argument", - ): - il.add_value("name", ["marta", "other"], Compose(float)) - - -class InitializationFromDictTest(unittest.TestCase): - item_class = dict - - def test_keep_single_value(self): - """Loaded item should contain values from the initial item""" - input_item = self.item_class(name="foo") - il = ItemLoader(item=input_item) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo"]}) - - def test_keep_list(self): - """Loaded item should contain values from the initial item""" - input_item = self.item_class(name=["foo", "bar"]) - il = ItemLoader(item=input_item) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo", "bar"]}) - - def test_add_value_singlevalue_singlevalue(self): - """Values added after initialization should be appended""" - input_item = self.item_class(name="foo") - il = ItemLoader(item=input_item) - il.add_value("name", "bar") - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo", "bar"]}) - - def test_add_value_singlevalue_list(self): - """Values added after initialization should be appended""" - input_item = self.item_class(name="foo") - il = ItemLoader(item=input_item) - il.add_value("name", ["item", "loader"]) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo", "item", "loader"]}) - - def test_add_value_list_singlevalue(self): - """Values added after initialization should be appended""" - input_item = self.item_class(name=["foo", "bar"]) - il = ItemLoader(item=input_item) - il.add_value("name", "qwerty") - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo", "bar", "qwerty"]}) - - def test_add_value_list_list(self): - """Values added after initialization should be appended""" - input_item = self.item_class(name=["foo", "bar"]) - il = ItemLoader(item=input_item) - il.add_value("name", ["item", "loader"]) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo", "bar", "item", "loader"]}) - - def test_get_output_value_singlevalue(self): - """Getting output value must not remove value from item""" - input_item = self.item_class(name="foo") - il = ItemLoader(item=input_item) - self.assertEqual(il.get_output_value("name"), ["foo"]) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(loaded_item, {"name": ["foo"]}) - - def test_get_output_value_list(self): - """Getting output value must not remove value from item""" - input_item = self.item_class(name=["foo", "bar"]) - il = ItemLoader(item=input_item) - self.assertEqual(il.get_output_value("name"), ["foo", "bar"]) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(loaded_item, {"name": ["foo", "bar"]}) - - def test_values_single(self): - """Values from initial item must be added to loader._values""" - input_item = self.item_class(name="foo") - il = ItemLoader(item=input_item) - self.assertEqual(il._values.get("name"), ["foo"]) - - def test_values_list(self): - """Values from initial item must be added to loader._values""" - input_item = self.item_class(name=["foo", "bar"]) - il = ItemLoader(item=input_item) - self.assertEqual(il._values.get("name"), ["foo", "bar"]) - - -class BaseNoInputReprocessingLoader(ItemLoader): - title_in = MapCompose(str.upper) - title_out = TakeFirst() - - -class NoInputReprocessingDictLoader(BaseNoInputReprocessingLoader): - default_item_class = dict - - -class NoInputReprocessingFromDictTest(unittest.TestCase): - """ - Loaders initialized from loaded items must not reprocess fields (dict instances) - """ - - def test_avoid_reprocessing_with_initial_values_single(self): - il = NoInputReprocessingDictLoader(item={"title": "foo"}) - il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "foo"}) - self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "foo"} - ) - - def test_avoid_reprocessing_with_initial_values_list(self): - il = NoInputReprocessingDictLoader(item={"title": ["foo", "bar"]}) - il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "foo"}) - self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "foo"} - ) - - def test_avoid_reprocessing_without_initial_values_single(self): - il = NoInputReprocessingDictLoader() - il.add_value("title", "foo") - il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "FOO"}) - self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "FOO"} - ) - - def test_avoid_reprocessing_without_initial_values_list(self): - il = NoInputReprocessingDictLoader() - il.add_value("title", ["foo", "bar"]) - il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "FOO"}) - self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "FOO"} - ) - - -class TestOutputProcessorDict(unittest.TestCase): - def test_output_processor(self): - class TempDict(dict): - def __init__(self, *args, **kwargs): - super().__init__(self, *args, **kwargs) - self.setdefault("temp", 0.3) - - class TempLoader(ItemLoader): - default_item_class = TempDict - default_input_processor = Identity() - default_output_processor = Compose(TakeFirst()) - - loader = TempLoader() - item = loader.load_item() - self.assertIsInstance(item, TempDict) - self.assertEqual(dict(item), {"temp": 0.3}) - - -class ProcessorsTest(unittest.TestCase): - def test_take_first(self): - proc = TakeFirst() - self.assertEqual(proc([None, "", "hello", "world"]), "hello") - self.assertEqual(proc([None, "", 0, "hello", "world"]), 0) - - def test_identity(self): - proc = Identity() - self.assertEqual( - proc([None, "", "hello", "world"]), [None, "", "hello", "world"] - ) - - def test_join(self): - proc = Join() - with pytest.raises(TypeError): - proc([None, "", "hello", "world"]) - self.assertEqual(proc(["", "hello", "world"]), " hello world") - self.assertEqual(proc(["hello", "world"]), "hello world") - self.assertIsInstance(proc(["hello", "world"]), str) - - def test_compose(self): - proc = Compose(lambda v: v[0], str.upper) - self.assertEqual(proc(["hello", "world"]), "HELLO") - proc = Compose(str.upper) - self.assertEqual(proc(None), None) - proc = Compose(str.upper, stop_on_none=False) - with pytest.raises( - ValueError, - match="Error in Compose with .* error='TypeError: (descriptor 'upper'|'str' object expected)", - ): - proc(None) - proc = Compose(str.upper, lambda x: x + 1) - with pytest.raises( - ValueError, - match="Error in Compose with .* error='TypeError: (can only|unsupported operand)", - ): - proc("hello") - - def test_mapcompose(self): - def filter_world(x): - return None if x == "world" else x - - proc = MapCompose(filter_world, str.upper) - self.assertEqual( - proc(["hello", "world", "this", "is", "scrapy"]), - ["HELLO", "THIS", "IS", "SCRAPY"], - ) - proc = MapCompose(filter_world, str.upper) - self.assertEqual(proc(None), []) - proc = MapCompose(filter_world, str.upper) - with pytest.raises( - ValueError, - match="Error in MapCompose with .* error='TypeError: (descriptor 'upper'|'str' object expected)", - ): - proc([1]) - proc = MapCompose(filter_world, lambda x: x + 1) - with pytest.raises( - ValueError, - match="Error in MapCompose with .* error='TypeError: (can only|unsupported operand)", - ): - proc("hello") - - -class SelectJmesTestCase(unittest.TestCase): - test_list_equals = { - "simple": ("foo.bar", {"foo": {"bar": "baz"}}, "baz"), - "invalid": ("foo.bar.baz", {"foo": {"bar": "baz"}}, None), - "top_level": ("foo", {"foo": {"bar": "baz"}}, {"bar": "baz"}), - "double_vs_single_quote_string": ("foo.bar", {"foo": {"bar": "baz"}}, "baz"), - "dict": ( - "foo.bar[*].name", - {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, - ["one", "two"], - ), - "list": ("[1]", [1, 2], 2), - } - - def test_output(self): - for k, v in self.test_list_equals.items(): - expr, test_list, expected = v - test = SelectJmes(expr)(test_list) - self.assertEqual( - test, expected, msg=f'test "{k}" got {test} expected {expected}' - ) - - -# Functions as processors - - -def function_processor_strip(iterable): - return [x.strip() for x in iterable] - - -def function_processor_upper(iterable): - return [x.upper() for x in iterable] - - -class FunctionProcessorItem(Item): - foo = Field( - input_processor=function_processor_strip, - output_processor=function_processor_upper, - ) - - -class FunctionProcessorDictLoader(ItemLoader): - default_item_class = dict - foo_in = function_processor_strip - foo_out = function_processor_upper - - -class FunctionProcessorTestCase(unittest.TestCase): - def test_processor_defined_in_item_loader(self): - lo = FunctionProcessorDictLoader() - lo.add_value("foo", " bar ") - lo.add_value("foo", [" asdf ", " qwerty "]) - self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]})