diff --git a/pyproject.toml b/pyproject.toml index ad62ea212..82d8056f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -378,8 +378,6 @@ ignore = [ # Temporarily silenced PT rules # Use a regular `assert` instead of unittest-style `assertEqual` "PT009", - # Use `pytest.raises` instead of unittest-style `assertRaises` - "PT027", ] [tool.ruff.lint.per-file-ignores] diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index a04e0107b..324a9b955 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -214,7 +214,7 @@ class DownloaderAwarePriorityQueue: "DownloaderAwarePriorityQueue accepts " "``slot_startprios`` as a dict; " f"{slot_startprios.__class__!r} instance " - "is passed. Most likely, it means the state is" + "is passed. Most likely, it means the state is " "created by an incompatible priority queue. " "Only a crawl started with the same priority " "queue class can be resumed." diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 425188d32..df5ebfa7b 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -185,9 +185,8 @@ class CrawlerTestCase(BaseCrawlerTest): def test_get_downloader_middleware_not_crawling(self): crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) - self.assertRaises( - RuntimeError, crawler.get_downloader_middleware, DefaultSpider - ) + with pytest.raises(RuntimeError): + crawler.get_downloader_middleware(DefaultSpider) @inlineCallbacks def test_get_downloader_middleware_no_engine(self): @@ -266,7 +265,8 @@ class CrawlerTestCase(BaseCrawlerTest): def test_get_extension_not_crawling(self): crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) - self.assertRaises(RuntimeError, crawler.get_extension, DefaultSpider) + with pytest.raises(RuntimeError): + crawler.get_extension(DefaultSpider) @inlineCallbacks def test_get_extension_no_engine(self): @@ -345,7 +345,8 @@ class CrawlerTestCase(BaseCrawlerTest): def test_get_item_pipeline_not_crawling(self): crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) - self.assertRaises(RuntimeError, crawler.get_item_pipeline, DefaultSpider) + with pytest.raises(RuntimeError): + crawler.get_item_pipeline(DefaultSpider) @inlineCallbacks def test_get_item_pipeline_no_engine(self): @@ -424,7 +425,8 @@ class CrawlerTestCase(BaseCrawlerTest): def test_get_spider_middleware_not_crawling(self): crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) - self.assertRaises(RuntimeError, crawler.get_spider_middleware, DefaultSpider) + with pytest.raises(RuntimeError): + crawler.get_spider_middleware(DefaultSpider) @inlineCallbacks def test_get_spider_middleware_no_engine(self): @@ -537,7 +539,8 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, } ) - self.assertRaises(MultipleInvalid, CrawlerRunner, settings) + with pytest.raises(MultipleInvalid): + CrawlerRunner(settings) def test_crawler_runner_accepts_dict(self): runner = CrawlerRunner({"foo": "bar"}) @@ -630,13 +633,15 @@ class CrawlerRunnerHasSpider(unittest.TestCase): } ) else: - msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" - with self.assertRaisesRegex(Exception, msg): - runner = CrawlerRunner( - settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - } - ) + runner = CrawlerRunner( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + ) + with pytest.raises( + Exception, + match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)", + ): yield runner.crawl(NoRequestsSpider) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 174bf841e..17d5c2d0a 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -248,5 +248,5 @@ class Https2ProxyTestCase(BaseTestClasses.Http11ProxyTestCase): @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): - with self.assertRaises(NotImplementedError): + with pytest.raises(NotImplementedError): yield super().test_download_with_proxy_https_timeout() diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 42051042c..49498375c 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -1,4 +1,5 @@ import asyncio +from gzip import BadGzipFile from unittest import mock import pytest @@ -106,7 +107,8 @@ class DefaultsTest(ManagerTestCase): "Location": "http://example.com/login", }, ) - self.assertRaises(OSError, self._download, request=req, response=resp) + with pytest.raises(BadGzipFile): + self._download(request=req, response=resp) class ResponseFromProcessRequestTest(ManagerTestCase): diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 772769690..694a669d4 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -83,11 +83,10 @@ class CookiesMiddlewareTest(TestCase): self.assertEqual(req2.headers.get("Cookie"), b"C1=value1") def test_setting_false_cookies_enabled(self): - self.assertRaises( - NotConfigured, - CookiesMiddleware.from_crawler, - get_crawler(settings_dict={"COOKIES_ENABLED": False}), - ) + with pytest.raises(NotConfigured): + CookiesMiddleware.from_crawler( + get_crawler(settings_dict={"COOKIES_ENABLED": False}) + ) def test_setting_default_cookies_enabled(self): self.assertIsInstance( diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 581fc1974..0f1489344 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -1,5 +1,6 @@ import unittest +import pytest from w3lib.http import basic_auth_header from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware @@ -29,8 +30,8 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase): self.spider = LegacySpider("foo") def test_auth(self): - with self.assertRaises(AttributeError): - mw = HttpAuthMiddleware() + mw = HttpAuthMiddleware() + with pytest.raises(AttributeError): mw.spider_opened(self.spider) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 74db93f8a..de3a9689b 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -5,6 +5,8 @@ import time import unittest from contextlib import contextmanager +import pytest + from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware from scrapy.exceptions import IgnoreRequest from scrapy.http import HtmlResponse, Request, Response @@ -192,9 +194,8 @@ class DummyPolicyTest(_BaseTest): def test_middleware_ignore_missing(self): with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw: - self.assertRaises( - IgnoreRequest, mw.process_request, self.request, self.spider - ) + with pytest.raises(IgnoreRequest): + mw.process_request(self.request, self.spider) mw.process_response(self.request, self.response, self.spider) response = mw.process_request(self.request, self.spider) assert isinstance(response, HtmlResponse) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index a1c5883ec..b3e3b98d7 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -4,6 +4,7 @@ from logging import WARNING from pathlib import Path from unittest import SkipTest, TestCase +import pytest from testfixtures import LogCapture from w3lib.encoding import resolve_encoding @@ -87,11 +88,10 @@ class HttpCompressionTest(TestCase): ) def test_setting_false_compression_enabled(self): - self.assertRaises( - NotConfigured, - HttpCompressionMiddleware.from_crawler, - get_crawler(settings_dict={"COMPRESSION_ENABLED": False}), - ) + with pytest.raises(NotConfigured): + HttpCompressionMiddleware.from_crawler( + get_crawler(settings_dict={"COMPRESSION_ENABLED": False}) + ) def test_setting_default_compression_enabled(self): self.assertIsInstance( @@ -520,13 +520,8 @@ class HttpCompressionTest(TestCase): mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") - self.assertRaises( - IgnoreRequest, - mw.process_response, - response.request, - response, - spider, - ) + with pytest.raises(IgnoreRequest): + mw.process_response(response.request, response, spider) def test_compression_bomb_setting_br(self): try: @@ -561,13 +556,8 @@ class HttpCompressionTest(TestCase): mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") - self.assertRaises( - IgnoreRequest, - mw.process_response, - response.request, - response, - spider, - ) + with pytest.raises(IgnoreRequest): + mw.process_response(response.request, response, spider) def test_compression_bomb_spider_attr_br(self): try: @@ -600,13 +590,8 @@ class HttpCompressionTest(TestCase): response = self._getresponse(f"bomb-{compression_id}") response.meta["download_maxsize"] = 10_000_000 - self.assertRaises( - IgnoreRequest, - mw.process_response, - response.request, - response, - spider, - ) + with pytest.raises(IgnoreRequest): + mw.process_response(response.request, response, spider) def test_compression_bomb_request_meta_br(self): try: diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index f950906e9..47abeee7a 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -72,9 +72,8 @@ class Base: assert isinstance(req, Request) assert "redirect_times" in req.meta self.assertEqual(req.meta["redirect_times"], 1) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) + with pytest.raises(IgnoreRequest): + self.mw.process_response(req, rsp, self.spider) def test_ttl(self): self.mw.max_redirect_times = 100 @@ -83,9 +82,8 @@ class Base: req = self.mw.process_response(req, rsp, self.spider) assert isinstance(req, Request) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) + with pytest.raises(IgnoreRequest): + self.mw.process_response(req, rsp, self.spider) def test_redirect_urls(self): req1 = Request("http://scrapytest.org/first") diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 6b9b39413..36f48db69 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,6 +1,7 @@ import logging import unittest +import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.internet.error import ( @@ -407,7 +408,7 @@ class GetRetryRequestTest(unittest.TestCase): def test_no_spider(self): request = Request("https://example.com") - with self.assertRaises(TypeError): + with pytest.raises(TypeError): get_retry_request(request) # pylint: disable=missing-kwoa def test_max_retry_times_setting(self): diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 535e07c1f..9b95400fd 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,5 +1,6 @@ from unittest import mock +import pytest from twisted.internet import error, reactor from twisted.internet.defer import Deferred, DeferredList, maybeDeferred from twisted.python import failure @@ -26,7 +27,8 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): def test_robotstxt_settings(self): self.crawler.settings = Settings() self.crawler.settings.set("USER_AGENT", "CustomAgent") - self.assertRaises(NotConfigured, RobotsTxtMiddleware, self.crawler) + with pytest.raises(NotConfigured): + RobotsTxtMiddleware(self.crawler) def _get_successful_crawler(self): crawler = self.crawler diff --git a/tests/test_exporters.py b/tests/test_exporters.py index eb8d309b6..48728e078 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -10,6 +10,7 @@ from io import BytesIO from typing import Any import lxml.etree +import pytest from itemadapter import ItemAdapter from scrapy.exporters import ( @@ -147,7 +148,7 @@ class PythonItemExporterTest(BaseItemExporterTest): return PythonItemExporter(**kwargs) def test_invalid_option(self): - with self.assertRaisesRegex(TypeError, "Unexpected options: invalid_option"): + with pytest.raises(TypeError, match="Unexpected options: invalid_option"): PythonItemExporter(invalid_option="something") def test_nested_item(self): @@ -388,7 +389,7 @@ class CsvItemExporterTest(BaseItemExporterTest): ) def test_errors_default(self): - with self.assertRaises(UnicodeEncodeError): + with pytest.raises(UnicodeEncodeError): self.assertExportResult( item={"text": "W\u0275\u200brd"}, expected=None, @@ -549,7 +550,8 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.ie = self._get_exporter(sort_keys=True) self.test_export_item() self._check_output() - self.assertRaises(TypeError, self._get_exporter, foo_unknown_keyword_bar=True) + with pytest.raises(TypeError): + self._get_exporter(foo_unknown_keyword_bar=True) def test_nonstring_types_item(self): item = self._get_nonstring_types_item() @@ -602,7 +604,8 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): i3 = MyItem(name="Jesus", age="44") self.ie.start_exporting() self.ie.export_item(i1) - self.assertRaises(TypeError, self.ie.export_item, i2) + with pytest.raises(TypeError): + self.ie.export_item(i2) self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) @@ -657,7 +660,8 @@ class JsonItemExporterToBytesTest(BaseItemExporterTest): i3 = MyItem(name="Jesus", age="44") self.ie.start_exporting() self.ie.export_item(i1) - self.assertRaises(UnicodeEncodeError, self.ie.export_item, i2) + with pytest.raises(UnicodeEncodeError): + self.ie.export_item(i2) self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue(), encoding="latin")) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 1620d2d41..b4c1b9631 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -233,7 +233,8 @@ class BlockingFeedStorageTest(unittest.TestCase): invalid_path = tests_path / "invalid_path" spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)}) - self.assertRaises(OSError, b.open, spider=spider) + with pytest.raises(OSError, match="Not a Directory:"): + b.open(spider=spider) @pytest.mark.requires_boto3 @@ -2437,7 +2438,8 @@ class BatchDeliveriesTest(FeedExportTestBase): "FEED_EXPORT_BATCH_ITEM_COUNT": 1, } crawler = get_crawler(settings_dict=settings) - self.assertRaises(NotConfigured, FeedExporter, crawler) + with pytest.raises(NotConfigured): + FeedExporter(crawler) @defer.inlineCallbacks def test_export_no_items_not_store_empty(self): @@ -2758,7 +2760,7 @@ class FeedExportInitTest(unittest.TestCase): }, } crawler = get_crawler(settings_dict=settings) - with self.assertRaises(NotConfigured): + with pytest.raises(NotConfigured): FeedExporter.from_crawler(crawler) def test_unsupported_format(self): @@ -2770,7 +2772,7 @@ class FeedExportInitTest(unittest.TestCase): }, } crawler = get_crawler(settings_dict=settings) - with self.assertRaises(NotConfigured): + with pytest.raises(NotConfigured): FeedExporter.from_crawler(crawler) def test_absolute_pathlib_as_uri(self): @@ -2863,7 +2865,7 @@ class URIParamsTest: with warnings.catch_warnings(): warnings.simplefilter("error", ScrapyDeprecationWarning) - with self.assertRaises(KeyError): + with pytest.raises(KeyError): feed_exporter.open_spider(spider) def test_params_as_is(self): diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index ddc772236..0881bbeca 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING from unittest import mock, skipIf from urllib.parse import urlencode +import pytest from twisted.internet import reactor from twisted.internet.defer import ( CancelledError, @@ -406,7 +407,7 @@ class Https2ClientProtocolTestCase(TestCase): "scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2" ): request = Request(url=self.get_url("/status?n=200")) - with self.assertRaises(ResponseFailed): + with pytest.raises(ResponseFailed): yield self.make_request(request) def test_cancel_request(self): @@ -560,7 +561,7 @@ class Https2ClientProtocolTestCase(TestCase): return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) def test_invalid_request_type(self): - with self.assertRaises(TypeError): + with pytest.raises(TypeError): self.make_request("https://InvalidDataTypePassed.com") def test_query_parameters(self): diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 7db1eb8c5..0bbbcda46 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -1,6 +1,8 @@ import copy import unittest +import pytest + from scrapy.http import Headers @@ -13,7 +15,8 @@ class HeadersTest(unittest.TestCase): assert h["Content-Type"] assert h["Content-Length"] - self.assertRaises(KeyError, h.__getitem__, "Accept") + with pytest.raises(KeyError): + h["Accept"] self.assertEqual(h.get("Accept"), None) self.assertEqual(h.getlist("Accept"), []) @@ -152,15 +155,11 @@ class HeadersTest(unittest.TestCase): self.assertEqual(h1.getlist("hey"), [b"5"]) def test_invalid_value(self): - self.assertRaisesRegex( - TypeError, "Unsupported value type", Headers, {"foo": object()} - ) - self.assertRaisesRegex( - TypeError, "Unsupported value type", Headers().__setitem__, "foo", object() - ) - self.assertRaisesRegex( - TypeError, "Unsupported value type", Headers().setdefault, "foo", object() - ) - self.assertRaisesRegex( - TypeError, "Unsupported value type", Headers().setlist, "foo", [object()] - ) + with pytest.raises(TypeError, match="Unsupported value type"): + Headers({"foo": object()}) + with pytest.raises(TypeError, match="Unsupported value type"): + Headers()["foo"] = object() + with pytest.raises(TypeError, match="Unsupported value type"): + Headers().setdefault("foo", object()) + with pytest.raises(TypeError, match="Unsupported value type"): + Headers().setlist("foo", [object()]) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index a8ab8240f..e5291157d 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -7,6 +7,8 @@ from typing import Any from unittest import mock from urllib.parse import parse_qs, unquote_to_bytes +import pytest + from scrapy.http import ( FormRequest, Headers, @@ -28,10 +30,12 @@ class RequestTest(unittest.TestCase): def test_init(self): # Request requires url in the __init__ method - self.assertRaises(Exception, self.request_class) + with pytest.raises(TypeError): + self.request_class() # url argument must be basestring - self.assertRaises(TypeError, self.request_class, 123) + with pytest.raises(TypeError): + self.request_class(123) r = self.request_class("http://www.example.com") r = self.request_class("http://www.example.com") @@ -64,9 +68,13 @@ class RequestTest(unittest.TestCase): self.request_class("data:,Hello%2C%20World!") def test_url_no_scheme(self): - self.assertRaises(ValueError, self.request_class, "foo") - self.assertRaises(ValueError, self.request_class, "/foo/") - self.assertRaises(ValueError, self.request_class, "/foo:bar") + msg = "Missing scheme in request url:" + with pytest.raises(ValueError, match=msg): + self.request_class("foo") + with pytest.raises(ValueError, match=msg): + self.request_class("/foo/") + with pytest.raises(ValueError, match=msg): + self.request_class("/foo:bar") def test_headers(self): # Different ways of setting headers attribute @@ -273,8 +281,10 @@ class RequestTest(unittest.TestCase): def test_immutable_attributes(self): r = self.request_class("http://example.com") - self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com") - self.assertRaises(AttributeError, setattr, r, "body", "xxx") + with pytest.raises(AttributeError): + r.url = "http://example2.com" + with pytest.raises(AttributeError): + r.body = "xxx" def test_callback_and_errback(self): def a_function(): @@ -309,11 +319,11 @@ class RequestTest(unittest.TestCase): self.assertIs(r5.errback, NO_CALLBACK) def test_callback_and_errback_type(self): - with self.assertRaises(TypeError): + with pytest.raises(TypeError): self.request_class("http://example.com", callback="a_function") - with self.assertRaises(TypeError): + with pytest.raises(TypeError): self.request_class("http://example.com", errback="a_function") - with self.assertRaises(TypeError): + with pytest.raises(TypeError): self.request_class( url="http://example.com", callback="a_function", @@ -321,7 +331,7 @@ class RequestTest(unittest.TestCase): ) def test_no_callback(self): - with self.assertRaises(RuntimeError): + with pytest.raises(RuntimeError): NO_CALLBACK() def test_from_curl(self): @@ -403,13 +413,11 @@ class RequestTest(unittest.TestCase): # If `ignore_unknown_options` is set to `False` it raises an error with # the unknown options: --foo and -z - self.assertRaises( - ValueError, - lambda: self.request_class.from_curl( + with pytest.raises(ValueError, match="Unrecognized options:"): + self.request_class.from_curl( 'curl -X PATCH "http://example.org" --foo -z', ignore_unknown_options=False, - ), - ) + ) class FormRequestTest(RequestTest): @@ -428,7 +436,7 @@ class FormRequestTest(RequestTest): data = (("a", "one"), ("a", "two"), ("b", "2")) url = self.request_class( "http://www.example.com/?a=0&b=1&c=3#fragment", method="GET", formdata=data - ).url.split("#")[0] + ).url.split("#", maxsplit=1)[0] fs = _qs(self.request_class(url, method="GET", formdata=data)) self.assertEqual(set(fs[b"a"]), {b"one", b"two"}) self.assertEqual(fs[b"b"], [b"2"]) @@ -897,12 +905,11 @@ class FormRequestTest(RequestTest): """ ) - self.assertRaises( + with pytest.raises( ValueError, - self.request_class.from_response, - response, - clickdata={"type": "submit"}, - ) + match="Multiple elements found .* matching the criteria in clickdata", + ): + self.request_class.from_response(response, clickdata={"type": "submit"}) def test_from_response_non_matching_clickdata(self): response = _buildresponse( @@ -910,12 +917,12 @@ class FormRequestTest(RequestTest): """ ) - self.assertRaises( - ValueError, - self.request_class.from_response, - response, - clickdata={"nonexistent": "notme"}, - ) + with pytest.raises( + ValueError, match="No clickable element matching clickdata:" + ): + self.request_class.from_response( + response, clickdata={"nonexistent": "notme"} + ) def test_from_response_nr_index_clickdata(self): response = _buildresponse( @@ -937,13 +944,15 @@ class FormRequestTest(RequestTest): """ ) - self.assertRaises( - ValueError, self.request_class.from_response, response, clickdata={"nr": 1} - ) + with pytest.raises( + ValueError, match="No clickable element matching clickdata:" + ): + self.request_class.from_response(response, clickdata={"nr": 1}) def test_from_response_errors_noform(self): response = _buildresponse("""""") - self.assertRaises(ValueError, self.request_class.from_response, response) + with pytest.raises(ValueError, match="No
element found in"): + self.request_class.from_response(response) def test_from_response_invalid_html5(self): response = _buildresponse( @@ -963,9 +972,8 @@ class FormRequestTest(RequestTest):
""" ) - self.assertRaises( - IndexError, self.request_class.from_response, response, formnumber=1 - ) + with pytest.raises(IndexError): + self.request_class.from_response(response, formnumber=1) def test_from_response_noformname(self): response = _buildresponse( @@ -1021,13 +1029,8 @@ class FormRequestTest(RequestTest): """ ) - self.assertRaises( - IndexError, - self.request_class.from_response, - response, - formname="form3", - formnumber=2, - ) + with pytest.raises(IndexError): + self.request_class.from_response(response, formname="form3", formnumber=2) def test_from_response_formid_exists(self): response = _buildresponse( @@ -1086,13 +1089,8 @@ class FormRequestTest(RequestTest): """ ) - self.assertRaises( - IndexError, - self.request_class.from_response, - response, - formid="form3", - formnumber=2, - ) + with pytest.raises(IndexError): + self.request_class.from_response(response, formid="form3", formnumber=2) def test_from_response_select(self): res = _buildresponse( @@ -1245,12 +1243,10 @@ class FormRequestTest(RequestTest): fs = _qs(r1) self.assertEqual(fs[b"three"], [b"3"]) - self.assertRaises( - ValueError, - self.request_class.from_response, - response, - formxpath="//form/input[@name='abc']", - ) + with pytest.raises(ValueError, match="No
element found with"): + self.request_class.from_response( + response, formxpath="//form/input[@name='abc']" + ) def test_from_response_unicode_xpath(self): response = _buildresponse(b'
') @@ -1261,13 +1257,8 @@ class FormRequestTest(RequestTest): self.assertEqual(fs, {}) xpath = "//form[@name='\u03b1']" - self.assertRaisesRegex( - ValueError, - re.escape(xpath), - self.request_class.from_response, - response, - formxpath=xpath, - ) + with pytest.raises(ValueError, match=re.escape(xpath)): + self.request_class.from_response(response, formxpath=xpath) def test_from_response_button_submit(self): response = _buildresponse( @@ -1393,12 +1384,8 @@ class FormRequestTest(RequestTest): fs = _qs(r1) self.assertEqual(fs[b"three"], [b"3"]) - self.assertRaises( - ValueError, - self.request_class.from_response, - response, - formcss="input[name='abc']", - ) + with pytest.raises(ValueError, match="No
element found with"): + self.request_class.from_response(response, formcss="input[name='abc']") def test_from_response_valid_form_methods(self): form_methods = [ @@ -1424,13 +1411,11 @@ class FormRequestTest(RequestTest):
""" ) - with self.assertRaises(ValueError) as context: + with pytest.raises( + ValueError, match="formdata should be a dict or iterable of tuples" + ): FormRequest.from_response(response, formdata=123) - self.assertIn( - "formdata should be a dict or iterable of tuples", str(context.exception) - ) - def test_form_response_with_custom_invalid_formdata_value_error(self): """Test that a ValueError is raised for fault-inducing iterable formdata input""" response = _buildresponse( @@ -1441,13 +1426,11 @@ class FormRequestTest(RequestTest): """ ) - with self.assertRaises(ValueError) as context: + with pytest.raises( + ValueError, match="formdata should be a dict or iterable of tuples" + ): FormRequest.from_response(response, formdata=("a",)) - self.assertIn( - "formdata should be a dict or iterable of tuples", str(context.exception) - ) - def test_get_form_with_xpath_no_form_parent(self): """Test that _get_from raised a ValueError when an XPath selects an element not nested within a
and no parent is found""" @@ -1462,11 +1445,9 @@ class FormRequestTest(RequestTest): """ ) - with self.assertRaises(ValueError) as context: + with pytest.raises(ValueError, match="No element found with"): FormRequest.from_response(response, formxpath='//div[@id="outside-form"]/p') - self.assertIn("No element found with", str(context.exception)) - def _buildresponse(body, **kwargs): kwargs.setdefault("body", body) @@ -1507,8 +1488,10 @@ class XmlRpcRequestTest(RequestTest): self._test_request(params=("response",), methodresponse="login") self._test_request(params=("pas£",), encoding="utf-8") self._test_request(params=(None,), allow_none=1) - self.assertRaises(TypeError, self._test_request) - self.assertRaises(TypeError, self._test_request, params=(None,)) + with pytest.raises(TypeError): + self._test_request() + with pytest.raises(TypeError): + self._test_request(params=(None,)) def test_latin1(self): self._test_request(params=("pas£",), encoding="latin1") diff --git a/tests/test_http_response.py b/tests/test_http_response.py index dde883451..5a943f084 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -27,14 +27,15 @@ class BaseResponseTest(unittest.TestCase): def test_init(self): # Response requires url in the constructor - self.assertRaises(Exception, self.response_class) + with pytest.raises(TypeError): + self.response_class() self.assertTrue( isinstance(self.response_class("http://example.com/"), self.response_class) ) - self.assertRaises(TypeError, self.response_class, b"http://example.com") - self.assertRaises( - TypeError, self.response_class, url="http://example.com", body={} - ) + with pytest.raises(TypeError): + self.response_class(b"http://example.com") + with pytest.raises(TypeError): + self.response_class(url="http://example.com", body={}) # body can be str or None self.assertTrue( isinstance( @@ -77,12 +78,8 @@ class BaseResponseTest(unittest.TestCase): self.assertEqual(r.status, 301) r = self.response_class("http://www.example.com", status="301") self.assertEqual(r.status, 301) - self.assertRaises( - ValueError, - self.response_class, - "http://example.com", - status="lala200", - ) + with pytest.raises(ValueError, match=r"invalid literal for int\(\)"): + self.response_class("http://example.com", status="lala200") def test_copy(self): """Test Response copy""" @@ -122,14 +119,12 @@ class BaseResponseTest(unittest.TestCase): def test_unavailable_meta(self): r1 = self.response_class("http://www.example.com", body=b"Some body") - with self.assertRaisesRegex(AttributeError, r"Response\.meta not available"): + with pytest.raises(AttributeError, match=r"Response\.meta not available"): r1.meta def test_unavailable_cb_kwargs(self): r1 = self.response_class("http://www.example.com", body=b"Some body") - with self.assertRaisesRegex( - AttributeError, r"Response\.cb_kwargs not available" - ): + with pytest.raises(AttributeError, match=r"Response\.cb_kwargs not available"): r1.cb_kwargs def test_copy_inherited_classes(self): @@ -179,8 +174,10 @@ class BaseResponseTest(unittest.TestCase): def test_immutable_attributes(self): r = self.response_class("http://example.com") - self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com") - self.assertRaises(AttributeError, setattr, r, "body", "xxx") + with pytest.raises(AttributeError): + r.url = "http://example2.com" + with pytest.raises(AttributeError): + r.body = "xxx" def test_urljoin(self): """Test urljoin shortcut (only for existence, since behavior equals urljoin)""" @@ -192,10 +189,14 @@ class BaseResponseTest(unittest.TestCase): r = self.response_class("http://example.com", body=b"hello") if self.response_class == Response: msg = "Response content isn't text" - self.assertRaisesRegex(AttributeError, msg, getattr, r, "text") - self.assertRaisesRegex(NotSupported, msg, r.css, "body") - self.assertRaisesRegex(NotSupported, msg, r.xpath, "//body") - self.assertRaisesRegex(NotSupported, msg, r.jmespath, "body") + with pytest.raises(AttributeError, match=msg): + r.text + with pytest.raises(NotSupported, match=msg): + r.css("body") + with pytest.raises(NotSupported, match=msg): + r.xpath("//body") + with pytest.raises(NotSupported, match=msg): + r.jmespath("body") else: r.text r.css("body") @@ -216,7 +217,8 @@ class BaseResponseTest(unittest.TestCase): def test_follow_None_url(self): r = self.response_class("http://example.com") - self.assertRaises(ValueError, r.follow, None) + with pytest.raises(ValueError, match="url can't be None"): + r.follow(None) @pytest.mark.xfail( parse_version(w3lib_version) < parse_version("2.1.1"), @@ -279,18 +281,20 @@ class BaseResponseTest(unittest.TestCase): def test_follow_all_invalid(self): r = self.response_class("http://example.com") if self.response_class == Response: - with self.assertRaises(TypeError): + with pytest.raises(TypeError): list(r.follow_all(urls=None)) - with self.assertRaises(TypeError): + with pytest.raises(TypeError): list(r.follow_all(urls=12345)) - with self.assertRaises(ValueError): + with pytest.raises(ValueError, match="url can't be None"): list(r.follow_all(urls=[None])) else: - with self.assertRaises(ValueError): + with pytest.raises( + ValueError, match="Please supply exactly one of the following arguments" + ): list(r.follow_all(urls=None)) - with self.assertRaises(TypeError): + with pytest.raises(TypeError): list(r.follow_all(urls=12345)) - with self.assertRaises(ValueError): + with pytest.raises(ValueError, match="url can't be None"): list(r.follow_all(urls=[None])) def test_follow_all_whitespace(self): @@ -399,12 +403,8 @@ class TextResponseTest(BaseResponseTest): "\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 " "\u0442\u0435\u043a\u0441\u0442" ) - self.assertRaises( - TypeError, - self.response_class, - "http://www.example.com", - body="unicode body", - ) + with pytest.raises(TypeError): + self.response_class("http://www.example.com", body="unicode body") original_string = unicode_string.encode("cp1251") r1 = self.response_class( @@ -483,12 +483,8 @@ class TextResponseTest(BaseResponseTest): self._assert_response_values(r9, "cp1252", "€") # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies - self.assertRaises( - TypeError, - self.response_class, - "http://www.example.com", - body="\xa3", - ) + with pytest.raises(TypeError): + self.response_class("http://www.example.com", body="\xa3") def test_declared_encoding_invalid(self): """Check that unknown declared encodings are ignored""" @@ -679,20 +675,20 @@ class TextResponseTest(BaseResponseTest): self._assert_followed_url(sel, url, response=resp) # non-a elements are not supported - self.assertRaises(ValueError, resp.follow, resp.css("div")[0]) + with pytest.raises( + ValueError, match="Only and elements are supported" + ): + resp.follow(resp.css("div")[0]) def test_follow_selector_list(self): resp = self._links_response() - self.assertRaisesRegex(ValueError, "SelectorList", resp.follow, resp.css("a")) + with pytest.raises(ValueError, match="SelectorList"): + resp.follow(resp.css("a")) def test_follow_selector_invalid(self): resp = self._links_response() - self.assertRaisesRegex( - ValueError, - "Unsupported", - resp.follow, - resp.xpath("count(//div)")[0], - ) + with pytest.raises(ValueError, match="Unsupported"): + resp.follow(resp.xpath("count(//div)")[0]) def test_follow_selector_attribute(self): resp = self._links_response() @@ -704,7 +700,8 @@ class TextResponseTest(BaseResponseTest): url="http://example.com", body=b"click me", ) - self.assertRaisesRegex(ValueError, "no href", resp.follow, resp.css("a")[0]) + with pytest.raises(ValueError, match="no href"): + resp.follow(resp.css("a")[0]) def test_follow_whitespace_selector(self): resp = self.response_class( @@ -812,7 +809,9 @@ class TextResponseTest(BaseResponseTest): def test_follow_all_too_many_arguments(self): response = self._links_response() - with self.assertRaises(ValueError): + with pytest.raises( + ValueError, match="Please supply exactly one of the following arguments" + ): response.follow_all( css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]', @@ -825,7 +824,9 @@ class TextResponseTest(BaseResponseTest): text_body = b"""text""" text_response = self.response_class("http://www.example.com", body=text_body) - with self.assertRaises(ValueError): + with pytest.raises( + ValueError, match="(Expecting value|Unexpected '<'): line 1" + ): text_response.json() def test_cache_json_response(self): @@ -1023,10 +1024,8 @@ class CustomResponseTest(TextResponseTest): self.assertEqual(r4.bar, "bar") self.assertIsNone(r4.lost) - with self.assertRaises(TypeError) as ctx: + with pytest.raises( + TypeError, + match=r"__init__\(\) got an unexpected keyword argument 'unknown'", + ): r1.replace(unknown="unknown") - self.assertTrue( - str(ctx.exception).endswith( - "__init__() got an unexpected keyword argument 'unknown'" - ) - ) diff --git a/tests/test_item.py b/tests/test_item.py index 0399c8f8d..47c5c3db6 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -2,6 +2,8 @@ import unittest from abc import ABCMeta from unittest import mock +import pytest + from scrapy.item import Field, Item, ItemMeta @@ -22,7 +24,8 @@ class ItemTest(unittest.TestCase): name = Field() i = TestItem() - self.assertRaises(KeyError, i.__getitem__, "name") + with pytest.raises(KeyError): + i["name"] i2 = TestItem(name="john doe") self.assertEqual(i2["name"], "john doe") @@ -33,15 +36,18 @@ class ItemTest(unittest.TestCase): i4 = TestItem(i3) self.assertEqual(i4["name"], "john doe") - self.assertRaises(KeyError, TestItem, {"name": "john doe", "other": "foo"}) + with pytest.raises(KeyError): + TestItem({"name": "john doe", "other": "foo"}) def test_invalid_field(self): class TestItem(Item): pass i = TestItem() - self.assertRaises(KeyError, i.__setitem__, "field", "text") - self.assertRaises(KeyError, i.__getitem__, "field") + with pytest.raises(KeyError): + i["field"] = "text" + with pytest.raises(KeyError): + i["field"] def test_repr(self): class TestItem(Item): @@ -72,14 +78,16 @@ class ItemTest(unittest.TestCase): name = Field() i = TestItem() - self.assertRaises(AttributeError, getattr, i, "name") + with pytest.raises(AttributeError): + i.name def test_raise_setattr(self): class TestItem(Item): name = Field() i = TestItem() - self.assertRaises(AttributeError, setattr, i, "name", "john") + with pytest.raises(AttributeError): + i.name = "john" def test_custom_methods(self): class TestItem(Item): @@ -92,7 +100,8 @@ class ItemTest(unittest.TestCase): self["name"] = name i = TestItem() - self.assertRaises(KeyError, i.get_name) + with pytest.raises(KeyError): + i.get_name() i["name"] = "lala" self.assertEqual(i.get_name(), "lala") i.change_name("other") @@ -223,7 +232,8 @@ class ItemTest(unittest.TestCase): class D(B, C): pass - self.assertRaises(KeyError, D, not_allowed="value") + with pytest.raises(KeyError): + D(not_allowed="value") self.assertEqual(D(save="X")["save"], "X") self.assertEqual(D.fields, {"save": {"default": "A"}, "load": {"default": "A"}}) @@ -231,7 +241,8 @@ class ItemTest(unittest.TestCase): class E(C, B): pass - self.assertRaises(KeyError, E, not_allowed="value") + with pytest.raises(KeyError): + E(not_allowed="value") self.assertEqual(E(save="X")["save"], "X") self.assertEqual(E.fields, {"save": {"default": "A"}, "load": {"default": "A"}}) diff --git a/tests/test_link.py b/tests/test_link.py index 35723bbd6..ed9d27a37 100644 --- a/tests/test_link.py +++ b/tests/test_link.py @@ -1,5 +1,7 @@ import unittest +import pytest + from scrapy.link import Link @@ -53,5 +55,5 @@ class LinkTest(unittest.TestCase): self._assert_same_links(l1, l2) def test_bytes_url(self): - with self.assertRaises(TypeError): + with pytest.raises(TypeError): Link(b"http://www.example.com/\xc2\xa3") diff --git a/tests/test_loader.py b/tests/test_loader.py index b52d5ea2e..1a933bb8d 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -4,6 +4,7 @@ import dataclasses import unittest import attr +import pytest from itemadapter import ItemAdapter from itemloaders.processors import Compose, Identity, MapCompose, TakeFirst @@ -69,7 +70,8 @@ def processor_with_args(value, other=None, loader_context=None): class BasicItemLoaderTest(unittest.TestCase): def test_add_value_on_unknown_field(self): il = ProcessorItemLoader() - self.assertRaises(KeyError, il.add_value, "wrong_field", ["lala", "lolo"]) + with pytest.raises(KeyError): + il.add_value("wrong_field", ["lala", "lolo"]) def test_load_item_using_default_loader(self): i = SummaryItem() @@ -294,12 +296,18 @@ class SelectortemLoaderTest(unittest.TestCase): def test_init_method_errors(self): l = ProcessorItemLoader() - self.assertRaises(RuntimeError, l.add_xpath, "url", "//a/@href") - self.assertRaises(RuntimeError, l.replace_xpath, "url", "//a/@href") - self.assertRaises(RuntimeError, l.get_xpath, "//a/@href") - self.assertRaises(RuntimeError, l.add_css, "name", "#name::text") - self.assertRaises(RuntimeError, l.replace_css, "name", "#name::text") - self.assertRaises(RuntimeError, l.get_css, "#name::text") + with pytest.raises(RuntimeError): + l.add_xpath("url", "//a/@href") + with pytest.raises(RuntimeError): + l.replace_xpath("url", "//a/@href") + with pytest.raises(RuntimeError): + l.get_xpath("//a/@href") + with pytest.raises(RuntimeError): + l.add_css("name", "#name::text") + with pytest.raises(RuntimeError): + l.replace_css("name", "#name::text") + with pytest.raises(RuntimeError): + l.get_css("#name::text") def test_init_method_with_selector(self): sel = Selector(text="
marta
") diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 1e504f539..0d7921b1d 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -6,6 +6,7 @@ Once we remove the references from scrapy, we can remove these tests. import unittest from functools import partial +import pytest from itemloaders.processors import ( Compose, Identity, @@ -435,7 +436,13 @@ class BasicItemLoaderTest(unittest.TestCase): name_in = MapCompose(float) il = TestItemLoader() - self.assertRaises(ValueError, il.add_value, "name", ["marta", "other"]) + with pytest.raises( + ValueError, + match="Error with input processor MapCompose: .* " + "error='ValueError: Error in MapCompose .* " + "error='ValueError: could not convert", + ): + il.add_value("name", ["marta", "other"]) def test_error_output_processor(self): class TestItem(Item): @@ -447,7 +454,12 @@ class BasicItemLoaderTest(unittest.TestCase): il = TestItemLoader() il.add_value("name", "marta") - with self.assertRaises(ValueError): + with pytest.raises( + ValueError, + match="Error with output processor: .* " + "error='ValueError: Error in Compose .* " + "error='ValueError: could not convert", + ): il.load_item() def test_error_processor_as_argument(self): @@ -458,9 +470,13 @@ class BasicItemLoaderTest(unittest.TestCase): default_item_class = TestItem il = TestItemLoader() - self.assertRaises( - ValueError, il.add_value, "name", ["marta", "other"], Compose(float) - ) + with pytest.raises( + ValueError, + match=r"Error with processor Compose .* " + r"error='ValueError: Error in Compose .* " + r"error='TypeError: float\(\) argument", + ): + il.add_value("name", ["marta", "other"], Compose(float)) class InitializationFromDictTest(unittest.TestCase): @@ -630,7 +646,8 @@ class ProcessorsTest(unittest.TestCase): def test_join(self): proc = Join() - self.assertRaises(TypeError, proc, [None, "", "hello", "world"]) + with pytest.raises(TypeError): + proc([None, "", "hello", "world"]) self.assertEqual(proc(["", "hello", "world"]), " hello world") self.assertEqual(proc(["hello", "world"]), "hello world") self.assertIsInstance(proc(["hello", "world"]), str) @@ -641,9 +658,17 @@ class ProcessorsTest(unittest.TestCase): proc = Compose(str.upper) self.assertEqual(proc(None), None) proc = Compose(str.upper, stop_on_none=False) - self.assertRaises(ValueError, proc, None) + with pytest.raises( + ValueError, + match="Error in Compose with .* error='TypeError: (descriptor 'upper'|'str' object expected)", + ): + proc(None) proc = Compose(str.upper, lambda x: x + 1) - self.assertRaises(ValueError, proc, "hello") + with pytest.raises( + ValueError, + match="Error in Compose with .* error='TypeError: (can only|unsupported operand)", + ): + proc("hello") def test_mapcompose(self): def filter_world(x): @@ -657,9 +682,17 @@ class ProcessorsTest(unittest.TestCase): proc = MapCompose(filter_world, str.upper) self.assertEqual(proc(None), []) proc = MapCompose(filter_world, str.upper) - self.assertRaises(ValueError, proc, [1]) + with pytest.raises( + ValueError, + match="Error in MapCompose with .* error='TypeError: (descriptor 'upper'|'str' object expected)", + ): + proc([1]) proc = MapCompose(filter_world, lambda x: x + 1) - self.assertRaises(ValueError, proc, "hello") + with pytest.raises( + ValueError, + match="Error in MapCompose with .* error='TypeError: (can only|unsupported operand)", + ): + proc("hello") class SelectJmesTestCase(unittest.TestCase): diff --git a/tests/test_logstats.py b/tests/test_logstats.py index a4b002e34..6bc5b6f1f 100644 --- a/tests/test_logstats.py +++ b/tests/test_logstats.py @@ -1,6 +1,8 @@ import unittest from datetime import datetime +import pytest + from scrapy.extensions.logstats import LogStats from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider @@ -18,8 +20,9 @@ class TestLogStats(unittest.TestCase): def test_stats_calculations(self): logstats = LogStats.from_crawler(self.crawler) - with self.assertRaises(AttributeError): + with pytest.raises(AttributeError): logstats.pagesprev + with pytest.raises(AttributeError): logstats.itemsprev logstats.spider_opened(self.spider) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 3d049843a..1d89e44ce 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -7,6 +7,7 @@ from shutil import rmtree from tempfile import mkdtemp import attr +import pytest from itemadapter import ItemAdapter from twisted.trial import unittest @@ -146,11 +147,11 @@ class ImagesPipelineTestCase(unittest.TestCase): resp3 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf3.getvalue()) req = Request(url="https://dev.mydeco.com/mydeco.gif") - with self.assertRaises(ImageException): + with pytest.raises(ImageException): next(self.pipeline.get_images(response=resp1, request=req, info=object())) - with self.assertRaises(ImageException): + with pytest.raises(ImageException): next(self.pipeline.get_images(response=resp2, request=req, info=object())) - with self.assertRaises(ImageException): + with pytest.raises(ImageException): next(self.pipeline.get_images(response=resp3, request=req, info=object())) def test_get_images(self): diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index 1584014b8..c223c4562 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -1,6 +1,7 @@ import tempfile import unittest +import pytest import queuelib from scrapy.http.request import Request @@ -40,9 +41,9 @@ class PriorityQueueTest(unittest.TestCase): self.crawler, FifoMemoryQueue, temp_dir ) queue.push(Request("https://example.org")) - with self.assertRaises( + with pytest.raises( NotImplementedError, - msg="The underlying queue class does not implement 'peek'", + match="The underlying queue class does not implement 'peek'", ): queue.peek() queue.close() @@ -129,9 +130,9 @@ class DownloaderAwarePriorityQueueTest(unittest.TestCase): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined") self.queue.push(Request("https://example.org")) - with self.assertRaises( + with pytest.raises( NotImplementedError, - msg="The underlying queue class does not implement 'peek'", + match="The underlying queue class does not implement 'peek'", ): self.queue.peek() diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index 85133038a..2c605a015 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -1,5 +1,7 @@ import unittest +import pytest + from scrapy import Request, Spider from scrapy.http import FormRequest, JsonRequest from scrapy.utils.request import request_from_dict @@ -134,11 +136,15 @@ class RequestSerializationTest(unittest.TestCase): def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) - self.assertRaises(ValueError, r.to_dict, spider=self.spider) + with pytest.raises( + ValueError, match="is not an instance method in: " - ), + match=r"\.process_spider_output must return an iterable, got ", ): yield self._get_middleware_result( ProcessSpiderOutputNonIterableMiddleware, @@ -312,9 +310,9 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): @defer.inlineCallbacks def test_coroutine(self): - with self.assertRaisesRegex( + with pytest.raises( _InvalidOutput, - r"\.process_spider_output must be an asynchronous generator", + match=r"\.process_spider_output must be an asynchronous generator", ): yield self._get_middleware_result( ProcessSpiderOutputCoroutineMiddleware, @@ -518,8 +516,8 @@ class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase): @defer.inlineCallbacks def _test_asyncgen_nodowngrade(self, *mw_classes): - with self.assertRaisesRegex( - _InvalidOutput, "Async iterable returned from .+ cannot be downgraded" + with pytest.raises( + _InvalidOutput, match="Async iterable returned from .+ cannot be downgraded" ): yield self._get_middleware_result(*mw_classes) diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 307054de7..f9eb93d6b 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,6 +1,7 @@ import logging from unittest import TestCase +import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase as TrialTestCase @@ -68,9 +69,8 @@ class TestHttpErrorMiddleware(TestCase): def test_process_spider_input(self): self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) - self.assertRaises( - HttpError, self.mw.process_spider_input, self.res404, self.spider - ) + with pytest.raises(HttpError): + self.mw.process_spider_input(self.res404, self.spider) def test_process_spider_exception(self): self.assertEqual( @@ -105,9 +105,8 @@ class TestHttpErrorMiddlewareSettings(TestCase): def test_process_spider_input(self): self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) - self.assertRaises( - HttpError, self.mw.process_spider_input, self.res404, self.spider - ) + with pytest.raises(HttpError): + self.mw.process_spider_input(self.res404, self.spider) self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider)) def test_meta_overrides_settings(self): @@ -120,14 +119,14 @@ class TestHttpErrorMiddlewareSettings(TestCase): res402.request = request self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) - self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) + with pytest.raises(HttpError): + self.mw.process_spider_input(res402, self.spider) def test_spider_override_settings(self): self.spider.handle_httpstatus_list = [404] self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) - self.assertRaises( - HttpError, self.mw.process_spider_input, self.res402, self.spider - ) + with pytest.raises(HttpError): + self.mw.process_spider_input(self.res402, self.spider) class TestHttpErrorMiddlewareHandleAll(TestCase): @@ -151,7 +150,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): res402.request = request self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) - self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) + with pytest.raises(HttpError): + self.mw.process_spider_input(res402, self.spider) def test_httperror_allow_all_false(self): crawler = get_crawler(_HttpErrorSpider) @@ -167,7 +167,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): res402 = self.res402.copy() res402.request = request_httpstatus_true - self.assertRaises(HttpError, mw.process_spider_input, res404, self.spider) + with pytest.raises(HttpError): + mw.process_spider_input(res404, self.spider) self.assertIsNone(mw.process_spider_input(res402, self.spider)) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 4945ac25d..01a87c645 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -5,6 +5,8 @@ from typing import Any from unittest import TestCase from urllib.parse import urlparse +import pytest + from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.http import Request, Response from scrapy.settings import Settings @@ -884,7 +886,7 @@ class TestSettingsPolicyByName(TestCase): def test_invalid_name(self): settings = Settings({"REFERRER_POLICY": "some-custom-unknown-policy"}) - with self.assertRaises(RuntimeError): + with pytest.raises(RuntimeError): RefererMiddleware(settings) def test_multiple_policy_tokens(self): @@ -925,7 +927,7 @@ class TestSettingsPolicyByName(TestCase): ) } ) - with self.assertRaises(RuntimeError): + with pytest.raises(RuntimeError): RefererMiddleware(settings) diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index 59d18d92e..72692afab 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -2,6 +2,7 @@ import shutil from datetime import datetime, timezone from tempfile import mkdtemp +import pytest from twisted.trial import unittest from scrapy.exceptions import NotConfigured @@ -42,4 +43,5 @@ class SpiderStateTest(unittest.TestCase): def test_not_configured(self): crawler = get_crawler(Spider) - self.assertRaises(NotConfigured, SpiderState.from_crawler, crawler) + with pytest.raises(NotConfigured): + SpiderState.from_crawler(crawler) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index a2e7ae65d..8556b75dd 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -1,6 +1,7 @@ import pickle import sys +import pytest from queuelib.tests import test_queue as t from scrapy.http import Request @@ -30,10 +31,17 @@ class MyLoader(ItemLoader): def nonserializable_object_test(self): q = self.queue() - self.assertRaises(ValueError, q.push, lambda x: x) + with pytest.raises( + ValueError, + match="unmarshallable object|Can't (get|pickle) local object|Can't pickle .*: it's not found as", + ): + q.push(lambda x: x) # Selectors should fail (lxml.html.HtmlElement objects can't be pickled) sel = Selector(text="

some text

") - self.assertRaises(ValueError, q.push, sel) + with pytest.raises( + ValueError, match="unmarshallable object|can't pickle Selector objects" + ): + q.push(sel) class FifoDiskQueueTestMixin: diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index 04eeae4dc..88f6657d8 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -6,6 +6,7 @@ import shutil import tempfile import unittest +import pytest import queuelib from scrapy.http import Request @@ -69,9 +70,9 @@ class RequestQueueTestMixin: req = Request("http://www.example.com") q.push(req) self.assertEqual(len(q), 1) - with self.assertRaises( + with pytest.raises( NotImplementedError, - msg="The underlying queue class does not implement 'peek'", + match="The underlying queue class does not implement 'peek'", ): q.peek() self.assertEqual(q.pop().url, req.url) @@ -120,9 +121,9 @@ class FifoQueueMixin(RequestQueueTestMixin): q.push(req1) q.push(req2) q.push(req3) - with self.assertRaises( + with pytest.raises( NotImplementedError, - msg="The underlying queue class does not implement 'peek'", + match="The underlying queue class does not implement 'peek'", ): q.peek() self.assertEqual(len(q), 3) @@ -176,9 +177,9 @@ class LifoQueueMixin(RequestQueueTestMixin): q.push(req1) q.push(req2) q.push(req3) - with self.assertRaises( + with pytest.raises( NotImplementedError, - msg="The underlying queue class does not implement 'peek'", + match="The underlying queue class does not implement 'peek'", ): q.peek() self.assertEqual(len(q), 3) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index cbea41129..e27bb7b74 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,5 +1,7 @@ import unittest +import pytest + from scrapy.exceptions import UsageError from scrapy.settings import BaseSettings, Settings from scrapy.utils.conf import ( @@ -32,7 +34,9 @@ class BuildComponentListTest(unittest.TestCase): ) # Same priority raises ValueError duplicate_bs.set("ONE", duplicate_bs["ONE"], priority=20) - with self.assertRaises(ValueError): + with pytest.raises( + ValueError, match="Some paths in .* convert to the same object" + ): build_component_list(duplicate_bs, convert=lambda x: x.lower()) def test_valid_numbers(self): @@ -58,21 +62,13 @@ class UtilsConfTestCase(unittest.TestCase): class FeedExportConfigTestCase(unittest.TestCase): def test_feed_export_config_invalid_format(self): settings = Settings() - self.assertRaises( - UsageError, - feed_process_params_from_cli, - settings, - ["items.dat"], - ) + with pytest.raises(UsageError): + feed_process_params_from_cli(settings, ["items.dat"]) def test_feed_export_config_mismatch(self): settings = Settings() - self.assertRaises( - UsageError, - feed_process_params_from_cli, - settings, - ["items1.dat", "items2.dat"], - ) + with pytest.raises(UsageError): + feed_process_params_from_cli(settings, ["items1.dat", "items2.dat"]) def test_feed_export_config_explicit_formats(self): settings = Settings() @@ -117,11 +113,9 @@ class FeedExportConfigTestCase(unittest.TestCase): ) def test_output_and_overwrite_output(self): - with self.assertRaises(UsageError): + with pytest.raises(UsageError): feed_process_params_from_cli( - Settings(), - ["output1.json"], - overwrite_output=["output2.json"], + Settings(), ["output1.json"], overwrite_output=["output2.json"] ) def test_feed_complete_default_values_from_settings_empty(self): diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index 5d99161bf..a5b438645 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -1,6 +1,7 @@ import unittest import warnings +import pytest from w3lib.http import basic_auth_header from scrapy import Request @@ -205,11 +206,11 @@ class CurlToRequestKwargsTest(unittest.TestCase): self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) def test_too_few_arguments_error(self): - self.assertRaisesRegex( + with pytest.raises( ValueError, - r"too few arguments|the following arguments are required:\s*url", - lambda: curl_to_request_kwargs("curl"), - ) + match=r"too few arguments|the following arguments are required:\s*url", + ): + curl_to_request_kwargs("curl") def test_ignore_unknown_options(self): # case 1: ignore_unknown_options=True: @@ -220,16 +221,11 @@ class CurlToRequestKwargsTest(unittest.TestCase): self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) # case 2: ignore_unknown_options=False (raise exception): - self.assertRaisesRegex( - ValueError, - "Unrecognized options:.*--bar.*--baz", - lambda: curl_to_request_kwargs( + with pytest.raises(ValueError, match="Unrecognized options:.*--bar.*--baz"): + curl_to_request_kwargs( "curl --bar --baz http://www.example.com", ignore_unknown_options=False - ), - ) + ) def test_must_start_with_curl_error(self): - self.assertRaises( - ValueError, - lambda: curl_to_request_kwargs("carl -X POST http://example.org"), - ) + with pytest.raises(ValueError, match="A curl command must start"): + curl_to_request_kwargs("carl -X POST http://example.org") diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index fadbc6daa..2e35d339a 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -87,8 +87,10 @@ class CaseInsensitiveDictMixin: def test_delete(self): d = self.dict_class({"key_lower": 1}) del d["key_LOWER"] - self.assertRaises(KeyError, d.__getitem__, "key_LOWER") - self.assertRaises(KeyError, d.__getitem__, "key_lower") + with pytest.raises(KeyError): + d["key_LOWER"] + with pytest.raises(KeyError): + d["key_lower"] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_getdefault(self): @@ -138,7 +140,8 @@ class CaseInsensitiveDictMixin: d = self.dict_class() d["a"] = 1 self.assertEqual(d.pop("A"), 1) - self.assertRaises(KeyError, d.pop, "A") + with pytest.raises(KeyError): + d.pop("A") def test_normkey(self): class MyDict(self.dict_class): @@ -279,8 +282,8 @@ class SequenceExcludeTest(unittest.TestCase): self.assertIn(set("bar"), d) # supplied sequence is a set, so checking for list (non)inclusion fails - self.assertRaises(TypeError, (0, 1, 2) in d) - self.assertRaises(TypeError, d.__contains__, ["a", "b", "c"]) + with pytest.raises(TypeError): + ["a", "b", "c"] in d # noqa: B015 for v in [-3, "test", 1.1]: self.assertNotIn(v, d) diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index dc5fbd3c3..e917b6947 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -3,6 +3,8 @@ import unittest import warnings from unittest import mock +import pytest + from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.deprecate import create_deprecated_class, update_classpath @@ -181,7 +183,8 @@ class WarnWhenSubclassedTest(unittest.TestCase): assert not issubclass(OutdatedUserClass1, OutdatedUserClass1a) assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1) - self.assertRaises(TypeError, issubclass, object(), DeprecatedName) + with pytest.raises(TypeError): + issubclass(object(), DeprecatedName) def test_isinstance(self): with warnings.catch_warnings(): diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 7b7a25db8..d40cae9c7 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -1,6 +1,8 @@ import unittest +from gzip import BadGzipFile from pathlib import Path +import pytest from w3lib.encoding import html_to_unicode from scrapy.http import Response @@ -27,9 +29,8 @@ class GunzipTest(unittest.TestCase): assert text.endswith(b"