diff --git a/tests/test_http_request.py b/tests/test_http_request.py index e5291157d..6bf0b8e3f 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,6 +1,5 @@ import json import re -import unittest import warnings import xmlrpc.client from typing import Any @@ -22,7 +21,7 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode -class RequestTest(unittest.TestCase): +class TestRequest: request_class = Request default_method = "GET" default_headers: dict[bytes, list[bytes]] = {} @@ -40,12 +39,12 @@ class RequestTest(unittest.TestCase): r = self.request_class("http://www.example.com") assert isinstance(r.url, str) - self.assertEqual(r.url, "http://www.example.com") - self.assertEqual(r.method, self.default_method) + assert r.url == "http://www.example.com" + assert r.method == self.default_method assert isinstance(r.headers, Headers) - self.assertEqual(r.headers, self.default_headers) - self.assertEqual(r.meta, self.default_meta) + assert r.headers == self.default_headers + assert r.meta == self.default_meta meta = {"lala": "lolo"} headers = {b"caca": b"coco"} @@ -54,9 +53,9 @@ class RequestTest(unittest.TestCase): ) assert r.meta is not meta - self.assertEqual(r.meta, meta) + assert r.meta == meta assert r.headers is not headers - self.assertEqual(r.headers[b"caca"], b"coco") + assert r.headers[b"caca"] == b"coco" def test_url_scheme(self): # This test passes by not raising any (ValueError) exception @@ -83,61 +82,61 @@ class RequestTest(unittest.TestCase): r = self.request_class(url=url, headers=headers) p = self.request_class(url=url, headers=r.headers) - self.assertEqual(r.headers, p.headers) - self.assertFalse(r.headers is headers) - self.assertFalse(p.headers is r.headers) + assert r.headers == p.headers + assert r.headers is not headers + assert p.headers is not r.headers # headers must not be unicode h = Headers({"key1": "val1", "key2": "val2"}) h["newkey"] = "newval" for k, v in h.items(): - self.assertIsInstance(k, bytes) + assert isinstance(k, bytes) for s in v: - self.assertIsInstance(s, bytes) + assert isinstance(s, bytes) def test_eq(self): url = "http://www.scrapy.org" r1 = self.request_class(url=url) r2 = self.request_class(url=url) - self.assertNotEqual(r1, r2) + assert r1 != r2 set_ = set() set_.add(r1) set_.add(r2) - self.assertEqual(len(set_), 2) + assert len(set_) == 2 def test_url(self): r = self.request_class(url="http://www.scrapy.org/path") - self.assertEqual(r.url, "http://www.scrapy.org/path") + assert r.url == "http://www.scrapy.org/path" def test_url_quoting(self): r = self.request_class(url="http://www.scrapy.org/blank%20space") - self.assertEqual(r.url, "http://www.scrapy.org/blank%20space") + assert r.url == "http://www.scrapy.org/blank%20space" r = self.request_class(url="http://www.scrapy.org/blank space") - self.assertEqual(r.url, "http://www.scrapy.org/blank%20space") + assert r.url == "http://www.scrapy.org/blank%20space" def test_url_encoding(self): r = self.request_class(url="http://www.scrapy.org/price/£") - self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") + assert r.url == "http://www.scrapy.org/price/%C2%A3" def test_url_encoding_other(self): # encoding affects only query part of URI, not path # path part should always be UTF-8 encoded before percent-escaping r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8") - self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") + assert r.url == "http://www.scrapy.org/price/%C2%A3" r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1") - self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") + assert r.url == "http://www.scrapy.org/price/%C2%A3" def test_url_encoding_query(self): r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ") - self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") + assert r1.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5" # should be same as above r2 = self.request_class( url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8" ) - self.assertEqual(r2.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") + assert r2.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5" def test_url_encoding_query_latin1(self): # encoding is used for encoding query-string before percent-escaping; @@ -145,7 +144,7 @@ class RequestTest(unittest.TestCase): r3 = self.request_class( url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1" ) - self.assertEqual(r3.url, "http://www.scrapy.org/price/%C2%B5?currency=%A3") + assert r3.url == "http://www.scrapy.org/price/%C2%B5?currency=%A3" def test_url_encoding_nonutf8_untouched(self): # percent-escaping sequences that do not match valid UTF-8 sequences @@ -164,16 +163,16 @@ class RequestTest(unittest.TestCase): # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different # URI from "http://www.example.org/r%E9sum%E9.html". r1 = self.request_class(url="http://www.scrapy.org/price/%a3") - self.assertEqual(r1.url, "http://www.scrapy.org/price/%a3") + assert r1.url == "http://www.scrapy.org/price/%a3" r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") - self.assertEqual(r2.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") + assert r2.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3" r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3") - self.assertEqual(r3.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") + assert r3.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3" r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html") - self.assertEqual(r4.url, "http://www.example.org/r%E9sum%E9.html") + assert r4.url == "http://www.example.org/r%E9sum%E9.html" def test_body(self): r1 = self.request_class(url="http://www.example.com/") @@ -181,19 +180,19 @@ class RequestTest(unittest.TestCase): r2 = self.request_class(url="http://www.example.com/", body=b"") assert isinstance(r2.body, bytes) - self.assertEqual(r2.encoding, "utf-8") # default encoding + assert r2.encoding == "utf-8" # default encoding r3 = self.request_class( url="http://www.example.com/", body="Price: \xa3100", encoding="utf-8" ) assert isinstance(r3.body, bytes) - self.assertEqual(r3.body, b"Price: \xc2\xa3100") + assert r3.body == b"Price: \xc2\xa3100" r4 = self.request_class( url="http://www.example.com/", body="Price: \xa3100", encoding="latin1" ) assert isinstance(r4.body, bytes) - self.assertEqual(r4.body, b"Price: \xa3100") + assert r4.body == b"Price: \xa3100" def test_copy(self): """Test Request copy""" @@ -219,25 +218,25 @@ class RequestTest(unittest.TestCase): # make sure flags list is shallow copied assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" - self.assertEqual(r1.flags, r2.flags) + assert r1.flags == r2.flags # make sure cb_kwargs dict is shallow copied assert r1.cb_kwargs is not r2.cb_kwargs, ( "cb_kwargs must be a shallow copy, not identical" ) - self.assertEqual(r1.cb_kwargs, r2.cb_kwargs) + assert r1.cb_kwargs == r2.cb_kwargs # make sure meta dict is shallow copied assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical" - self.assertEqual(r1.meta, r2.meta) + assert r1.meta == r2.meta # make sure headers attribute is shallow copied assert r1.headers is not r2.headers, ( "headers must be a shallow copy, not identical" ) - self.assertEqual(r1.headers, r2.headers) - self.assertEqual(r1.encoding, r2.encoding) - self.assertEqual(r1.dont_filter, r2.dont_filter) + assert r1.headers == r2.headers + assert r1.encoding == r2.encoding + assert r1.dont_filter == r2.dont_filter # Request.body can be identical since it's an immutable object (str) @@ -258,10 +257,10 @@ class RequestTest(unittest.TestCase): hdrs = Headers(r1.headers) hdrs[b"key"] = b"value" r2 = r1.replace(method="POST", body="New body", headers=hdrs) - self.assertEqual(r1.url, r2.url) - self.assertEqual((r1.method, r2.method), ("GET", "POST")) - self.assertEqual((r1.body, r2.body), (b"", b"New body")) - self.assertEqual((r1.headers, r2.headers), (self.default_headers, hdrs)) + assert r1.url == r2.url + assert (r1.method, r2.method) == ("GET", "POST") + assert (r1.body, r2.body) == (b"", b"New body") + assert (r1.headers, r2.headers) == (self.default_headers, hdrs) # Empty attributes (which may fail if not compared properly) r3 = self.request_class( @@ -270,9 +269,9 @@ class RequestTest(unittest.TestCase): r4 = r3.replace( url="http://www.example.com/2", body=b"", meta={}, dont_filter=False ) - self.assertEqual(r4.url, "http://www.example.com/2") - self.assertEqual(r4.body, b"") - self.assertEqual(r4.meta, {}) + assert r4.url == "http://www.example.com/2" + assert r4.body == b"" + assert r4.meta == {} assert r4.dont_filter is False def test_method_always_str(self): @@ -291,32 +290,32 @@ class RequestTest(unittest.TestCase): pass r1 = self.request_class("http://example.com") - self.assertIsNone(r1.callback) - self.assertIsNone(r1.errback) + assert r1.callback is None + assert r1.errback is None r2 = self.request_class("http://example.com", callback=a_function) - self.assertIs(r2.callback, a_function) - self.assertIsNone(r2.errback) + assert r2.callback is a_function + assert r2.errback is None r3 = self.request_class("http://example.com", errback=a_function) - self.assertIsNone(r3.callback) - self.assertIs(r3.errback, a_function) + assert r3.callback is None + assert r3.errback is a_function r4 = self.request_class( url="http://example.com", callback=a_function, errback=a_function, ) - self.assertIs(r4.callback, a_function) - self.assertIs(r4.errback, a_function) + assert r4.callback is a_function + assert r4.errback is a_function r5 = self.request_class( url="http://example.com", callback=NO_CALLBACK, errback=NO_CALLBACK, ) - self.assertIs(r5.callback, NO_CALLBACK) - self.assertIs(r5.errback, NO_CALLBACK) + assert r5.callback is NO_CALLBACK + assert r5.errback is NO_CALLBACK def test_callback_and_errback_type(self): with pytest.raises(TypeError): @@ -354,53 +353,46 @@ class RequestTest(unittest.TestCase): "2%3A15&comments=' --compressed" ) r = self.request_class.from_curl(curl_command) - self.assertEqual(r.method, "POST") - self.assertEqual(r.url, "http://httpbin.org/post") - self.assertEqual( - r.body, - b"custname=John+Smith&custtel=500&custemail=jsmith%40" + assert r.method == "POST" + assert r.url == "http://httpbin.org/post" + assert ( + r.body == b"custname=John+Smith&custtel=500&custemail=jsmith%40" b"example.org&size=small&topping=cheese&topping=onion" - b"&delivery=12%3A15&comments=", - ) - self.assertEqual( - r.cookies, - { - "_gauges_unique_year": "1", - "_gauges_unique": "1", - "_gauges_unique_month": "1", - "_gauges_unique_hour": "1", - "_gauges_unique_day": "1", - }, - ) - self.assertEqual( - r.headers, - { - b"Origin": [b"http://httpbin.org"], - b"Accept-Encoding": [b"gzip, deflate"], - b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"], - b"Upgrade-Insecure-Requests": [b"1"], - b"User-Agent": [ - b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537." - b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202" - b".75 Chrome/62.0.3202.75 Safari/537.36" - ], - b"Content-Type": [b"application /x-www-form-urlencoded"], - b"Accept": [ - b"text/html,application/xhtml+xml,application/xml;q=0." - b"9,image/webp,image/apng,*/*;q=0.8" - ], - b"Cache-Control": [b"max-age=0"], - b"Referer": [b"http://httpbin.org/forms/post"], - b"Connection": [b"keep-alive"], - }, + b"&delivery=12%3A15&comments=" ) + assert r.cookies == { + "_gauges_unique_year": "1", + "_gauges_unique": "1", + "_gauges_unique_month": "1", + "_gauges_unique_hour": "1", + "_gauges_unique_day": "1", + } + assert r.headers == { + b"Origin": [b"http://httpbin.org"], + b"Accept-Encoding": [b"gzip, deflate"], + b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"], + b"Upgrade-Insecure-Requests": [b"1"], + b"User-Agent": [ + b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537." + b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202" + b".75 Chrome/62.0.3202.75 Safari/537.36" + ], + b"Content-Type": [b"application /x-www-form-urlencoded"], + b"Accept": [ + b"text/html,application/xhtml+xml,application/xml;q=0." + b"9,image/webp,image/apng,*/*;q=0.8" + ], + b"Cache-Control": [b"max-age=0"], + b"Referer": [b"http://httpbin.org/forms/post"], + b"Connection": [b"keep-alive"], + } def test_from_curl_with_kwargs(self): r = self.request_class.from_curl( 'curl -X PATCH "http://example.org"', method="POST", meta={"key": "value"} ) - self.assertEqual(r.method, "POST") - self.assertEqual(r.meta, {"key": "value"}) + assert r.method == "POST" + assert r.meta == {"key": "value"} def test_from_curl_ignore_unknown_options(self): # By default: it works and ignores the unknown options: --foo and -z @@ -409,7 +401,7 @@ class RequestTest(unittest.TestCase): r = self.request_class.from_curl( 'curl -X DELETE "http://example.org" --foo -z', ) - self.assertEqual(r.method, "DELETE") + assert r.method == "DELETE" # If `ignore_unknown_options` is set to `False` it raises an error with # the unknown options: --foo and -z @@ -420,17 +412,17 @@ class RequestTest(unittest.TestCase): ) -class FormRequestTest(RequestTest): +class TestFormRequest(TestRequest): request_class = FormRequest def assertQueryEqual(self, first, second, msg=None): first = to_unicode(first).split("&") second = to_unicode(second).split("&") - return self.assertEqual(sorted(first), sorted(second), msg) + assert sorted(first) == sorted(second), msg def test_empty_formdata(self): r1 = self.request_class("http://www.example.com", formdata={}) - self.assertEqual(r1.body, b"") + assert r1.body == b"" def test_formdata_overrides_querystring(self): data = (("a", "one"), ("a", "two"), ("b", "2")) @@ -438,69 +430,61 @@ class FormRequestTest(RequestTest): "http://www.example.com/?a=0&b=1&c=3#fragment", method="GET", formdata=data ).url.split("#", maxsplit=1)[0] fs = _qs(self.request_class(url, method="GET", formdata=data)) - self.assertEqual(set(fs[b"a"]), {b"one", b"two"}) - self.assertEqual(fs[b"b"], [b"2"]) - self.assertIsNone(fs.get(b"c")) + assert set(fs[b"a"]) == {b"one", b"two"} + assert fs[b"b"] == [b"2"] + assert fs.get(b"c") is None data = {"a": "1", "b": "2"} fs = _qs( self.request_class("http://www.example.com/", method="GET", formdata=data) ) - self.assertEqual(fs[b"a"], [b"1"]) - self.assertEqual(fs[b"b"], [b"2"]) + assert fs[b"a"] == [b"1"] + assert fs[b"b"] == [b"2"] def test_default_encoding_bytes(self): # using default encoding (utf-8) data = {b"one": b"two", b"price": b"\xc2\xa3 100"} r2 = self.request_class("http://www.example.com", formdata=data) - self.assertEqual(r2.method, "POST") - self.assertEqual(r2.encoding, "utf-8") + assert r2.method == "POST" + assert r2.encoding == "utf-8" self.assertQueryEqual(r2.body, b"price=%C2%A3+100&one=two") - self.assertEqual( - r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" - ) + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_default_encoding_textual_data(self): # using default encoding (utf-8) data = {"µ one": "two", "price": "£ 100"} r2 = self.request_class("http://www.example.com", formdata=data) - self.assertEqual(r2.method, "POST") - self.assertEqual(r2.encoding, "utf-8") + assert r2.method == "POST" + assert r2.encoding == "utf-8" self.assertQueryEqual(r2.body, b"price=%C2%A3+100&%C2%B5+one=two") - self.assertEqual( - r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" - ) + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_default_encoding_mixed_data(self): # using default encoding (utf-8) data = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"} r2 = self.request_class("http://www.example.com", formdata=data) - self.assertEqual(r2.method, "POST") - self.assertEqual(r2.encoding, "utf-8") + assert r2.method == "POST" + assert r2.encoding == "utf-8" self.assertQueryEqual(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100") - self.assertEqual( - r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" - ) + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_custom_encoding_bytes(self): data = {b"\xb5 one": b"two", b"price": b"\xa3 100"} r2 = self.request_class( "http://www.example.com", formdata=data, encoding="latin1" ) - self.assertEqual(r2.method, "POST") - self.assertEqual(r2.encoding, "latin1") + assert r2.method == "POST" + assert r2.encoding == "latin1" self.assertQueryEqual(r2.body, b"price=%A3+100&%B5+one=two") - self.assertEqual( - r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" - ) + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_custom_encoding_textual_data(self): data = {"price": "£ 100"} r3 = self.request_class( "http://www.example.com", formdata=data, encoding="latin1" ) - self.assertEqual(r3.encoding, "latin1") - self.assertEqual(r3.body, b"price=%A3+100") + assert r3.encoding == "latin1" + assert r3.body == b"price=%A3+100" def test_multi_key_values(self): # using multiples values for a single key @@ -523,16 +507,14 @@ class FormRequestTest(RequestTest): response, formdata={"one": ["two", "three"], "six": "seven"} ) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers[b"Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req) - self.assertEqual(set(fs[b"test"]), {b"val1", b"val2"}) - self.assertEqual(set(fs[b"one"]), {b"two", b"three"}) - self.assertEqual(fs[b"test2"], [b"xxx"]) - self.assertEqual(fs[b"six"], [b"seven"]) + assert set(fs[b"test"]) == {b"val1", b"val2"} + assert set(fs[b"one"]) == {b"two", b"three"} + assert fs[b"test2"] == [b"xxx"] + assert fs[b"six"] == [b"seven"] def test_from_response_post_nonascii_bytes_utf8(self): response = _buildresponse( @@ -547,16 +529,14 @@ class FormRequestTest(RequestTest): response, formdata={"one": ["two", "three"], "six": "seven"} ) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers[b"Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req, to_unicode=True) - self.assertEqual(set(fs["test £"]), {"val1", "val2"}) - self.assertEqual(set(fs["one"]), {"two", "three"}) - self.assertEqual(fs["test2"], ["xxx µ"]) - self.assertEqual(fs["six"], ["seven"]) + assert set(fs["test £"]) == {"val1", "val2"} + assert set(fs["one"]) == {"two", "three"} + assert fs["test2"] == ["xxx µ"] + assert fs["six"] == ["seven"] def test_from_response_post_nonascii_bytes_latin1(self): response = _buildresponse( @@ -572,16 +552,14 @@ class FormRequestTest(RequestTest): response, formdata={"one": ["two", "three"], "six": "seven"} ) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers[b"Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req, to_unicode=True, encoding="latin1") - self.assertEqual(set(fs["test £"]), {"val1", "val2"}) - self.assertEqual(set(fs["one"]), {"two", "three"}) - self.assertEqual(fs["test2"], ["xxx µ"]) - self.assertEqual(fs["six"], ["seven"]) + assert set(fs["test £"]) == {"val1", "val2"} + assert set(fs["one"]) == {"two", "three"} + assert fs["test2"] == ["xxx µ"] + assert fs["six"] == ["seven"] def test_from_response_post_nonascii_unicode(self): response = _buildresponse( @@ -596,16 +574,14 @@ class FormRequestTest(RequestTest): response, formdata={"one": ["two", "three"], "six": "seven"} ) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers[b"Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req, to_unicode=True) - self.assertEqual(set(fs["test £"]), {"val1", "val2"}) - self.assertEqual(set(fs["one"]), {"two", "three"}) - self.assertEqual(fs["test2"], ["xxx µ"]) - self.assertEqual(fs["six"], ["seven"]) + assert set(fs["test £"]) == {"val1", "val2"} + assert set(fs["one"]) == {"two", "three"} + assert fs["test2"] == ["xxx µ"] + assert fs["six"] == ["seven"] def test_from_response_duplicate_form_key(self): response = _buildresponse("
", url="http://www.example.com") @@ -614,8 +590,8 @@ class FormRequestTest(RequestTest): method="GET", formdata=(("foo", "bar"), ("foo", "baz")), ) - self.assertEqual(urlparse_cached(req).hostname, "www.example.com") - self.assertEqual(urlparse_cached(req).query, "foo=bar&foo=baz") + assert urlparse_cached(req).hostname == "www.example.com" + assert urlparse_cached(req).query == "foo=bar&foo=baz" def test_from_response_override_duplicate_form_key(self): response = _buildresponse( @@ -628,8 +604,8 @@ class FormRequestTest(RequestTest): response, formdata=(("two", "2"), ("two", "4")) ) fs = _qs(req) - self.assertEqual(fs[b"one"], [b"1"]) - self.assertEqual(fs[b"two"], [b"2", b"4"]) + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2", b"4"] def test_from_response_extra_headers(self): response = _buildresponse( @@ -644,11 +620,9 @@ class FormRequestTest(RequestTest): formdata={"one": ["two", "three"], "six": "seven"}, headers={"Accept-Encoding": "gzip,deflate"}, ) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers["Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.headers["Accept-Encoding"], b"gzip,deflate") + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.headers["Accept-Encoding"] == b"gzip,deflate" def test_from_response_get(self): response = _buildresponse( @@ -662,14 +636,14 @@ class FormRequestTest(RequestTest): r1 = self.request_class.from_response( response, formdata={"one": ["two", "three"], "six": "seven"} ) - self.assertEqual(r1.method, "GET") - self.assertEqual(urlparse_cached(r1).hostname, "www.example.com") - self.assertEqual(urlparse_cached(r1).path, "/this/get.php") + assert r1.method == "GET" + assert urlparse_cached(r1).hostname == "www.example.com" + assert urlparse_cached(r1).path == "/this/get.php" fs = _qs(r1) - self.assertEqual(set(fs[b"test"]), {b"val1", b"val2"}) - self.assertEqual(set(fs[b"one"]), {b"two", b"three"}) - self.assertEqual(fs[b"test2"], [b"xxx"]) - self.assertEqual(fs[b"six"], [b"seven"]) + assert set(fs[b"test"]) == {b"val1", b"val2"} + assert set(fs[b"one"]) == {b"two", b"three"} + assert fs[b"test2"] == [b"xxx"] + assert fs[b"six"] == [b"seven"] def test_from_response_override_params(self): response = _buildresponse( @@ -680,8 +654,8 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response, formdata={"two": "2"}) fs = _qs(req) - self.assertEqual(fs[b"one"], [b"1"]) - self.assertEqual(fs[b"two"], [b"2"]) + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2"] def test_from_response_drop_params(self): response = _buildresponse( @@ -692,8 +666,8 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response, formdata={"two": None}) fs = _qs(req) - self.assertEqual(fs[b"one"], [b"1"]) - self.assertNotIn(b"two", fs) + assert fs[b"one"] == [b"1"] + assert b"two" not in fs def test_from_response_override_method(self): response = _buildresponse( @@ -702,9 +676,9 @@ class FormRequestTest(RequestTest): """ ) request = FormRequest.from_response(response) - self.assertEqual(request.method, "GET") + assert request.method == "GET" request = FormRequest.from_response(response, method="POST") - self.assertEqual(request.method, "POST") + assert request.method == "POST" def test_from_response_override_url(self): response = _buildresponse( @@ -713,11 +687,11 @@ class FormRequestTest(RequestTest): """ ) request = FormRequest.from_response(response) - self.assertEqual(request.url, "http://example.com/app") + assert request.url == "http://example.com/app" request = FormRequest.from_response(response, url="http://foo.bar/absolute") - self.assertEqual(request.url, "http://foo.bar/absolute") + assert request.url == "http://foo.bar/absolute" request = FormRequest.from_response(response, url="/relative") - self.assertEqual(request.url, "http://example.com/relative") + assert request.url == "http://example.com/relative" def test_from_response_case_insensitive(self): response = _buildresponse( @@ -729,9 +703,9 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response) fs = _qs(req) - self.assertEqual(fs[b"clickable1"], [b"clicked1"]) - self.assertFalse(b"i1" in fs, fs) # xpath in _get_inputs() - self.assertFalse(b"clickable2" in fs, fs) # xpath in _get_clickable() + assert fs[b"clickable1"] == [b"clicked1"] + assert b"i1" not in fs, fs # xpath in _get_inputs() + assert b"clickable2" not in fs, fs # xpath in _get_clickable() def test_from_response_submit_first_clickable(self): response = _buildresponse( @@ -744,10 +718,10 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response, formdata={"two": "2"}) fs = _qs(req) - self.assertEqual(fs[b"clickable1"], [b"clicked1"]) - self.assertFalse(b"clickable2" in fs, fs) - self.assertEqual(fs[b"one"], [b"1"]) - self.assertEqual(fs[b"two"], [b"2"]) + assert fs[b"clickable1"] == [b"clicked1"] + assert b"clickable2" not in fs, fs + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2"] def test_from_response_submit_not_first_clickable(self): response = _buildresponse( @@ -762,10 +736,10 @@ class FormRequestTest(RequestTest): response, formdata={"two": "2"}, clickdata={"name": "clickable2"} ) fs = _qs(req) - self.assertEqual(fs[b"clickable2"], [b"clicked2"]) - self.assertFalse(b"clickable1" in fs, fs) - self.assertEqual(fs[b"one"], [b"1"]) - self.assertEqual(fs[b"two"], [b"2"]) + assert fs[b"clickable2"] == [b"clicked2"] + assert b"clickable1" not in fs, fs + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2"] def test_from_response_dont_submit_image_as_input(self): response = _buildresponse( @@ -777,7 +751,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v"]}) + assert fs == {b"i1": [b"i1v"]} def test_from_response_dont_submit_reset_as_input(self): response = _buildresponse( @@ -790,7 +764,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v"], b"i2": [b"i2v"]}) + assert fs == {b"i1": [b"i1v"], b"i2": [b"i2v"]} def test_from_response_clickdata_does_not_ignore_image(self): response = _buildresponse( @@ -801,7 +775,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v"], b"i2": [b"i2v"]}) + assert fs == {b"i1": [b"i1v"], b"i2": [b"i2v"]} def test_from_response_multiple_clickdata(self): response = _buildresponse( @@ -816,9 +790,9 @@ class FormRequestTest(RequestTest): response, clickdata={"name": "clickable", "value": "clicked2"} ) fs = _qs(req) - self.assertEqual(fs[b"clickable"], [b"clicked2"]) - self.assertEqual(fs[b"one"], [b"clicked1"]) - self.assertEqual(fs[b"two"], [b"clicked2"]) + assert fs[b"clickable"] == [b"clicked2"] + assert fs[b"one"] == [b"clicked1"] + assert fs[b"two"] == [b"clicked2"] def test_from_response_unicode_clickdata(self): response = _buildresponse( @@ -833,7 +807,7 @@ class FormRequestTest(RequestTest): response, clickdata={"name": "price in \u00a3"} ) fs = _qs(req, to_unicode=True) - self.assertTrue(fs["price in \u00a3"]) + assert fs["price in \u00a3"] def test_from_response_unicode_clickdata_latin1(self): response = _buildresponse( @@ -849,7 +823,7 @@ class FormRequestTest(RequestTest): response, clickdata={"name": "price in \u00a5"} ) fs = _qs(req, to_unicode=True, encoding="latin1") - self.assertTrue(fs["price in \u00a5"]) + assert fs["price in \u00a5"] def test_from_response_multiple_forms_clickdata(self): response = _buildresponse( @@ -867,9 +841,9 @@ class FormRequestTest(RequestTest): response, formname="form2", clickdata={"name": "clickable"} ) fs = _qs(req) - self.assertEqual(fs[b"clickable"], [b"clicked2"]) - self.assertEqual(fs[b"field2"], [b"value2"]) - self.assertFalse(b"field1" in fs, fs) + assert fs[b"clickable"] == [b"clicked2"] + assert fs[b"field2"] == [b"value2"] + assert b"field1" not in fs, fs def test_from_response_override_clickable(self): response = _buildresponse( @@ -879,7 +853,7 @@ class FormRequestTest(RequestTest): response, formdata={"clickme": "two"}, clickdata={"name": "clickme"} ) fs = _qs(req) - self.assertEqual(fs[b"clickme"], [b"two"]) + assert fs[b"clickme"] == [b"two"] def test_from_response_dont_click(self): response = _buildresponse( @@ -892,8 +866,8 @@ class FormRequestTest(RequestTest): ) r1 = self.request_class.from_response(response, dont_click=True) fs = _qs(r1) - self.assertFalse(b"clickable1" in fs, fs) - self.assertFalse(b"clickable2" in fs, fs) + assert b"clickable1" not in fs, fs + assert b"clickable2" not in fs, fs def test_from_response_ambiguous_clickdata(self): response = _buildresponse( @@ -934,8 +908,8 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response, clickdata={"nr": 1}) fs = _qs(req) - self.assertIn(b"clickable2", fs) - self.assertNotIn(b"clickable1", fs) + assert b"clickable2" in fs + assert b"clickable1" not in fs def test_from_response_invalid_nr_index_clickdata(self): response = _buildresponse( @@ -962,7 +936,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response, formdata={"bar": "buz"}) fs = _qs(req) - self.assertEqual(fs, {b"foo": [b"xxx"], b"bar": [b"buz"]}) + assert fs == {b"foo": [b"xxx"], b"bar": [b"buz"]} def test_from_response_errors_formnumber(self): response = _buildresponse( @@ -983,12 +957,10 @@ class FormRequestTest(RequestTest): """ ) r1 = self.request_class.from_response(response, formdata={"two": "3"}) - self.assertEqual(r1.method, "POST") - self.assertEqual( - r1.headers["Content-type"], b"application/x-www-form-urlencoded" - ) + assert r1.method == "POST" + assert r1.headers["Content-type"] == b"application/x-www-form-urlencoded" fs = _qs(r1) - self.assertEqual(fs, {b"one": [b"1"], b"two": [b"3"]}) + assert fs == {b"one": [b"1"], b"two": [b"3"]} def test_from_response_formname_exists(self): response = _buildresponse( @@ -1002,9 +974,9 @@ class FormRequestTest(RequestTest): """ ) r1 = self.request_class.from_response(response, formname="form2") - self.assertEqual(r1.method, "POST") + assert r1.method == "POST" fs = _qs(r1) - self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]}) + assert fs == {b"four": [b"4"], b"three": [b"3"]} def test_from_response_formname_nonexistent(self): response = _buildresponse( @@ -1016,9 +988,9 @@ class FormRequestTest(RequestTest): """ ) r1 = self.request_class.from_response(response, formname="form3") - self.assertEqual(r1.method, "POST") + assert r1.method == "POST" fs = _qs(r1) - self.assertEqual(fs, {b"one": [b"1"]}) + assert fs == {b"one": [b"1"]} def test_from_response_formname_errors_formnumber(self): response = _buildresponse( @@ -1044,9 +1016,9 @@ class FormRequestTest(RequestTest): """ ) r1 = self.request_class.from_response(response, formid="form2") - self.assertEqual(r1.method, "POST") + assert r1.method == "POST" fs = _qs(r1) - self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]}) + assert fs == {b"four": [b"4"], b"three": [b"3"]} def test_from_response_formname_nonexistent_fallback_formid(self): response = _buildresponse( @@ -1062,9 +1034,9 @@ class FormRequestTest(RequestTest): r1 = self.request_class.from_response( response, formname="form3", formid="form2" ) - self.assertEqual(r1.method, "POST") + assert r1.method == "POST" fs = _qs(r1) - self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]}) + assert fs == {b"four": [b"4"], b"three": [b"3"]} def test_from_response_formid_nonexistent(self): response = _buildresponse( @@ -1076,9 +1048,9 @@ class FormRequestTest(RequestTest): """ ) r1 = self.request_class.from_response(response, formid="form3") - self.assertEqual(r1.method, "POST") + assert r1.method == "POST" fs = _qs(r1) - self.assertEqual(fs, {b"one": [b"1"]}) + assert fs == {b"one": [b"1"]} def test_from_response_formid_errors_formnumber(self): response = _buildresponse( @@ -1122,7 +1094,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(res) fs = _qs(req, to_unicode=True) - self.assertEqual(fs, {"i1": ["i1v2"], "i2": ["i2v1"], "i4": ["i4v2", "i4v3"]}) + assert fs == {"i1": ["i1v2"], "i2": ["i2v1"], "i4": ["i4v2", "i4v3"]} def test_from_response_radio(self): res = _buildresponse( @@ -1139,7 +1111,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"iv2"], b"i2": [b"on"]}) + assert fs == {b"i1": [b"iv2"], b"i2": [b"on"]} def test_from_response_checkbox(self): res = _buildresponse( @@ -1156,7 +1128,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"iv2"], b"i2": [b"on"]}) + assert fs == {b"i1": [b"iv2"], b"i2": [b"on"]} def test_from_response_input_text(self): res = _buildresponse( @@ -1170,7 +1142,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v1"], b"i2": [b""], b"i4": [b"i4v1"]}) + assert fs == {b"i1": [b"i1v1"], b"i2": [b""], b"i4": [b"i4v1"]} def test_from_response_input_hidden(self): res = _buildresponse( @@ -1183,7 +1155,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v1"], b"i2": [b""]}) + assert fs == {b"i1": [b"i1v1"], b"i2": [b""]} def test_from_response_input_textarea(self): res = _buildresponse( @@ -1196,7 +1168,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v"], b"i2": [b""], b"i3": [b""]}) + assert fs == {b"i1": [b"i1v"], b"i2": [b""], b"i3": [b""]} def test_from_response_descendants(self): res = _buildresponse( @@ -1218,7 +1190,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(set(fs), {b"h2", b"i2", b"i1", b"i3", b"h1", b"i5", b"i4"}) + assert set(fs) == {b"h2", b"i2", b"i1", b"i3", b"h1", b"i5", b"i4"} def test_from_response_xpath(self): response = _buildresponse( @@ -1235,13 +1207,13 @@ class FormRequestTest(RequestTest): response, formxpath="//form[@action='post.php']" ) fs = _qs(r1) - self.assertEqual(fs[b"one"], [b"1"]) + assert fs[b"one"] == [b"1"] r1 = self.request_class.from_response( response, formxpath="//form/input[@name='four']" ) fs = _qs(r1) - self.assertEqual(fs[b"three"], [b"3"]) + assert fs[b"three"] == [b"3"] with pytest.raises(ValueError, match="No
element found with"): self.request_class.from_response( @@ -1254,7 +1226,7 @@ class FormRequestTest(RequestTest): response, formxpath="//form[@name='\u044a']" ) fs = _qs(r) - self.assertEqual(fs, {}) + assert not fs xpath = "//form[@name='\u03b1']" with pytest.raises(ValueError, match=re.escape(xpath)): @@ -1270,15 +1242,13 @@ class FormRequestTest(RequestTest): url="http://www.example.com/this/list.html", ) req = self.request_class.from_response(response) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers["Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req) - self.assertEqual(fs[b"test1"], [b"val1"]) - self.assertEqual(fs[b"test2"], [b"val2"]) - self.assertEqual(fs[b"button1"], [b"submit1"]) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b"submit1"] def test_from_response_button_notype(self): response = _buildresponse( @@ -1290,15 +1260,13 @@ class FormRequestTest(RequestTest): url="http://www.example.com/this/list.html", ) req = self.request_class.from_response(response) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers["Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req) - self.assertEqual(fs[b"test1"], [b"val1"]) - self.assertEqual(fs[b"test2"], [b"val2"]) - self.assertEqual(fs[b"button1"], [b"submit1"]) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b"submit1"] def test_from_response_submit_novalue(self): response = _buildresponse( @@ -1310,15 +1278,13 @@ class FormRequestTest(RequestTest): url="http://www.example.com/this/list.html", ) req = self.request_class.from_response(response) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers["Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req) - self.assertEqual(fs[b"test1"], [b"val1"]) - self.assertEqual(fs[b"test2"], [b"val2"]) - self.assertEqual(fs[b"button1"], [b""]) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b""] def test_from_response_button_novalue(self): response = _buildresponse( @@ -1330,15 +1296,13 @@ class FormRequestTest(RequestTest): url="http://www.example.com/this/list.html", ) req = self.request_class.from_response(response) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers["Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req) - self.assertEqual(fs[b"test1"], [b"val1"]) - self.assertEqual(fs[b"test2"], [b"val2"]) - self.assertEqual(fs[b"button1"], [b""]) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b""] def test_html_base_form_action(self): response = _buildresponse( @@ -1356,12 +1320,12 @@ class FormRequestTest(RequestTest): url="http://a.com/", ) req = self.request_class.from_response(response) - self.assertEqual(req.url, "http://b.com/test_form") + assert req.url == "http://b.com/test_form" def test_spaces_in_action(self): resp = _buildresponse('
') req = self.request_class.from_response(resp) - self.assertEqual(req.url, "http://example.com/path") + assert req.url == "http://example.com/path" def test_from_response_css(self): response = _buildresponse( @@ -1378,11 +1342,11 @@ class FormRequestTest(RequestTest): response, formcss="form[action='post.php']" ) fs = _qs(r1) - self.assertEqual(fs[b"one"], [b"1"]) + assert fs[b"one"] == [b"1"] r1 = self.request_class.from_response(response, formcss="input[name='four']") fs = _qs(r1) - self.assertEqual(fs[b"three"], [b"3"]) + assert fs[b"three"] == [b"3"] with pytest.raises(ValueError, match="No
element found with"): self.request_class.from_response(response, formcss="input[name='abc']") @@ -1400,7 +1364,7 @@ class FormRequestTest(RequestTest): "
" ) r = self.request_class.from_response(response) - self.assertEqual(r.method, expected) + assert r.method == expected def test_form_response_with_invalid_formdata_type_error(self): """Test that a ValueError is raised for non-iterable and non-dict formdata input""" @@ -1464,23 +1428,20 @@ def _qs(req, encoding="utf-8", to_unicode=False): return parse_qs(uqs, True) -class XmlRpcRequestTest(RequestTest): +class TestXmlRpcRequest(TestRequest): request_class = XmlRpcRequest default_method = "POST" default_headers = {b"Content-Type": [b"text/xml"]} def _test_request(self, **kwargs): r = self.request_class("http://scrapytest.org/rpc2", **kwargs) - self.assertEqual(r.headers[b"Content-Type"], b"text/xml") - self.assertEqual( - r.body, - to_bytes( - xmlrpc.client.dumps(**kwargs), encoding=kwargs.get("encoding", "utf-8") - ), + assert r.headers[b"Content-Type"] == b"text/xml" + assert r.body == to_bytes( + xmlrpc.client.dumps(**kwargs), encoding=kwargs.get("encoding", "utf-8") ) - self.assertEqual(r.method, "POST") - self.assertEqual(r.encoding, kwargs.get("encoding", "utf-8")) - self.assertTrue(r.dont_filter, True) + assert r.method == "POST" + assert r.encoding == kwargs.get("encoding", "utf-8") + assert r.dont_filter, True def test_xmlrpc_dumps(self): self._test_request(params=("value",)) @@ -1497,7 +1458,7 @@ class XmlRpcRequestTest(RequestTest): self._test_request(params=("pas£",), encoding="latin1") -class JsonRequestTest(RequestTest): +class TestJsonRequest(TestRequest): request_class = JsonRequest default_method = "GET" default_headers = { @@ -1505,49 +1466,51 @@ class JsonRequestTest(RequestTest): b"Accept": [b"application/json, text/javascript, */*; q=0.01"], } - def setUp(self): + def setup_method(self): warnings.simplefilter("always") - super().setUp() + + def teardown_method(self): + warnings.resetwarnings() def test_data(self): r1 = self.request_class(url="http://www.example.com/") - self.assertEqual(r1.body, b"") + assert r1.body == b"" body = b"body" r2 = self.request_class(url="http://www.example.com/", body=body) - self.assertEqual(r2.body, body) + assert r2.body == body data = { "name": "value", } r3 = self.request_class(url="http://www.example.com/", data=data) - self.assertEqual(r3.body, to_bytes(json.dumps(data))) + assert r3.body == to_bytes(json.dumps(data)) # empty data r4 = self.request_class(url="http://www.example.com/", data=[]) - self.assertEqual(r4.body, to_bytes(json.dumps([]))) + assert r4.body == to_bytes(json.dumps([])) def test_data_method(self): # data is not passed r1 = self.request_class(url="http://www.example.com/") - self.assertEqual(r1.method, "GET") + assert r1.method == "GET" body = b"body" r2 = self.request_class(url="http://www.example.com/", body=body) - self.assertEqual(r2.method, "GET") + assert r2.method == "GET" data = { "name": "value", } r3 = self.request_class(url="http://www.example.com/", data=data) - self.assertEqual(r3.method, "POST") + assert r3.method == "POST" # method passed explicitly r4 = self.request_class(url="http://www.example.com/", data=data, method="GET") - self.assertEqual(r4.method, "GET") + assert r4.method == "GET" r5 = self.request_class(url="http://www.example.com/", data=[]) - self.assertEqual(r5.method, "POST") + assert r5.method == "POST" def test_body_data(self): """passing both body and data should result a warning""" @@ -1557,10 +1520,10 @@ class JsonRequestTest(RequestTest): } with warnings.catch_warnings(record=True) as _warnings: r5 = self.request_class(url="http://www.example.com/", body=body, data=data) - self.assertEqual(r5.body, body) - self.assertEqual(r5.method, "GET") - self.assertEqual(len(_warnings), 1) - self.assertIn("data will be ignored", str(_warnings[0].message)) + assert r5.body == body + assert r5.method == "GET" + assert len(_warnings) == 1 + assert "data will be ignored" in str(_warnings[0].message) def test_empty_body_data(self): """passing any body value and data should result a warning""" @@ -1569,10 +1532,10 @@ class JsonRequestTest(RequestTest): } with warnings.catch_warnings(record=True) as _warnings: r6 = self.request_class(url="http://www.example.com/", body=b"", data=data) - self.assertEqual(r6.body, b"") - self.assertEqual(r6.method, "GET") - self.assertEqual(len(_warnings), 1) - self.assertIn("data will be ignored", str(_warnings[0].message)) + assert r6.body == b"" + assert r6.method == "GET" + assert len(_warnings) == 1 + assert "data will be ignored" in str(_warnings[0].message) def test_body_none_data(self): data = { @@ -1580,15 +1543,15 @@ class JsonRequestTest(RequestTest): } with warnings.catch_warnings(record=True) as _warnings: r7 = self.request_class(url="http://www.example.com/", body=None, data=data) - self.assertEqual(r7.body, to_bytes(json.dumps(data))) - self.assertEqual(r7.method, "POST") - self.assertEqual(len(_warnings), 0) + assert r7.body == to_bytes(json.dumps(data)) + assert r7.method == "POST" + assert len(_warnings) == 0 def test_body_data_none(self): with warnings.catch_warnings(record=True) as _warnings: r8 = self.request_class(url="http://www.example.com/", body=None, data=None) - self.assertEqual(r8.method, "GET") - self.assertEqual(len(_warnings), 0) + assert r8.method == "GET" + assert len(_warnings) == 0 def test_dumps_sort_keys(self): """Test that sort_keys=True is passed to json.dumps by default""" @@ -1598,7 +1561,7 @@ class JsonRequestTest(RequestTest): with mock.patch("json.dumps", return_value=b"") as mock_dumps: self.request_class(url="http://www.example.com/", data=data) kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs["sort_keys"], True) + assert kwargs["sort_keys"] is True def test_dumps_kwargs(self): """Test that dumps_kwargs are passed to json.dumps""" @@ -1614,8 +1577,8 @@ class JsonRequestTest(RequestTest): url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs ) kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs["ensure_ascii"], True) - self.assertEqual(kwargs["allow_nan"], True) + assert kwargs["ensure_ascii"] is True + assert kwargs["allow_nan"] is True def test_replace_data(self): data1 = { @@ -1626,7 +1589,7 @@ class JsonRequestTest(RequestTest): } r1 = self.request_class(url="http://www.example.com/", data=data1) r2 = r1.replace(data=data2) - self.assertEqual(r2.body, to_bytes(json.dumps(data2))) + assert r2.body == to_bytes(json.dumps(data2)) def test_replace_sort_keys(self): """Test that replace provides sort_keys=True to json.dumps""" @@ -1640,7 +1603,7 @@ class JsonRequestTest(RequestTest): with mock.patch("json.dumps", return_value=b"") as mock_dumps: r1.replace(data=data2) kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs["sort_keys"], True) + assert kwargs["sort_keys"] is True def test_replace_dumps_kwargs(self): """Test that dumps_kwargs are provided to json.dumps when replace is called""" @@ -1660,8 +1623,8 @@ class JsonRequestTest(RequestTest): with mock.patch("json.dumps", return_value=b"") as mock_dumps: r1.replace(data=data2) kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs["ensure_ascii"], True) - self.assertEqual(kwargs["allow_nan"], True) + assert kwargs["ensure_ascii"] is True + assert kwargs["allow_nan"] is True def test_replacement_both_body_and_data_warns(self): """Test that we get a warning if both body and data are passed""" @@ -1677,11 +1640,6 @@ class JsonRequestTest(RequestTest): with warnings.catch_warnings(record=True) as _warnings: r1.replace(data=data2, body=body2) - self.assertIn( - "Both body and data passed. data will be ignored", - str(_warnings[0].message), + assert "Both body and data passed. data will be ignored" in str( + _warnings[0].message ) - - def tearDown(self): - warnings.resetwarnings() - super().tearDown() diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 5a943f084..fdef5adea 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,5 +1,4 @@ import codecs -import unittest from unittest import mock import pytest @@ -22,62 +21,56 @@ from scrapy.utils.python import to_unicode from tests import get_testdata -class BaseResponseTest(unittest.TestCase): +class TestResponseBase: response_class = Response def test_init(self): # Response requires url in the constructor with pytest.raises(TypeError): self.response_class() - self.assertTrue( - isinstance(self.response_class("http://example.com/"), self.response_class) + assert isinstance( + self.response_class("http://example.com/"), self.response_class ) with pytest.raises(TypeError): self.response_class(b"http://example.com") with pytest.raises(TypeError): self.response_class(url="http://example.com", body={}) # body can be str or None - self.assertTrue( - isinstance( - self.response_class("http://example.com/", body=b""), - self.response_class, - ) + assert isinstance( + self.response_class("http://example.com/", body=b""), + self.response_class, ) - self.assertTrue( - isinstance( - self.response_class("http://example.com/", body=b"body"), - self.response_class, - ) + assert isinstance( + self.response_class("http://example.com/", body=b"body"), + self.response_class, ) # test presence of all optional parameters - self.assertTrue( - isinstance( - self.response_class( - "http://example.com/", body=b"", headers={}, status=200 - ), - self.response_class, - ) + assert isinstance( + self.response_class( + "http://example.com/", body=b"", headers={}, status=200 + ), + self.response_class, ) r = self.response_class("http://www.example.com") assert isinstance(r.url, str) - self.assertEqual(r.url, "http://www.example.com") - self.assertEqual(r.status, 200) + assert r.url == "http://www.example.com" + assert r.status == 200 assert isinstance(r.headers, Headers) - self.assertEqual(r.headers, {}) + assert not r.headers headers = {"foo": "bar"} body = b"a body" r = self.response_class("http://www.example.com", headers=headers, body=body) assert r.headers is not headers - self.assertEqual(r.headers[b"foo"], b"bar") + assert r.headers[b"foo"] == b"bar" r = self.response_class("http://www.example.com", status=301) - self.assertEqual(r.status, 301) + assert r.status == 301 r = self.response_class("http://www.example.com", status="301") - self.assertEqual(r.status, 301) + assert r.status == 301 with pytest.raises(ValueError, match=r"invalid literal for int\(\)"): self.response_class("http://example.com", status="lala200") @@ -88,18 +81,18 @@ class BaseResponseTest(unittest.TestCase): r1.flags.append("cached") r2 = r1.copy() - self.assertEqual(r1.status, r2.status) - self.assertEqual(r1.body, r2.body) + assert r1.status == r2.status + assert r1.body == r2.body # make sure flags list is shallow copied assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" - self.assertEqual(r1.flags, r2.flags) + assert r1.flags == r2.flags # make sure headers attribute is shallow copied assert r1.headers is not r2.headers, ( "headers must be a shallow copy, not identical" ) - self.assertEqual(r1.headers, r2.headers) + assert r1.headers == r2.headers def test_copy_meta(self): req = Request("http://www.example.com") @@ -144,16 +137,16 @@ class BaseResponseTest(unittest.TestCase): r1 = self.response_class("http://www.example.com") r2 = r1.replace(status=301, body=b"New body", headers=hdrs) assert r1.body == b"" - self.assertEqual(r1.url, r2.url) - self.assertEqual((r1.status, r2.status), (200, 301)) - self.assertEqual((r1.body, r2.body), (b"", b"New body")) - self.assertEqual((r1.headers, r2.headers), ({}, hdrs)) + assert r1.url == r2.url + assert (r1.status, r2.status) == (200, 301) + assert (r1.body, r2.body) == (b"", b"New body") + assert (r1.headers, r2.headers) == ({}, hdrs) # Empty attributes (which may fail if not compared properly) r3 = self.response_class("http://www.example.com", flags=["cached"]) r4 = r3.replace(body=b"", flags=[]) - self.assertEqual(r4.body, b"") - self.assertEqual(r4.flags, []) + assert r4.body == b"" + assert not r4.flags def _assert_response_values(self, response, encoding, body): if isinstance(body, str): @@ -166,11 +159,11 @@ class BaseResponseTest(unittest.TestCase): assert isinstance(response.body, bytes) assert isinstance(response.text, str) self._assert_response_encoding(response, encoding) - self.assertEqual(response.body, body_bytes) - self.assertEqual(response.text, body_unicode) + assert response.body == body_bytes + assert response.text == body_unicode def _assert_response_encoding(self, response, encoding): - self.assertEqual(response.encoding, resolve_encoding(encoding)) + assert response.encoding == resolve_encoding(encoding) def test_immutable_attributes(self): r = self.response_class("http://example.com") @@ -183,7 +176,7 @@ class BaseResponseTest(unittest.TestCase): """Test urljoin shortcut (only for existence, since behavior equals urljoin)""" joined = self.response_class("http://www.example.com").urljoin("/test") absolute = "http://www.example.com/test" - self.assertEqual(joined, absolute) + assert joined == absolute def test_shortcut_attributes(self): r = self.response_class("http://example.com", body=b"hello") @@ -241,7 +234,7 @@ class BaseResponseTest(unittest.TestCase): def test_follow_flags(self): res = self.response_class("http://example.com/") fol = res.follow("http://example.com/", flags=["cached", "allowed"]) - self.assertEqual(fol.flags, ["cached", "allowed"]) + assert fol.flags == ["cached", "allowed"] # Response.follow_all @@ -276,7 +269,7 @@ class BaseResponseTest(unittest.TestCase): def test_follow_all_empty(self): r = self.response_class("http://example.com") - self.assertEqual([], list(r.follow_all([]))) + assert not list(r.follow_all([])) def test_follow_all_invalid(self): r = self.response_class("http://example.com") @@ -327,13 +320,13 @@ class BaseResponseTest(unittest.TestCase): ] fol = re.follow_all(urls, flags=["cached", "allowed"]) for req in fol: - self.assertEqual(req.flags, ["cached", "allowed"]) + assert req.flags == ["cached", "allowed"] def _assert_followed_url(self, follow_obj, target_url, response=None): if response is None: response = self._links_response() req = response.follow(follow_obj) - self.assertEqual(req.url, target_url) + assert req.url == target_url return req def _assert_followed_all_urls(self, follow_obj, target_urls, response=None): @@ -341,7 +334,7 @@ class BaseResponseTest(unittest.TestCase): response = self._links_response() followed = response.follow_all(follow_obj) for req, target in zip(followed, target_urls): - self.assertEqual(req.url, target) + assert req.url == target yield req def _links_response(self): @@ -353,7 +346,7 @@ class BaseResponseTest(unittest.TestCase): return self.response_class("http://example.com/index", body=body) -class TextResponseTest(BaseResponseTest): +class TestTextResponse(TestResponseBase): response_class = TextResponse def test_replace(self): @@ -365,10 +358,10 @@ class TextResponseTest(BaseResponseTest): r3 = r1.replace(url="http://www.example.com/other", encoding="latin1") assert isinstance(r2, self.response_class) - self.assertEqual(r2.url, "http://www.example.com/other") + assert r2.url == "http://www.example.com/other" self._assert_response_encoding(r2, "cp852") - self.assertEqual(r3.url, "http://www.example.com/other") - self.assertEqual(r3._declared_encoding(), "latin1") + assert r3.url == "http://www.example.com/other" + assert r3._declared_encoding() == "latin1" def test_unicode_url(self): # instantiate with unicode url without encoding (should set default encoding) @@ -382,21 +375,21 @@ class TextResponseTest(BaseResponseTest): resp = self.response_class( url="http://www.example.com/price/\xa3", encoding="utf-8" ) - self.assertEqual(resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3")) + assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3") resp = self.response_class( url="http://www.example.com/price/\xa3", encoding="latin-1" ) - self.assertEqual(resp.url, "http://www.example.com/price/\xa3") + assert resp.url == "http://www.example.com/price/\xa3" resp = self.response_class( "http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]}, ) - self.assertEqual(resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3")) + assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3") resp = self.response_class( "http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, ) - self.assertEqual(resp.url, "http://www.example.com/price/\xa3") + assert resp.url == "http://www.example.com/price/\xa3" def test_unicode_body(self): unicode_string = ( @@ -412,8 +405,8 @@ class TextResponseTest(BaseResponseTest): ) # check response.text - self.assertTrue(isinstance(r1.text, str)) - self.assertEqual(r1.text, unicode_string) + assert isinstance(r1.text, str) + assert r1.text == unicode_string def test_encoding(self): r1 = self.response_class( @@ -458,18 +451,18 @@ class TextResponseTest(BaseResponseTest): }, ) - self.assertEqual(r1._headers_encoding(), "utf-8") - self.assertEqual(r2._headers_encoding(), None) - self.assertEqual(r2._declared_encoding(), "utf-8") + assert r1._headers_encoding() == "utf-8" + assert r2._headers_encoding() is None + assert r2._declared_encoding() == "utf-8" self._assert_response_encoding(r2, "utf-8") - self.assertEqual(r3._headers_encoding(), "cp1252") - self.assertEqual(r3._declared_encoding(), "cp1252") - self.assertEqual(r4._headers_encoding(), None) - self.assertEqual(r5._headers_encoding(), None) - self.assertEqual(r8._headers_encoding(), "cp1251") - self.assertEqual(r9._headers_encoding(), None) - self.assertEqual(r8._declared_encoding(), "utf-8") - self.assertEqual(r9._declared_encoding(), None) + assert r3._headers_encoding() == "cp1252" + assert r3._declared_encoding() == "cp1252" + assert r4._headers_encoding() is None + assert r5._headers_encoding() is None + assert r8._headers_encoding() == "cp1251" + assert r9._headers_encoding() is None + assert r8._declared_encoding() == "utf-8" + assert r9._declared_encoding() is None self._assert_response_encoding(r5, "utf-8") self._assert_response_encoding(r8, "utf-8") self._assert_response_encoding(r9, "cp1252") @@ -493,7 +486,7 @@ class TextResponseTest(BaseResponseTest): headers={"Content-type": ["text/html; charset=UNKNOWN"]}, body=b"\xc2\xa3", ) - self.assertEqual(r._declared_encoding(), None) + assert r._declared_encoding() is None self._assert_response_values(r, "utf-8", "\xa3") def test_utf16(self): @@ -511,14 +504,11 @@ class TextResponseTest(BaseResponseTest): headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xef\xbb\xbfWORD\xe3\xab", ) - self.assertEqual(r6.encoding, "utf-8") - self.assertIn( - r6.text, - { - "WORD\ufffd\ufffd", # w3lib < 1.19.0 - "WORD\ufffd", # w3lib >= 1.19.0 - }, - ) + assert r6.encoding == "utf-8" + assert r6.text in { + "WORD\ufffd\ufffd", # w3lib < 1.19.0 + "WORD\ufffd", # w3lib >= 1.19.0 + } def test_bom_is_removed_from_body(self): # Inferring encoding from body also cache decoded body as sideeffect, @@ -532,21 +522,21 @@ class TextResponseTest(BaseResponseTest): # Test response without content-type and BOM encoding response = self.response_class(url, body=body) - self.assertEqual(response.encoding, "utf-8") - self.assertEqual(response.text, "WORD") + assert response.encoding == "utf-8" + assert response.text == "WORD" response = self.response_class(url, body=body) - self.assertEqual(response.text, "WORD") - self.assertEqual(response.encoding, "utf-8") + assert response.text == "WORD" + assert response.encoding == "utf-8" # Body caching sideeffect isn't triggered when encoding is declared in # content-type header but BOM still need to be removed from decoded # body response = self.response_class(url, headers=headers, body=body) - self.assertEqual(response.encoding, "utf-8") - self.assertEqual(response.text, "WORD") + assert response.encoding == "utf-8" + assert response.text == "WORD" response = self.response_class(url, headers=headers, body=body) - self.assertEqual(response.text, "WORD") - self.assertEqual(response.encoding, "utf-8") + assert response.text == "WORD" + assert response.encoding == "utf-8" def test_replace_wrong_encoding(self): """Test invalid chars are replaced properly""" @@ -577,49 +567,47 @@ class TextResponseTest(BaseResponseTest): body = b"Some page" response = self.response_class("http://www.example.com", body=body) - self.assertIsInstance(response.selector, Selector) - self.assertEqual(response.selector.type, "html") - self.assertIs(response.selector, response.selector) # property is cached - self.assertIs(response.selector.response, response) + assert isinstance(response.selector, Selector) + assert response.selector.type == "html" + assert response.selector is response.selector # property is cached + assert response.selector.response is response - self.assertEqual( - response.selector.xpath("//title/text()").getall(), ["Some page"] - ) - self.assertEqual(response.selector.css("title::text").getall(), ["Some page"]) - self.assertEqual(response.selector.re("Some (.*)"), ["page"]) + assert response.selector.xpath("//title/text()").getall() == ["Some page"] + assert response.selector.css("title::text").getall() == ["Some page"] + assert response.selector.re("Some (.*)") == ["page"] def test_selector_shortcuts(self): body = b"Some page" response = self.response_class("http://www.example.com", body=body) - self.assertEqual( - response.xpath("//title/text()").getall(), - response.selector.xpath("//title/text()").getall(), + assert ( + response.xpath("//title/text()").getall() + == response.selector.xpath("//title/text()").getall() ) - self.assertEqual( - response.css("title::text").getall(), - response.selector.css("title::text").getall(), + assert ( + response.css("title::text").getall() + == response.selector.css("title::text").getall() ) def test_selector_shortcuts_kwargs(self): body = b'Some page

A nice paragraph.

' response = self.response_class("http://www.example.com", body=body) - self.assertEqual( + assert ( response.xpath( "normalize-space(//p[@class=$pclass])", pclass="content" - ).getall(), - response.xpath('normalize-space(//p[@class="content"])').getall(), + ).getall() + == response.xpath('normalize-space(//p[@class="content"])').getall() ) - self.assertEqual( + assert ( response.xpath( "//title[count(following::p[@class=$pclass])=$pcount]/text()", pclass="content", pcount=1, - ).getall(), - response.xpath( + ).getall() + == response.xpath( '//title[count(following::p[@class="content"])=1]/text()' - ).getall(), + ).getall() ) def test_urljoin_with_base_url(self): @@ -629,21 +617,21 @@ class TextResponseTest(BaseResponseTest): "/test" ) absolute = "https://example.net/test" - self.assertEqual(joined, absolute) + assert joined == absolute body = b'' joined = self.response_class("http://www.example.com", body=body).urljoin( "test" ) absolute = "http://www.example.com/test" - self.assertEqual(joined, absolute) + assert joined == absolute body = b'' joined = self.response_class("http://www.example.com", body=body).urljoin( "test" ) absolute = "http://www.example.com/elsewhere/test" - self.assertEqual(joined, absolute) + assert joined == absolute def test_follow_selector(self): resp = self._links_response() @@ -728,7 +716,7 @@ class TextResponseTest(BaseResponseTest): "http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82", response=resp1, ) - self.assertEqual(req.encoding, "utf8") + assert req.encoding == "utf8" resp2 = self.response_class( "http://example.com", @@ -742,12 +730,12 @@ class TextResponseTest(BaseResponseTest): "http://example.com/foo?%EF%F0%E8%E2%E5%F2", response=resp2, ) - self.assertEqual(req.encoding, "cp1251") + assert req.encoding == "cp1251" def test_follow_flags(self): res = self.response_class("http://example.com/") fol = res.follow("http://example.com/", flags=["cached", "allowed"]) - self.assertEqual(fol.flags, ["cached", "allowed"]) + assert fol.flags == ["cached", "allowed"] def test_follow_all_flags(self): re = self.response_class("http://www.example.com/") @@ -758,7 +746,7 @@ class TextResponseTest(BaseResponseTest): ] fol = re.follow_all(urls, flags=["cached", "allowed"]) for req in fol: - self.assertEqual(req.flags, ["cached", "allowed"]) + assert req.flags == ["cached", "allowed"] def test_follow_all_css(self): expected = [ @@ -767,7 +755,7 @@ class TextResponseTest(BaseResponseTest): ] response = self._links_response() extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')] - self.assertEqual(expected, extracted) + assert expected == extracted def test_follow_all_css_skip_invalid(self): expected = [ @@ -777,9 +765,9 @@ class TextResponseTest(BaseResponseTest): ] response = self._links_response_no_href() extracted1 = [r.url for r in response.follow_all(css=".pagination a")] - self.assertEqual(expected, extracted1) + assert expected == extracted1 extracted2 = [r.url for r in response.follow_all(response.css(".pagination a"))] - self.assertEqual(expected, extracted2) + assert expected == extracted2 def test_follow_all_xpath(self): expected = [ @@ -788,7 +776,7 @@ class TextResponseTest(BaseResponseTest): ] response = self._links_response() extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]') - self.assertEqual(expected, [r.url for r in extracted]) + assert expected == [r.url for r in extracted] def test_follow_all_xpath_skip_invalid(self): expected = [ @@ -800,12 +788,12 @@ class TextResponseTest(BaseResponseTest): extracted1 = [ r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a') ] - self.assertEqual(expected, extracted1) + assert expected == extracted1 extracted2 = [ r.url for r in response.follow_all(response.xpath('//div[@id="pagination"]/a')) ] - self.assertEqual(expected, extracted2) + assert expected == extracted2 def test_follow_all_too_many_arguments(self): response = self._links_response() @@ -820,7 +808,7 @@ class TextResponseTest(BaseResponseTest): def test_json_response(self): json_body = b"""{"ip": "109.187.217.200"}""" json_response = self.response_class("http://www.example.com", body=json_body) - self.assertEqual(json_response.json(), {"ip": "109.187.217.200"}) + assert json_response.json() == {"ip": "109.187.217.200"} text_body = b"""text""" text_response = self.response_class("http://www.example.com", body=text_body) @@ -842,7 +830,7 @@ class TextResponseTest(BaseResponseTest): mock_json.assert_called_once_with(json_body) -class HtmlResponseTest(TextResponseTest): +class TestHtmlResponse(TestTextResponse): response_class = HtmlResponse def test_html_encoding(self): @@ -883,7 +871,7 @@ class HtmlResponseTest(TextResponseTest): self._assert_response_values(r1, "gb2312", body) -class XmlResponseTest(TextResponseTest): +class TestXmlResponse(TestTextResponse): response_class = XmlResponse def test_xml_encoding(self): @@ -917,20 +905,20 @@ class XmlResponseTest(TextResponseTest): body = b'value' response = self.response_class("http://www.example.com", body=body) - self.assertIsInstance(response.selector, Selector) - self.assertEqual(response.selector.type, "xml") - self.assertIs(response.selector, response.selector) # property is cached - self.assertIs(response.selector.response, response) + assert isinstance(response.selector, Selector) + assert response.selector.type == "xml" + assert response.selector is response.selector # property is cached + assert response.selector.response is response - self.assertEqual(response.selector.xpath("//elem/text()").getall(), ["value"]) + assert response.selector.xpath("//elem/text()").getall() == ["value"] def test_selector_shortcuts(self): body = b'value' response = self.response_class("http://www.example.com", body=body) - self.assertEqual( - response.xpath("//elem/text()").getall(), - response.selector.xpath("//elem/text()").getall(), + assert ( + response.xpath("//elem/text()").getall() + == response.selector.xpath("//elem/text()").getall() ) def test_selector_shortcuts_kwargs(self): @@ -940,21 +928,21 @@ class XmlResponseTest(TextResponseTest): """ response = self.response_class("http://www.example.com", body=body) - self.assertEqual( + assert ( response.xpath( "//s:elem/text()", namespaces={"s": "http://scrapy.org"} - ).getall(), - response.selector.xpath( + ).getall() + == response.selector.xpath( "//s:elem/text()", namespaces={"s": "http://scrapy.org"} - ).getall(), + ).getall() ) response.selector.register_namespace("s2", "http://scrapy.org") - self.assertEqual( + assert ( response.xpath( "//s1:elem/text()", namespaces={"s1": "http://scrapy.org"} - ).getall(), - response.selector.xpath("//s2:elem/text()").getall(), + ).getall() + == response.selector.xpath("//s2:elem/text()").getall() ) @@ -968,7 +956,7 @@ class CustomResponse(TextResponse): super().__init__(*args, **kwargs) -class CustomResponseTest(TextResponseTest): +class TestCustomResponse(TestTextResponse): response_class = CustomResponse def test_copy(self): @@ -981,11 +969,11 @@ class CustomResponseTest(TextResponseTest): lost="lost", ) r2 = r1.copy() - self.assertIsInstance(r2, self.response_class) - self.assertEqual(r1.foo, r2.foo) - self.assertEqual(r1.bar, r2.bar) - self.assertEqual(r1.lost, "lost") - self.assertIsNone(r2.lost) + assert isinstance(r2, self.response_class) + assert r1.foo == r2.foo + assert r1.bar == r2.bar + assert r1.lost == "lost" + assert r2.lost is None def test_replace(self): super().test_replace() @@ -998,31 +986,31 @@ class CustomResponseTest(TextResponseTest): ) r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost") - self.assertIsInstance(r2, self.response_class) - self.assertEqual(r1.foo, "foo") - self.assertEqual(r1.bar, "bar") - self.assertEqual(r1.lost, "lost") - self.assertEqual(r2.foo, "new-foo") - self.assertEqual(r2.bar, "new-bar") - self.assertEqual(r2.lost, "new-lost") + assert isinstance(r2, self.response_class) + assert r1.foo == "foo" + assert r1.bar == "bar" + assert r1.lost == "lost" + assert r2.foo == "new-foo" + assert r2.bar == "new-bar" + assert r2.lost == "new-lost" r3 = r1.replace(foo="new-foo", bar="new-bar") - self.assertIsInstance(r3, self.response_class) - self.assertEqual(r1.foo, "foo") - self.assertEqual(r1.bar, "bar") - self.assertEqual(r1.lost, "lost") - self.assertEqual(r3.foo, "new-foo") - self.assertEqual(r3.bar, "new-bar") - self.assertIsNone(r3.lost) + assert isinstance(r3, self.response_class) + assert r1.foo == "foo" + assert r1.bar == "bar" + assert r1.lost == "lost" + assert r3.foo == "new-foo" + assert r3.bar == "new-bar" + assert r3.lost is None r4 = r1.replace(foo="new-foo") - self.assertIsInstance(r4, self.response_class) - self.assertEqual(r1.foo, "foo") - self.assertEqual(r1.bar, "bar") - self.assertEqual(r1.lost, "lost") - self.assertEqual(r4.foo, "new-foo") - self.assertEqual(r4.bar, "bar") - self.assertIsNone(r4.lost) + assert isinstance(r4, self.response_class) + assert r1.foo == "foo" + assert r1.bar == "bar" + assert r1.lost == "lost" + assert r4.foo == "new-foo" + assert r4.bar == "bar" + assert r4.lost is None with pytest.raises( TypeError, diff --git a/tests/test_loader.py b/tests/test_loader.py index 1a933bb8d..224158e7f 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,7 +1,6 @@ from __future__ import annotations import dataclasses -import unittest import attr import pytest @@ -67,7 +66,7 @@ def processor_with_args(value, other=None, loader_context=None): return value -class BasicItemLoaderTest(unittest.TestCase): +class TestBasicItemLoader: def test_add_value_on_unknown_field(self): il = ProcessorItemLoader() with pytest.raises(KeyError): @@ -80,14 +79,14 @@ class BasicItemLoaderTest(unittest.TestCase): il.add_value("name", "marta") item = il.load_item() assert item is i - self.assertEqual(item["summary"], ["lala"]) - self.assertEqual(item["name"], ["marta"]) + assert item["summary"] == ["lala"] + assert item["name"] == ["marta"] def test_load_item_using_custom_loader(self): il = ProcessorItemLoader() il.add_value("name", "marta") item = il.load_item() - self.assertEqual(item["name"], ["Marta"]) + assert item["name"] == ["Marta"] class InitializationTestMixin: @@ -98,16 +97,16 @@ class InitializationTestMixin: input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo"]}) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo"]} def test_keep_list(self): """Loaded item should contain values from the initial item""" input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]}) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "bar"]} def test_add_value_singlevalue_singlevalue(self): """Values added after initialization should be appended""" @@ -115,8 +114,8 @@ class InitializationTestMixin: il = ItemLoader(item=input_item) il.add_value("name", "bar") loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]}) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "bar"]} def test_add_value_singlevalue_list(self): """Values added after initialization should be appended""" @@ -124,10 +123,8 @@ class InitializationTestMixin: il = ItemLoader(item=input_item) il.add_value("name", ["item", "loader"]) loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual( - ItemAdapter(loaded_item).asdict(), {"name": ["foo", "item", "loader"]} - ) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "item", "loader"]} def test_add_value_list_singlevalue(self): """Values added after initialization should be appended""" @@ -135,10 +132,8 @@ class InitializationTestMixin: il = ItemLoader(item=input_item) il.add_value("name", "qwerty") loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual( - ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar", "qwerty"]} - ) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "bar", "qwerty"]} def test_add_value_list_list(self): """Values added after initialization should be appended""" @@ -146,56 +141,55 @@ class InitializationTestMixin: il = ItemLoader(item=input_item) il.add_value("name", ["item", "loader"]) loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual( - ItemAdapter(loaded_item).asdict(), - {"name": ["foo", "bar", "item", "loader"]}, - ) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == { + "name": ["foo", "bar", "item", "loader"] + } def test_get_output_value_singlevalue(self): """Getting output value must not remove value from item""" input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) - self.assertEqual(il.get_output_value("name"), ["foo"]) + assert il.get_output_value("name") == ["foo"] loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo"]}) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo"]} def test_get_output_value_list(self): """Getting output value must not remove value from item""" input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) - self.assertEqual(il.get_output_value("name"), ["foo", "bar"]) + assert il.get_output_value("name") == ["foo", "bar"] loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]}) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "bar"]} def test_values_single(self): """Values from initial item must be added to loader._values""" input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) - self.assertEqual(il._values.get("name"), ["foo"]) + assert il._values.get("name") == ["foo"] def test_values_list(self): """Values from initial item must be added to loader._values""" input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) - self.assertEqual(il._values.get("name"), ["foo", "bar"]) + assert il._values.get("name") == ["foo", "bar"] -class InitializationFromDictTest(InitializationTestMixin, unittest.TestCase): +class TestInitializationFromDict(InitializationTestMixin): item_class = dict -class InitializationFromItemTest(InitializationTestMixin, unittest.TestCase): +class TestInitializationFromItem(InitializationTestMixin): item_class = NameItem -class InitializationFromAttrsItemTest(InitializationTestMixin, unittest.TestCase): +class TestInitializationFromAttrsItem(InitializationTestMixin): item_class = AttrsNameItem -class InitializationFromDataClassTest(InitializationTestMixin, unittest.TestCase): +class TestInitializationFromDataClass(InitializationTestMixin): item_class = NameDataClass @@ -212,7 +206,7 @@ class NoInputReprocessingItemLoader(BaseNoInputReprocessingLoader): default_item_class = NoInputReprocessingItem -class NoInputReprocessingFromItemTest(unittest.TestCase): +class TestNoInputReprocessingFromItem: """ Loaders initialized from loaded items must not reprocess fields (Item instances) """ @@ -220,41 +214,41 @@ class NoInputReprocessingFromItemTest(unittest.TestCase): def test_avoid_reprocessing_with_initial_values_single(self): il = NoInputReprocessingItemLoader(item=NoInputReprocessingItem(title="foo")) il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "foo"}) - self.assertEqual( - NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "foo"} - ) + assert il_loaded == {"title": "foo"} + assert NoInputReprocessingItemLoader(item=il_loaded).load_item() == { + "title": "foo" + } def test_avoid_reprocessing_with_initial_values_list(self): il = NoInputReprocessingItemLoader( item=NoInputReprocessingItem(title=["foo", "bar"]) ) il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "foo"}) - self.assertEqual( - NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "foo"} - ) + assert il_loaded == {"title": "foo"} + assert NoInputReprocessingItemLoader(item=il_loaded).load_item() == { + "title": "foo" + } def test_avoid_reprocessing_without_initial_values_single(self): il = NoInputReprocessingItemLoader() il.add_value("title", "FOO") il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "FOO"}) - self.assertEqual( - NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "FOO"} - ) + assert il_loaded == {"title": "FOO"} + assert NoInputReprocessingItemLoader(item=il_loaded).load_item() == { + "title": "FOO" + } def test_avoid_reprocessing_without_initial_values_list(self): il = NoInputReprocessingItemLoader() il.add_value("title", ["foo", "bar"]) il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "FOO"}) - self.assertEqual( - NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "FOO"} - ) + assert il_loaded == {"title": "FOO"} + assert NoInputReprocessingItemLoader(item=il_loaded).load_item() == { + "title": "FOO" + } -class TestOutputProcessorItem(unittest.TestCase): +class TestOutputProcessorItem: def test_output_processor(self): class TempItem(Item): temp = Field() @@ -270,11 +264,11 @@ class TestOutputProcessorItem(unittest.TestCase): loader = TempLoader() item = loader.load_item() - self.assertIsInstance(item, TempItem) - self.assertEqual(dict(item), {"temp": 0.3}) + assert isinstance(item, TempItem) + assert dict(item) == {"temp": 0.3} -class SelectortemLoaderTest(unittest.TestCase): +class TestSelectortemLoader: response = HtmlResponse( url="", encoding="utf-8", @@ -292,7 +286,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_init_method(self): l = ProcessorItemLoader() - self.assertEqual(l.selector, None) + assert l.selector is None def test_init_method_errors(self): l = ProcessorItemLoader() @@ -312,150 +306,149 @@ class SelectortemLoaderTest(unittest.TestCase): def test_init_method_with_selector(self): sel = Selector(text="
marta
") l = ProcessorItemLoader(selector=sel) - self.assertIs(l.selector, sel) + assert l.selector is sel l.add_xpath("name", "//div/text()") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] def test_init_method_with_selector_css(self): sel = Selector(text="
marta
") l = ProcessorItemLoader(selector=sel) - self.assertIs(l.selector, sel) + assert l.selector is sel l.add_css("name", "div::text") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] def test_init_method_with_base_response(self): """Selector should be None after initialization""" response = Response("https://scrapy.org") l = ProcessorItemLoader(response=response) - self.assertIs(l.selector, None) + assert l.selector is None def test_init_method_with_response(self): l = ProcessorItemLoader(response=self.response) - self.assertTrue(l.selector) + assert l.selector l.add_xpath("name", "//div/text()") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] def test_init_method_with_response_css(self): l = ProcessorItemLoader(response=self.response) - self.assertTrue(l.selector) + assert l.selector l.add_css("name", "div::text") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.add_css("url", "a::attr(href)") - self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + assert l.get_output_value("url") == ["http://www.scrapy.org"] # combining/accumulating CSS selectors and XPath expressions l.add_xpath("name", "//div/text()") - self.assertEqual(l.get_output_value("name"), ["Marta", "Marta"]) + assert l.get_output_value("name") == ["Marta", "Marta"] l.add_xpath("url", "//img/@src") - self.assertEqual( - l.get_output_value("url"), ["http://www.scrapy.org", "/images/logo.png"] - ) + assert l.get_output_value("url") == [ + "http://www.scrapy.org", + "/images/logo.png", + ] def test_add_xpath_re(self): l = ProcessorItemLoader(response=self.response) l.add_xpath("name", "//div/text()", re="ma") - self.assertEqual(l.get_output_value("name"), ["Ma"]) + assert l.get_output_value("name") == ["Ma"] def test_replace_xpath(self): l = ProcessorItemLoader(response=self.response) - self.assertTrue(l.selector) + assert l.selector l.add_xpath("name", "//div/text()") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.replace_xpath("name", "//p/text()") - self.assertEqual(l.get_output_value("name"), ["Paragraph"]) + assert l.get_output_value("name") == ["Paragraph"] l.replace_xpath("name", ["//p/text()", "//div/text()"]) - self.assertEqual(l.get_output_value("name"), ["Paragraph", "Marta"]) + assert l.get_output_value("name") == ["Paragraph", "Marta"] def test_get_xpath(self): l = ProcessorItemLoader(response=self.response) - self.assertEqual(l.get_xpath("//p/text()"), ["paragraph"]) - self.assertEqual(l.get_xpath("//p/text()", TakeFirst()), "paragraph") - self.assertEqual(l.get_xpath("//p/text()", TakeFirst(), re="pa"), "pa") + assert l.get_xpath("//p/text()") == ["paragraph"] + assert l.get_xpath("//p/text()", TakeFirst()) == "paragraph" + assert l.get_xpath("//p/text()", TakeFirst(), re="pa") == "pa" - self.assertEqual( - l.get_xpath(["//p/text()", "//div/text()"]), ["paragraph", "marta"] - ) + assert l.get_xpath(["//p/text()", "//div/text()"]) == ["paragraph", "marta"] def test_replace_xpath_multi_fields(self): l = ProcessorItemLoader(response=self.response) l.add_xpath(None, "//div/text()", TakeFirst(), lambda x: {"name": x}) - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.replace_xpath(None, "//p/text()", TakeFirst(), lambda x: {"name": x}) - self.assertEqual(l.get_output_value("name"), ["Paragraph"]) + assert l.get_output_value("name") == ["Paragraph"] def test_replace_xpath_re(self): l = ProcessorItemLoader(response=self.response) - self.assertTrue(l.selector) + assert l.selector l.add_xpath("name", "//div/text()") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.replace_xpath("name", "//div/text()", re="ma") - self.assertEqual(l.get_output_value("name"), ["Ma"]) + assert l.get_output_value("name") == ["Ma"] def test_add_css_re(self): l = ProcessorItemLoader(response=self.response) l.add_css("name", "div::text", re="ma") - self.assertEqual(l.get_output_value("name"), ["Ma"]) + assert l.get_output_value("name") == ["Ma"] l.add_css("url", "a::attr(href)", re="http://(.+)") - self.assertEqual(l.get_output_value("url"), ["www.scrapy.org"]) + assert l.get_output_value("url") == ["www.scrapy.org"] def test_replace_css(self): l = ProcessorItemLoader(response=self.response) - self.assertTrue(l.selector) + assert l.selector l.add_css("name", "div::text") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.replace_css("name", "p::text") - self.assertEqual(l.get_output_value("name"), ["Paragraph"]) + assert l.get_output_value("name") == ["Paragraph"] l.replace_css("name", ["p::text", "div::text"]) - self.assertEqual(l.get_output_value("name"), ["Paragraph", "Marta"]) + assert l.get_output_value("name") == ["Paragraph", "Marta"] l.add_css("url", "a::attr(href)", re="http://(.+)") - self.assertEqual(l.get_output_value("url"), ["www.scrapy.org"]) + assert l.get_output_value("url") == ["www.scrapy.org"] l.replace_css("url", "img::attr(src)") - self.assertEqual(l.get_output_value("url"), ["/images/logo.png"]) + assert l.get_output_value("url") == ["/images/logo.png"] def test_get_css(self): l = ProcessorItemLoader(response=self.response) - self.assertEqual(l.get_css("p::text"), ["paragraph"]) - self.assertEqual(l.get_css("p::text", TakeFirst()), "paragraph") - self.assertEqual(l.get_css("p::text", TakeFirst(), re="pa"), "pa") + assert l.get_css("p::text") == ["paragraph"] + assert l.get_css("p::text", TakeFirst()) == "paragraph" + assert l.get_css("p::text", TakeFirst(), re="pa") == "pa" - self.assertEqual(l.get_css(["p::text", "div::text"]), ["paragraph", "marta"]) - self.assertEqual( - l.get_css(["a::attr(href)", "img::attr(src)"]), - ["http://www.scrapy.org", "/images/logo.png"], - ) + assert l.get_css(["p::text", "div::text"]) == ["paragraph", "marta"] + assert l.get_css(["a::attr(href)", "img::attr(src)"]) == [ + "http://www.scrapy.org", + "/images/logo.png", + ] def test_replace_css_multi_fields(self): l = ProcessorItemLoader(response=self.response) l.add_css(None, "div::text", TakeFirst(), lambda x: {"name": x}) - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.replace_css(None, "p::text", TakeFirst(), lambda x: {"name": x}) - self.assertEqual(l.get_output_value("name"), ["Paragraph"]) + assert l.get_output_value("name") == ["Paragraph"] l.add_css(None, "a::attr(href)", TakeFirst(), lambda x: {"url": x}) - self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + assert l.get_output_value("url") == ["http://www.scrapy.org"] l.replace_css(None, "img::attr(src)", TakeFirst(), lambda x: {"url": x}) - self.assertEqual(l.get_output_value("url"), ["/images/logo.png"]) + assert l.get_output_value("url") == ["/images/logo.png"] def test_replace_css_re(self): l = ProcessorItemLoader(response=self.response) - self.assertTrue(l.selector) + assert l.selector l.add_css("url", "a::attr(href)") - self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + assert l.get_output_value("url") == ["http://www.scrapy.org"] l.replace_css("url", "a::attr(href)", re=r"http://www\.(.+)") - self.assertEqual(l.get_output_value("url"), ["scrapy.org"]) + assert l.get_output_value("url") == ["scrapy.org"] -class SubselectorLoaderTest(unittest.TestCase): +class TestSubselectorLoader: response = HtmlResponse( url="", encoding="utf-8", @@ -483,17 +476,13 @@ class SubselectorLoaderTest(unittest.TestCase): nl.add_css("name_div", "#id") nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall()) - self.assertEqual(l.get_output_value("name"), ["marta"]) - self.assertEqual(l.get_output_value("name_div"), ['
marta
']) - self.assertEqual(l.get_output_value("name_value"), ["marta"]) + assert l.get_output_value("name") == ["marta"] + assert l.get_output_value("name_div") == ['
marta
'] + assert l.get_output_value("name_value") == ["marta"] - self.assertEqual(l.get_output_value("name"), nl.get_output_value("name")) - self.assertEqual( - l.get_output_value("name_div"), nl.get_output_value("name_div") - ) - self.assertEqual( - l.get_output_value("name_value"), nl.get_output_value("name_value") - ) + assert l.get_output_value("name") == nl.get_output_value("name") + assert l.get_output_value("name_div") == nl.get_output_value("name_div") + assert l.get_output_value("name_value") == nl.get_output_value("name_value") def test_nested_css(self): l = NestedItemLoader(response=self.response) @@ -502,17 +491,13 @@ class SubselectorLoaderTest(unittest.TestCase): nl.add_css("name_div", "#id") nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall()) - self.assertEqual(l.get_output_value("name"), ["marta"]) - self.assertEqual(l.get_output_value("name_div"), ['
marta
']) - self.assertEqual(l.get_output_value("name_value"), ["marta"]) + assert l.get_output_value("name") == ["marta"] + assert l.get_output_value("name_div") == ['
marta
'] + assert l.get_output_value("name_value") == ["marta"] - self.assertEqual(l.get_output_value("name"), nl.get_output_value("name")) - self.assertEqual( - l.get_output_value("name_div"), nl.get_output_value("name_div") - ) - self.assertEqual( - l.get_output_value("name_value"), nl.get_output_value("name_value") - ) + assert l.get_output_value("name") == nl.get_output_value("name") + assert l.get_output_value("name_div") == nl.get_output_value("name_div") + assert l.get_output_value("name_value") == nl.get_output_value("name_value") def test_nested_replace(self): l = NestedItemLoader(response=self.response) @@ -520,11 +505,11 @@ class SubselectorLoaderTest(unittest.TestCase): nl2 = nl1.nested_xpath("a") l.add_xpath("url", "//footer/a/@href") - self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + assert l.get_output_value("url") == ["http://www.scrapy.org"] nl1.replace_xpath("url", "img/@src") - self.assertEqual(l.get_output_value("url"), ["/images/logo.png"]) + assert l.get_output_value("url") == ["/images/logo.png"] nl2.replace_xpath("url", "@href") - self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + assert l.get_output_value("url") == ["http://www.scrapy.org"] def test_nested_ordering(self): l = NestedItemLoader(response=self.response) @@ -536,15 +521,12 @@ class SubselectorLoaderTest(unittest.TestCase): nl2.add_xpath("url", "text()") l.add_xpath("url", "//footer/a/@href") - self.assertEqual( - l.get_output_value("url"), - [ - "/images/logo.png", - "http://www.scrapy.org", - "homepage", - "http://www.scrapy.org", - ], - ) + assert l.get_output_value("url") == [ + "/images/logo.png", + "http://www.scrapy.org", + "homepage", + "http://www.scrapy.org", + ] def test_nested_load_item(self): l = NestedItemLoader(response=self.response) @@ -561,9 +543,9 @@ class SubselectorLoaderTest(unittest.TestCase): assert item is nl1.item assert item is nl2.item - self.assertEqual(item["name"], ["marta"]) - self.assertEqual(item["url"], ["http://www.scrapy.org"]) - self.assertEqual(item["image"], ["/images/logo.png"]) + assert item["name"] == ["marta"] + assert item["url"] == ["http://www.scrapy.org"] + assert item["image"] == ["/images/logo.png"] # Functions as processors @@ -588,9 +570,9 @@ class FunctionProcessorItemLoader(ItemLoader): default_item_class = FunctionProcessorItem -class FunctionProcessorTestCase(unittest.TestCase): +class TestFunctionProcessor: def test_processor_defined_in_item(self): lo = FunctionProcessorItemLoader() lo.add_value("foo", " bar ") lo.add_value("foo", [" asdf ", " qwerty "]) - self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]}) + assert dict(lo.load_item()) == {"foo": ["BAR", "ASDF", "QWERTY"]} diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 909b365a9..d7d900546 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -1,4 +1,6 @@ -import unittest +# pylint: disable=unsubscriptable-object,unsupported-membership-test,use-implicit-booleaness-not-comparison +# (too many false positives) + from unittest import mock import pytest @@ -14,31 +16,31 @@ from scrapy.settings import ( from . import default_settings -class SettingsGlobalFuncsTest(unittest.TestCase): +class TestSettingsGlobalFuncs: def test_get_settings_priority(self): for prio_str, prio_num in SETTINGS_PRIORITIES.items(): - self.assertEqual(get_settings_priority(prio_str), prio_num) - self.assertEqual(get_settings_priority(99), 99) + assert get_settings_priority(prio_str) == prio_num + assert get_settings_priority(99) == 99 -class SettingsAttributeTest(unittest.TestCase): - def setUp(self): +class TestSettingsAttribute: + def setup_method(self): self.attribute = SettingsAttribute("value", 10) def test_set_greater_priority(self): self.attribute.set("value2", 20) - self.assertEqual(self.attribute.value, "value2") - self.assertEqual(self.attribute.priority, 20) + assert self.attribute.value == "value2" + assert self.attribute.priority == 20 def test_set_equal_priority(self): self.attribute.set("value2", 10) - self.assertEqual(self.attribute.value, "value2") - self.assertEqual(self.attribute.priority, 10) + assert self.attribute.value == "value2" + assert self.attribute.priority == 10 def test_set_less_priority(self): self.attribute.set("value2", 0) - self.assertEqual(self.attribute.value, "value") - self.assertEqual(self.attribute.priority, 10) + assert self.attribute.value == "value" + assert self.attribute.priority == 10 def test_overwrite_basesettings(self): original_dict = {"one": 10, "two": 20} @@ -47,61 +49,59 @@ class SettingsAttributeTest(unittest.TestCase): new_dict = {"three": 11, "four": 21} attribute.set(new_dict, 10) - self.assertIsInstance(attribute.value, BaseSettings) - self.assertCountEqual(attribute.value, new_dict) - self.assertCountEqual(original_settings, original_dict) + assert isinstance(attribute.value, BaseSettings) + assert set(attribute.value) == set(new_dict) + assert set(original_settings) == set(original_dict) new_settings = BaseSettings({"five": 12}, 0) attribute.set(new_settings, 0) # Insufficient priority - self.assertCountEqual(attribute.value, new_dict) + assert set(attribute.value) == set(new_dict) attribute.set(new_settings, 10) - self.assertCountEqual(attribute.value, new_settings) + assert set(attribute.value) == set(new_settings) def test_repr(self): - self.assertEqual( - repr(self.attribute), "" - ) + assert repr(self.attribute) == "" -class BaseSettingsTest(unittest.TestCase): - def setUp(self): +class TestBaseSettings: + def setup_method(self): self.settings = BaseSettings() def test_setdefault_not_existing_value(self): settings = BaseSettings() value = settings.setdefault("TEST_OPTION", "value") - self.assertEqual(settings["TEST_OPTION"], "value") - self.assertEqual(value, "value") - self.assertIsNotNone(value) + assert settings["TEST_OPTION"] == "value" + assert value == "value" + assert value is not None def test_setdefault_existing_value(self): settings = BaseSettings({"TEST_OPTION": "value"}) value = settings.setdefault("TEST_OPTION", None) - self.assertEqual(settings["TEST_OPTION"], "value") - self.assertEqual(value, "value") + assert settings["TEST_OPTION"] == "value" + assert value == "value" def test_set_new_attribute(self): self.settings.set("TEST_OPTION", "value", 0) - self.assertIn("TEST_OPTION", self.settings.attributes) + assert "TEST_OPTION" in self.settings.attributes attr = self.settings.attributes["TEST_OPTION"] - self.assertIsInstance(attr, SettingsAttribute) - self.assertEqual(attr.value, "value") - self.assertEqual(attr.priority, 0) + assert isinstance(attr, SettingsAttribute) + assert attr.value == "value" + assert attr.priority == 0 def test_set_settingsattribute(self): myattr = SettingsAttribute(0, 30) # Note priority 30 self.settings.set("TEST_ATTR", myattr, 10) - self.assertEqual(self.settings.get("TEST_ATTR"), 0) - self.assertEqual(self.settings.getpriority("TEST_ATTR"), 30) + assert self.settings.get("TEST_ATTR") == 0 + assert self.settings.getpriority("TEST_ATTR") == 30 def test_set_instance_identity_on_update(self): attr = SettingsAttribute("value", 0) self.settings.attributes = {"TEST_OPTION": attr} self.settings.set("TEST_OPTION", "othervalue", 10) - self.assertIn("TEST_OPTION", self.settings.attributes) - self.assertIs(attr, self.settings.attributes["TEST_OPTION"]) + assert "TEST_OPTION" in self.settings.attributes + assert attr is self.settings.attributes["TEST_OPTION"] def test_set_calls_settings_attributes_methods_on_update(self): attr = SettingsAttribute("value", 10) @@ -114,7 +114,7 @@ class BaseSettingsTest(unittest.TestCase): for priority in (0, 10, 20): self.settings.set("TEST_OPTION", "othervalue", priority) mock_set.assert_called_once_with("othervalue", priority) - self.assertFalse(mock_setattr.called) + assert not mock_setattr.called mock_set.reset_mock() mock_setattr.reset_mock() @@ -122,19 +122,19 @@ class BaseSettingsTest(unittest.TestCase): settings = BaseSettings() settings.set("key", "a", "default") settings["key"] = "b" - self.assertEqual(settings["key"], "b") - self.assertEqual(settings.getpriority("key"), 20) + assert settings["key"] == "b" + assert settings.getpriority("key") == 20 settings["key"] = "c" - self.assertEqual(settings["key"], "c") + assert settings["key"] == "c" settings["key2"] = "x" - self.assertIn("key2", settings) - self.assertEqual(settings["key2"], "x") - self.assertEqual(settings.getpriority("key2"), 20) + assert "key2" in settings + assert settings["key2"] == "x" + assert settings.getpriority("key2") == 20 def test_setdict_alias(self): with mock.patch.object(self.settings, "set") as mock_set: self.settings.setdict({"TEST_1": "value1", "TEST_2": "value2"}, 10) - self.assertEqual(mock_set.call_count, 2) + assert mock_set.call_count == 2 calls = [ mock.call("TEST_1", "value1", 10), mock.call("TEST_2", "value2", 10), @@ -149,10 +149,10 @@ class BaseSettingsTest(unittest.TestCase): self.settings.attributes = {} self.settings.setmodule(ModuleMock(), 10) - self.assertIn("UPPERCASE_VAR", self.settings.attributes) - self.assertNotIn("MIXEDcase_VAR", self.settings.attributes) - self.assertNotIn("lowercase_var", self.settings.attributes) - self.assertEqual(len(self.settings.attributes), 1) + assert "UPPERCASE_VAR" in self.settings.attributes + assert "MIXEDcase_VAR" not in self.settings.attributes + assert "lowercase_var" not in self.settings.attributes + assert len(self.settings.attributes) == 1 def test_setmodule_alias(self): with mock.patch.object(self.settings, "set") as mock_set: @@ -168,13 +168,13 @@ class BaseSettingsTest(unittest.TestCase): self.settings.attributes = {} self.settings.setmodule("tests.test_settings.default_settings", 10) - self.assertCountEqual(self.settings.attributes.keys(), ctrl_attributes.keys()) + assert set(self.settings.attributes) == set(ctrl_attributes) for key in ctrl_attributes: attr = self.settings.attributes[key] ctrl_attr = ctrl_attributes[key] - self.assertEqual(attr.value, ctrl_attr.value) - self.assertEqual(attr.priority, ctrl_attr.priority) + assert attr.value == ctrl_attr.value + assert attr.priority == ctrl_attr.priority def test_update(self): settings = BaseSettings({"key_lowprio": 0}, priority=0) @@ -186,21 +186,21 @@ class BaseSettingsTest(unittest.TestCase): custom_dict = {"key_lowprio": 2, "key_highprio": 12, "newkey_two": None} settings.update(custom_dict, priority=20) - self.assertEqual(settings["key_lowprio"], 2) - self.assertEqual(settings.getpriority("key_lowprio"), 20) - self.assertEqual(settings["key_highprio"], 10) - self.assertIn("newkey_two", settings) - self.assertEqual(settings.getpriority("newkey_two"), 20) + assert settings["key_lowprio"] == 2 + assert settings.getpriority("key_lowprio") == 20 + assert settings["key_highprio"] == 10 + assert "newkey_two" in settings + assert settings.getpriority("newkey_two") == 20 settings.update(custom_settings) - self.assertEqual(settings["key_lowprio"], 1) - self.assertEqual(settings.getpriority("key_lowprio"), 30) - self.assertEqual(settings["key_highprio"], 10) - self.assertIn("newkey_one", settings) - self.assertEqual(settings.getpriority("newkey_one"), 50) + assert settings["key_lowprio"] == 1 + assert settings.getpriority("key_lowprio") == 30 + assert settings["key_highprio"] == 10 + assert "newkey_one" in settings + assert settings.getpriority("newkey_one") == 50 settings.update({"key_lowprio": 3}, priority=20) - self.assertEqual(settings["key_lowprio"], 1) + assert settings["key_lowprio"] == 1 @pytest.mark.xfail( raises=TypeError, reason="BaseSettings.update doesn't support kwargs input" @@ -220,21 +220,21 @@ class BaseSettingsTest(unittest.TestCase): def test_update_jsonstring(self): settings = BaseSettings({"number": 0, "dict": BaseSettings({"key": "val"})}) settings.update('{"number": 1, "newnumber": 2}') - self.assertEqual(settings["number"], 1) - self.assertEqual(settings["newnumber"], 2) + assert settings["number"] == 1 + assert settings["newnumber"] == 2 settings.set("dict", '{"key": "newval", "newkey": "newval2"}') - self.assertEqual(settings["dict"]["key"], "newval") - self.assertEqual(settings["dict"]["newkey"], "newval2") + assert settings["dict"]["key"] == "newval" + assert settings["dict"]["newkey"] == "newval2" def test_delete(self): settings = BaseSettings({"key": None}) settings.set("key_highprio", None, priority=50) settings.delete("key") settings.delete("key_highprio") - self.assertNotIn("key", settings) - self.assertIn("key_highprio", settings) + assert "key" not in settings + assert "key_highprio" in settings del settings["key_highprio"] - self.assertNotIn("key_highprio", settings) + assert "key_highprio" not in settings with pytest.raises(KeyError): settings.delete("notkey") with pytest.raises(KeyError): @@ -271,40 +271,40 @@ class BaseSettingsTest(unittest.TestCase): for key, value in test_configuration.items() } - self.assertTrue(settings.getbool("TEST_ENABLED1")) - self.assertTrue(settings.getbool("TEST_ENABLED2")) - self.assertTrue(settings.getbool("TEST_ENABLED3")) - self.assertTrue(settings.getbool("TEST_ENABLED4")) - self.assertTrue(settings.getbool("TEST_ENABLED5")) - self.assertFalse(settings.getbool("TEST_ENABLEDx")) - self.assertTrue(settings.getbool("TEST_ENABLEDx", True)) - self.assertFalse(settings.getbool("TEST_DISABLED1")) - self.assertFalse(settings.getbool("TEST_DISABLED2")) - self.assertFalse(settings.getbool("TEST_DISABLED3")) - self.assertFalse(settings.getbool("TEST_DISABLED4")) - self.assertFalse(settings.getbool("TEST_DISABLED5")) - self.assertEqual(settings.getint("TEST_INT1"), 123) - self.assertEqual(settings.getint("TEST_INT2"), 123) - self.assertEqual(settings.getint("TEST_INTx"), 0) - self.assertEqual(settings.getint("TEST_INTx", 45), 45) - self.assertEqual(settings.getfloat("TEST_FLOAT1"), 123.45) - self.assertEqual(settings.getfloat("TEST_FLOAT2"), 123.45) - self.assertEqual(settings.getfloat("TEST_FLOATx"), 0.0) - self.assertEqual(settings.getfloat("TEST_FLOATx", 55.0), 55.0) - self.assertEqual(settings.getlist("TEST_LIST1"), ["one", "two"]) - self.assertEqual(settings.getlist("TEST_LIST2"), ["one", "two"]) - self.assertEqual(settings.getlist("TEST_LIST3"), []) - self.assertEqual(settings.getlist("TEST_LISTx"), []) - self.assertEqual(settings.getlist("TEST_LISTx", ["default"]), ["default"]) - self.assertEqual(settings["TEST_STR"], "value") - self.assertEqual(settings.get("TEST_STR"), "value") - self.assertEqual(settings["TEST_STRx"], None) - self.assertEqual(settings.get("TEST_STRx"), None) - self.assertEqual(settings.get("TEST_STRx", "default"), "default") - self.assertEqual(settings.getdict("TEST_DICT1"), {"key1": "val1", "ke2": 3}) - self.assertEqual(settings.getdict("TEST_DICT2"), {"key1": "val1", "ke2": 3}) - self.assertEqual(settings.getdict("TEST_DICT3"), {}) - self.assertEqual(settings.getdict("TEST_DICT3", {"key1": 5}), {"key1": 5}) + assert settings.getbool("TEST_ENABLED1") + assert settings.getbool("TEST_ENABLED2") + assert settings.getbool("TEST_ENABLED3") + assert settings.getbool("TEST_ENABLED4") + assert settings.getbool("TEST_ENABLED5") + assert not settings.getbool("TEST_ENABLEDx") + assert settings.getbool("TEST_ENABLEDx", True) + assert not settings.getbool("TEST_DISABLED1") + assert not settings.getbool("TEST_DISABLED2") + assert not settings.getbool("TEST_DISABLED3") + assert not settings.getbool("TEST_DISABLED4") + assert not settings.getbool("TEST_DISABLED5") + assert settings.getint("TEST_INT1") == 123 + assert settings.getint("TEST_INT2") == 123 + assert settings.getint("TEST_INTx") == 0 + assert settings.getint("TEST_INTx", 45) == 45 + assert settings.getfloat("TEST_FLOAT1") == 123.45 + assert settings.getfloat("TEST_FLOAT2") == 123.45 + assert settings.getfloat("TEST_FLOATx") == 0.0 + assert settings.getfloat("TEST_FLOATx", 55.0) == 55.0 + assert settings.getlist("TEST_LIST1") == ["one", "two"] + assert settings.getlist("TEST_LIST2") == ["one", "two"] + assert settings.getlist("TEST_LIST3") == [] + assert settings.getlist("TEST_LISTx") == [] + assert settings.getlist("TEST_LISTx", ["default"]) == ["default"] + assert settings["TEST_STR"] == "value" + assert settings.get("TEST_STR") == "value" + assert settings["TEST_STRx"] is None + assert settings.get("TEST_STRx") is None + assert settings.get("TEST_STRx", "default") == "default" + assert settings.getdict("TEST_DICT1") == {"key1": "val1", "ke2": 3} + assert settings.getdict("TEST_DICT2") == {"key1": "val1", "ke2": 3} + assert settings.getdict("TEST_DICT3") == {} + assert settings.getdict("TEST_DICT3", {"key1": 5}) == {"key1": 5} with pytest.raises( ValueError, match="dictionary update sequence element #0 has length 3; 2 is required|sequence of pairs expected", @@ -321,8 +321,8 @@ class BaseSettingsTest(unittest.TestCase): def test_getpriority(self): settings = BaseSettings({"key": "value"}, priority=99) - self.assertEqual(settings.getpriority("key"), 99) - self.assertEqual(settings.getpriority("nonexistentkey"), None) + assert settings.getpriority("key") == 99 + assert settings.getpriority("nonexistentkey") is None def test_getwithbase(self): s = BaseSettings( @@ -333,16 +333,16 @@ class BaseSettingsTest(unittest.TestCase): } ) s["TEST"].set(2, 200, "cmdline") - self.assertCountEqual(s.getwithbase("TEST"), {1: 1, 2: 200, 3: 30}) - self.assertCountEqual(s.getwithbase("HASNOBASE"), s["HASNOBASE"]) - self.assertEqual(s.getwithbase("NONEXISTENT"), {}) + assert set(s.getwithbase("TEST")) == {1, 2, 3} + assert set(s.getwithbase("HASNOBASE")) == set(s["HASNOBASE"]) + assert s.getwithbase("NONEXISTENT") == {} def test_maxpriority(self): # Empty settings should return 'default' - self.assertEqual(self.settings.maxpriority(), 0) + assert self.settings.maxpriority() == 0 self.settings.set("A", 0, 10) self.settings.set("B", 0, 30) - self.assertEqual(self.settings.maxpriority(), 30) + assert self.settings.maxpriority() == 30 def test_copy(self): values = { @@ -356,17 +356,15 @@ class BaseSettingsTest(unittest.TestCase): self.settings.setdict(values) copy = self.settings.copy() self.settings.set("TEST_BOOL", False) - self.assertTrue(copy.get("TEST_BOOL")) + assert copy.get("TEST_BOOL") test_list = self.settings.get("TEST_LIST") test_list.append("three") - self.assertListEqual(copy.get("TEST_LIST"), ["one", "two"]) + assert copy.get("TEST_LIST") == ["one", "two"] test_list_of_lists = self.settings.get("TEST_LIST_OF_LISTS") test_list_of_lists[0].append("first_three") - self.assertListEqual( - copy.get("TEST_LIST_OF_LISTS")[0], ["first_one", "first_two"] - ) + assert copy.get("TEST_LIST_OF_LISTS")[0] == ["first_one", "first_two"] def test_copy_to_dict(self): s = BaseSettings( @@ -379,17 +377,14 @@ class BaseSettingsTest(unittest.TestCase): "HASNOBASE": BaseSettings({3: 3000}, "default"), } ) - self.assertDictEqual( - s.copy_to_dict(), - { - "HASNOBASE": {3: 3000}, - "TEST": {1: 10, 3: 30}, - "TEST_BASE": {1: 1, 2: 2}, - "TEST_LIST": [1, 2], - "TEST_BOOLEAN": False, - "TEST_STRING": "a string", - }, - ) + assert s.copy_to_dict() == { + "HASNOBASE": {3: 3000}, + "TEST": {1: 10, 3: 30}, + "TEST_BASE": {1: 1, 2: 2}, + "TEST_LIST": [1, 2], + "TEST_BOOLEAN": False, + "TEST_STRING": "a string", + } def test_freeze(self): self.settings.freeze() @@ -400,55 +395,55 @@ class BaseSettingsTest(unittest.TestCase): def test_frozencopy(self): frozencopy = self.settings.frozencopy() - self.assertTrue(frozencopy.frozen) - self.assertIsNot(frozencopy, self.settings) + assert frozencopy.frozen + assert frozencopy is not self.settings -class SettingsTest(unittest.TestCase): - def setUp(self): +class TestSettings: + def setup_method(self): self.settings = Settings() @mock.patch.dict("scrapy.settings.SETTINGS_PRIORITIES", {"default": 10}) @mock.patch("scrapy.settings.default_settings", default_settings) def test_initial_defaults(self): settings = Settings() - self.assertEqual(len(settings.attributes), 2) - self.assertIn("TEST_DEFAULT", settings.attributes) + assert len(settings.attributes) == 2 + assert "TEST_DEFAULT" in settings.attributes attr = settings.attributes["TEST_DEFAULT"] - self.assertIsInstance(attr, SettingsAttribute) - self.assertEqual(attr.value, "defvalue") - self.assertEqual(attr.priority, 10) + assert isinstance(attr, SettingsAttribute) + assert attr.value == "defvalue" + assert attr.priority == 10 @mock.patch.dict("scrapy.settings.SETTINGS_PRIORITIES", {}) @mock.patch("scrapy.settings.default_settings", {}) def test_initial_values(self): settings = Settings({"TEST_OPTION": "value"}, 10) - self.assertEqual(len(settings.attributes), 1) - self.assertIn("TEST_OPTION", settings.attributes) + assert len(settings.attributes) == 1 + assert "TEST_OPTION" in settings.attributes attr = settings.attributes["TEST_OPTION"] - self.assertIsInstance(attr, SettingsAttribute) - self.assertEqual(attr.value, "value") - self.assertEqual(attr.priority, 10) + assert isinstance(attr, SettingsAttribute) + assert attr.value == "value" + assert attr.priority == 10 @mock.patch("scrapy.settings.default_settings", default_settings) def test_autopromote_dicts(self): settings = Settings() mydict = settings.get("TEST_DICT") - self.assertIsInstance(mydict, BaseSettings) - self.assertIn("key", mydict) - self.assertEqual(mydict["key"], "val") # pylint: disable=unsubscriptable-object - self.assertEqual(mydict.getpriority("key"), 0) + assert isinstance(mydict, BaseSettings) + assert "key" in mydict + assert mydict["key"] == "val" + assert mydict.getpriority("key") == 0 @mock.patch("scrapy.settings.default_settings", default_settings) def test_getdict_autodegrade_basesettings(self): settings = Settings() mydict = settings.getdict("TEST_DICT") - self.assertIsInstance(mydict, dict) - self.assertEqual(len(mydict), 1) - self.assertIn("key", mydict) - self.assertEqual(mydict["key"], "val") + assert isinstance(mydict, dict) + assert len(mydict) == 1 + assert "key" in mydict + assert mydict["key"] == "val" def test_passing_objects_as_values(self): from scrapy.core.downloader.handlers.file import FileDownloadHandler @@ -470,19 +465,19 @@ class SettingsTest(unittest.TestCase): } ) - self.assertIn("ITEM_PIPELINES", settings.attributes) + assert "ITEM_PIPELINES" in settings.attributes mypipeline, priority = settings.getdict("ITEM_PIPELINES").popitem() - self.assertEqual(priority, 800) - self.assertEqual(mypipeline, TestPipeline) - self.assertIsInstance(mypipeline(), TestPipeline) - self.assertEqual(mypipeline().process_item("item", None), "item") + assert priority == 800 + assert mypipeline == TestPipeline + assert isinstance(mypipeline(), TestPipeline) + assert mypipeline().process_item("item", None) == "item" myhandler = settings.getdict("DOWNLOAD_HANDLERS").pop("ftp") - self.assertEqual(myhandler, FileDownloadHandler) + assert myhandler == FileDownloadHandler myhandler_instance = build_from_crawler(myhandler, get_crawler()) - self.assertIsInstance(myhandler_instance, FileDownloadHandler) - self.assertTrue(hasattr(myhandler_instance, "download_request")) + assert isinstance(myhandler_instance, FileDownloadHandler) + assert hasattr(myhandler_instance, "download_request") def test_pop_item_with_default_value(self): settings = Settings() @@ -491,14 +486,14 @@ class SettingsTest(unittest.TestCase): settings.pop("DUMMY_CONFIG") dummy_config_value = settings.pop("DUMMY_CONFIG", "dummy_value") - self.assertEqual(dummy_config_value, "dummy_value") + assert dummy_config_value == "dummy_value" def test_pop_item_with_immutable_settings(self): settings = Settings( {"DUMMY_CONFIG": "dummy_value", "OTHER_DUMMY_CONFIG": "other_dummy_value"} ) - self.assertEqual(settings.pop("DUMMY_CONFIG"), "dummy_value") + assert settings.pop("DUMMY_CONFIG") == "dummy_value" settings.freeze()