mirror of https://github.com/scrapy/scrapy.git
Converting tests to plain asserts, part 8. (#6711)
This commit is contained in:
Andrey Rakhmatullin
GitHub
parent
380c2279b9
commit
d442227fa7
File diff suppressed because it is too large
Load Diff
|
|
@ -1,5 +1,4 @@
|
|||
import codecs
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
|
@ -22,62 +21,56 @@ from scrapy.utils.python import to_unicode
|
|||
from tests import get_testdata
|
||||
|
||||
|
||||
class BaseResponseTest(unittest.TestCase):
|
||||
class TestResponseBase:
|
||||
response_class = Response
|
||||
|
||||
def test_init(self):
|
||||
# Response requires url in the constructor
|
||||
with pytest.raises(TypeError):
|
||||
self.response_class()
|
||||
self.assertTrue(
|
||||
isinstance(self.response_class("http://example.com/"), self.response_class)
|
||||
assert isinstance(
|
||||
self.response_class("http://example.com/"), self.response_class
|
||||
)
|
||||
with pytest.raises(TypeError):
|
||||
self.response_class(b"http://example.com")
|
||||
with pytest.raises(TypeError):
|
||||
self.response_class(url="http://example.com", body={})
|
||||
# body can be str or None
|
||||
self.assertTrue(
|
||||
isinstance(
|
||||
self.response_class("http://example.com/", body=b""),
|
||||
self.response_class,
|
||||
)
|
||||
assert isinstance(
|
||||
self.response_class("http://example.com/", body=b""),
|
||||
self.response_class,
|
||||
)
|
||||
self.assertTrue(
|
||||
isinstance(
|
||||
self.response_class("http://example.com/", body=b"body"),
|
||||
self.response_class,
|
||||
)
|
||||
assert isinstance(
|
||||
self.response_class("http://example.com/", body=b"body"),
|
||||
self.response_class,
|
||||
)
|
||||
# test presence of all optional parameters
|
||||
self.assertTrue(
|
||||
isinstance(
|
||||
self.response_class(
|
||||
"http://example.com/", body=b"", headers={}, status=200
|
||||
),
|
||||
self.response_class,
|
||||
)
|
||||
assert isinstance(
|
||||
self.response_class(
|
||||
"http://example.com/", body=b"", headers={}, status=200
|
||||
),
|
||||
self.response_class,
|
||||
)
|
||||
|
||||
r = self.response_class("http://www.example.com")
|
||||
assert isinstance(r.url, str)
|
||||
self.assertEqual(r.url, "http://www.example.com")
|
||||
self.assertEqual(r.status, 200)
|
||||
assert r.url == "http://www.example.com"
|
||||
assert r.status == 200
|
||||
|
||||
assert isinstance(r.headers, Headers)
|
||||
self.assertEqual(r.headers, {})
|
||||
assert not r.headers
|
||||
|
||||
headers = {"foo": "bar"}
|
||||
body = b"a body"
|
||||
r = self.response_class("http://www.example.com", headers=headers, body=body)
|
||||
|
||||
assert r.headers is not headers
|
||||
self.assertEqual(r.headers[b"foo"], b"bar")
|
||||
assert r.headers[b"foo"] == b"bar"
|
||||
|
||||
r = self.response_class("http://www.example.com", status=301)
|
||||
self.assertEqual(r.status, 301)
|
||||
assert r.status == 301
|
||||
r = self.response_class("http://www.example.com", status="301")
|
||||
self.assertEqual(r.status, 301)
|
||||
assert r.status == 301
|
||||
with pytest.raises(ValueError, match=r"invalid literal for int\(\)"):
|
||||
self.response_class("http://example.com", status="lala200")
|
||||
|
||||
|
|
@ -88,18 +81,18 @@ class BaseResponseTest(unittest.TestCase):
|
|||
r1.flags.append("cached")
|
||||
r2 = r1.copy()
|
||||
|
||||
self.assertEqual(r1.status, r2.status)
|
||||
self.assertEqual(r1.body, r2.body)
|
||||
assert r1.status == r2.status
|
||||
assert r1.body == r2.body
|
||||
|
||||
# make sure flags list is shallow copied
|
||||
assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
|
||||
self.assertEqual(r1.flags, r2.flags)
|
||||
assert r1.flags == r2.flags
|
||||
|
||||
# make sure headers attribute is shallow copied
|
||||
assert r1.headers is not r2.headers, (
|
||||
"headers must be a shallow copy, not identical"
|
||||
)
|
||||
self.assertEqual(r1.headers, r2.headers)
|
||||
assert r1.headers == r2.headers
|
||||
|
||||
def test_copy_meta(self):
|
||||
req = Request("http://www.example.com")
|
||||
|
|
@ -144,16 +137,16 @@ class BaseResponseTest(unittest.TestCase):
|
|||
r1 = self.response_class("http://www.example.com")
|
||||
r2 = r1.replace(status=301, body=b"New body", headers=hdrs)
|
||||
assert r1.body == b""
|
||||
self.assertEqual(r1.url, r2.url)
|
||||
self.assertEqual((r1.status, r2.status), (200, 301))
|
||||
self.assertEqual((r1.body, r2.body), (b"", b"New body"))
|
||||
self.assertEqual((r1.headers, r2.headers), ({}, hdrs))
|
||||
assert r1.url == r2.url
|
||||
assert (r1.status, r2.status) == (200, 301)
|
||||
assert (r1.body, r2.body) == (b"", b"New body")
|
||||
assert (r1.headers, r2.headers) == ({}, hdrs)
|
||||
|
||||
# Empty attributes (which may fail if not compared properly)
|
||||
r3 = self.response_class("http://www.example.com", flags=["cached"])
|
||||
r4 = r3.replace(body=b"", flags=[])
|
||||
self.assertEqual(r4.body, b"")
|
||||
self.assertEqual(r4.flags, [])
|
||||
assert r4.body == b""
|
||||
assert not r4.flags
|
||||
|
||||
def _assert_response_values(self, response, encoding, body):
|
||||
if isinstance(body, str):
|
||||
|
|
@ -166,11 +159,11 @@ class BaseResponseTest(unittest.TestCase):
|
|||
assert isinstance(response.body, bytes)
|
||||
assert isinstance(response.text, str)
|
||||
self._assert_response_encoding(response, encoding)
|
||||
self.assertEqual(response.body, body_bytes)
|
||||
self.assertEqual(response.text, body_unicode)
|
||||
assert response.body == body_bytes
|
||||
assert response.text == body_unicode
|
||||
|
||||
def _assert_response_encoding(self, response, encoding):
|
||||
self.assertEqual(response.encoding, resolve_encoding(encoding))
|
||||
assert response.encoding == resolve_encoding(encoding)
|
||||
|
||||
def test_immutable_attributes(self):
|
||||
r = self.response_class("http://example.com")
|
||||
|
|
@ -183,7 +176,7 @@ class BaseResponseTest(unittest.TestCase):
|
|||
"""Test urljoin shortcut (only for existence, since behavior equals urljoin)"""
|
||||
joined = self.response_class("http://www.example.com").urljoin("/test")
|
||||
absolute = "http://www.example.com/test"
|
||||
self.assertEqual(joined, absolute)
|
||||
assert joined == absolute
|
||||
|
||||
def test_shortcut_attributes(self):
|
||||
r = self.response_class("http://example.com", body=b"hello")
|
||||
|
|
@ -241,7 +234,7 @@ class BaseResponseTest(unittest.TestCase):
|
|||
def test_follow_flags(self):
|
||||
res = self.response_class("http://example.com/")
|
||||
fol = res.follow("http://example.com/", flags=["cached", "allowed"])
|
||||
self.assertEqual(fol.flags, ["cached", "allowed"])
|
||||
assert fol.flags == ["cached", "allowed"]
|
||||
|
||||
# Response.follow_all
|
||||
|
||||
|
|
@ -276,7 +269,7 @@ class BaseResponseTest(unittest.TestCase):
|
|||
|
||||
def test_follow_all_empty(self):
|
||||
r = self.response_class("http://example.com")
|
||||
self.assertEqual([], list(r.follow_all([])))
|
||||
assert not list(r.follow_all([]))
|
||||
|
||||
def test_follow_all_invalid(self):
|
||||
r = self.response_class("http://example.com")
|
||||
|
|
@ -327,13 +320,13 @@ class BaseResponseTest(unittest.TestCase):
|
|||
]
|
||||
fol = re.follow_all(urls, flags=["cached", "allowed"])
|
||||
for req in fol:
|
||||
self.assertEqual(req.flags, ["cached", "allowed"])
|
||||
assert req.flags == ["cached", "allowed"]
|
||||
|
||||
def _assert_followed_url(self, follow_obj, target_url, response=None):
|
||||
if response is None:
|
||||
response = self._links_response()
|
||||
req = response.follow(follow_obj)
|
||||
self.assertEqual(req.url, target_url)
|
||||
assert req.url == target_url
|
||||
return req
|
||||
|
||||
def _assert_followed_all_urls(self, follow_obj, target_urls, response=None):
|
||||
|
|
@ -341,7 +334,7 @@ class BaseResponseTest(unittest.TestCase):
|
|||
response = self._links_response()
|
||||
followed = response.follow_all(follow_obj)
|
||||
for req, target in zip(followed, target_urls):
|
||||
self.assertEqual(req.url, target)
|
||||
assert req.url == target
|
||||
yield req
|
||||
|
||||
def _links_response(self):
|
||||
|
|
@ -353,7 +346,7 @@ class BaseResponseTest(unittest.TestCase):
|
|||
return self.response_class("http://example.com/index", body=body)
|
||||
|
||||
|
||||
class TextResponseTest(BaseResponseTest):
|
||||
class TestTextResponse(TestResponseBase):
|
||||
response_class = TextResponse
|
||||
|
||||
def test_replace(self):
|
||||
|
|
@ -365,10 +358,10 @@ class TextResponseTest(BaseResponseTest):
|
|||
r3 = r1.replace(url="http://www.example.com/other", encoding="latin1")
|
||||
|
||||
assert isinstance(r2, self.response_class)
|
||||
self.assertEqual(r2.url, "http://www.example.com/other")
|
||||
assert r2.url == "http://www.example.com/other"
|
||||
self._assert_response_encoding(r2, "cp852")
|
||||
self.assertEqual(r3.url, "http://www.example.com/other")
|
||||
self.assertEqual(r3._declared_encoding(), "latin1")
|
||||
assert r3.url == "http://www.example.com/other"
|
||||
assert r3._declared_encoding() == "latin1"
|
||||
|
||||
def test_unicode_url(self):
|
||||
# instantiate with unicode url without encoding (should set default encoding)
|
||||
|
|
@ -382,21 +375,21 @@ class TextResponseTest(BaseResponseTest):
|
|||
resp = self.response_class(
|
||||
url="http://www.example.com/price/\xa3", encoding="utf-8"
|
||||
)
|
||||
self.assertEqual(resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3"))
|
||||
assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3")
|
||||
resp = self.response_class(
|
||||
url="http://www.example.com/price/\xa3", encoding="latin-1"
|
||||
)
|
||||
self.assertEqual(resp.url, "http://www.example.com/price/\xa3")
|
||||
assert resp.url == "http://www.example.com/price/\xa3"
|
||||
resp = self.response_class(
|
||||
"http://www.example.com/price/\xa3",
|
||||
headers={"Content-type": ["text/html; charset=utf-8"]},
|
||||
)
|
||||
self.assertEqual(resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3"))
|
||||
assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3")
|
||||
resp = self.response_class(
|
||||
"http://www.example.com/price/\xa3",
|
||||
headers={"Content-type": ["text/html; charset=iso-8859-1"]},
|
||||
)
|
||||
self.assertEqual(resp.url, "http://www.example.com/price/\xa3")
|
||||
assert resp.url == "http://www.example.com/price/\xa3"
|
||||
|
||||
def test_unicode_body(self):
|
||||
unicode_string = (
|
||||
|
|
@ -412,8 +405,8 @@ class TextResponseTest(BaseResponseTest):
|
|||
)
|
||||
|
||||
# check response.text
|
||||
self.assertTrue(isinstance(r1.text, str))
|
||||
self.assertEqual(r1.text, unicode_string)
|
||||
assert isinstance(r1.text, str)
|
||||
assert r1.text == unicode_string
|
||||
|
||||
def test_encoding(self):
|
||||
r1 = self.response_class(
|
||||
|
|
@ -458,18 +451,18 @@ class TextResponseTest(BaseResponseTest):
|
|||
},
|
||||
)
|
||||
|
||||
self.assertEqual(r1._headers_encoding(), "utf-8")
|
||||
self.assertEqual(r2._headers_encoding(), None)
|
||||
self.assertEqual(r2._declared_encoding(), "utf-8")
|
||||
assert r1._headers_encoding() == "utf-8"
|
||||
assert r2._headers_encoding() is None
|
||||
assert r2._declared_encoding() == "utf-8"
|
||||
self._assert_response_encoding(r2, "utf-8")
|
||||
self.assertEqual(r3._headers_encoding(), "cp1252")
|
||||
self.assertEqual(r3._declared_encoding(), "cp1252")
|
||||
self.assertEqual(r4._headers_encoding(), None)
|
||||
self.assertEqual(r5._headers_encoding(), None)
|
||||
self.assertEqual(r8._headers_encoding(), "cp1251")
|
||||
self.assertEqual(r9._headers_encoding(), None)
|
||||
self.assertEqual(r8._declared_encoding(), "utf-8")
|
||||
self.assertEqual(r9._declared_encoding(), None)
|
||||
assert r3._headers_encoding() == "cp1252"
|
||||
assert r3._declared_encoding() == "cp1252"
|
||||
assert r4._headers_encoding() is None
|
||||
assert r5._headers_encoding() is None
|
||||
assert r8._headers_encoding() == "cp1251"
|
||||
assert r9._headers_encoding() is None
|
||||
assert r8._declared_encoding() == "utf-8"
|
||||
assert r9._declared_encoding() is None
|
||||
self._assert_response_encoding(r5, "utf-8")
|
||||
self._assert_response_encoding(r8, "utf-8")
|
||||
self._assert_response_encoding(r9, "cp1252")
|
||||
|
|
@ -493,7 +486,7 @@ class TextResponseTest(BaseResponseTest):
|
|||
headers={"Content-type": ["text/html; charset=UNKNOWN"]},
|
||||
body=b"\xc2\xa3",
|
||||
)
|
||||
self.assertEqual(r._declared_encoding(), None)
|
||||
assert r._declared_encoding() is None
|
||||
self._assert_response_values(r, "utf-8", "\xa3")
|
||||
|
||||
def test_utf16(self):
|
||||
|
|
@ -511,14 +504,11 @@ class TextResponseTest(BaseResponseTest):
|
|||
headers={"Content-type": ["text/html; charset=utf-8"]},
|
||||
body=b"\xef\xbb\xbfWORD\xe3\xab",
|
||||
)
|
||||
self.assertEqual(r6.encoding, "utf-8")
|
||||
self.assertIn(
|
||||
r6.text,
|
||||
{
|
||||
"WORD\ufffd\ufffd", # w3lib < 1.19.0
|
||||
"WORD\ufffd", # w3lib >= 1.19.0
|
||||
},
|
||||
)
|
||||
assert r6.encoding == "utf-8"
|
||||
assert r6.text in {
|
||||
"WORD\ufffd\ufffd", # w3lib < 1.19.0
|
||||
"WORD\ufffd", # w3lib >= 1.19.0
|
||||
}
|
||||
|
||||
def test_bom_is_removed_from_body(self):
|
||||
# Inferring encoding from body also cache decoded body as sideeffect,
|
||||
|
|
@ -532,21 +522,21 @@ class TextResponseTest(BaseResponseTest):
|
|||
|
||||
# Test response without content-type and BOM encoding
|
||||
response = self.response_class(url, body=body)
|
||||
self.assertEqual(response.encoding, "utf-8")
|
||||
self.assertEqual(response.text, "WORD")
|
||||
assert response.encoding == "utf-8"
|
||||
assert response.text == "WORD"
|
||||
response = self.response_class(url, body=body)
|
||||
self.assertEqual(response.text, "WORD")
|
||||
self.assertEqual(response.encoding, "utf-8")
|
||||
assert response.text == "WORD"
|
||||
assert response.encoding == "utf-8"
|
||||
|
||||
# Body caching sideeffect isn't triggered when encoding is declared in
|
||||
# content-type header but BOM still need to be removed from decoded
|
||||
# body
|
||||
response = self.response_class(url, headers=headers, body=body)
|
||||
self.assertEqual(response.encoding, "utf-8")
|
||||
self.assertEqual(response.text, "WORD")
|
||||
assert response.encoding == "utf-8"
|
||||
assert response.text == "WORD"
|
||||
response = self.response_class(url, headers=headers, body=body)
|
||||
self.assertEqual(response.text, "WORD")
|
||||
self.assertEqual(response.encoding, "utf-8")
|
||||
assert response.text == "WORD"
|
||||
assert response.encoding == "utf-8"
|
||||
|
||||
def test_replace_wrong_encoding(self):
|
||||
"""Test invalid chars are replaced properly"""
|
||||
|
|
@ -577,49 +567,47 @@ class TextResponseTest(BaseResponseTest):
|
|||
body = b"<html><head><title>Some page</title><body></body></html>"
|
||||
response = self.response_class("http://www.example.com", body=body)
|
||||
|
||||
self.assertIsInstance(response.selector, Selector)
|
||||
self.assertEqual(response.selector.type, "html")
|
||||
self.assertIs(response.selector, response.selector) # property is cached
|
||||
self.assertIs(response.selector.response, response)
|
||||
assert isinstance(response.selector, Selector)
|
||||
assert response.selector.type == "html"
|
||||
assert response.selector is response.selector # property is cached
|
||||
assert response.selector.response is response
|
||||
|
||||
self.assertEqual(
|
||||
response.selector.xpath("//title/text()").getall(), ["Some page"]
|
||||
)
|
||||
self.assertEqual(response.selector.css("title::text").getall(), ["Some page"])
|
||||
self.assertEqual(response.selector.re("Some (.*)</title>"), ["page"])
|
||||
assert response.selector.xpath("//title/text()").getall() == ["Some page"]
|
||||
assert response.selector.css("title::text").getall() == ["Some page"]
|
||||
assert response.selector.re("Some (.*)</title>") == ["page"]
|
||||
|
||||
def test_selector_shortcuts(self):
|
||||
body = b"<html><head><title>Some page</title><body></body></html>"
|
||||
response = self.response_class("http://www.example.com", body=body)
|
||||
|
||||
self.assertEqual(
|
||||
response.xpath("//title/text()").getall(),
|
||||
response.selector.xpath("//title/text()").getall(),
|
||||
assert (
|
||||
response.xpath("//title/text()").getall()
|
||||
== response.selector.xpath("//title/text()").getall()
|
||||
)
|
||||
self.assertEqual(
|
||||
response.css("title::text").getall(),
|
||||
response.selector.css("title::text").getall(),
|
||||
assert (
|
||||
response.css("title::text").getall()
|
||||
== response.selector.css("title::text").getall()
|
||||
)
|
||||
|
||||
def test_selector_shortcuts_kwargs(self):
|
||||
body = b'<html><head><title>Some page</title><body><p class="content">A nice paragraph.</p></body></html>'
|
||||
response = self.response_class("http://www.example.com", body=body)
|
||||
|
||||
self.assertEqual(
|
||||
assert (
|
||||
response.xpath(
|
||||
"normalize-space(//p[@class=$pclass])", pclass="content"
|
||||
).getall(),
|
||||
response.xpath('normalize-space(//p[@class="content"])').getall(),
|
||||
).getall()
|
||||
== response.xpath('normalize-space(//p[@class="content"])').getall()
|
||||
)
|
||||
self.assertEqual(
|
||||
assert (
|
||||
response.xpath(
|
||||
"//title[count(following::p[@class=$pclass])=$pcount]/text()",
|
||||
pclass="content",
|
||||
pcount=1,
|
||||
).getall(),
|
||||
response.xpath(
|
||||
).getall()
|
||||
== response.xpath(
|
||||
'//title[count(following::p[@class="content"])=1]/text()'
|
||||
).getall(),
|
||||
).getall()
|
||||
)
|
||||
|
||||
def test_urljoin_with_base_url(self):
|
||||
|
|
@ -629,21 +617,21 @@ class TextResponseTest(BaseResponseTest):
|
|||
"/test"
|
||||
)
|
||||
absolute = "https://example.net/test"
|
||||
self.assertEqual(joined, absolute)
|
||||
assert joined == absolute
|
||||
|
||||
body = b'<html><body><base href="/elsewhere"></body></html>'
|
||||
joined = self.response_class("http://www.example.com", body=body).urljoin(
|
||||
"test"
|
||||
)
|
||||
absolute = "http://www.example.com/test"
|
||||
self.assertEqual(joined, absolute)
|
||||
assert joined == absolute
|
||||
|
||||
body = b'<html><body><base href="/elsewhere/"></body></html>'
|
||||
joined = self.response_class("http://www.example.com", body=body).urljoin(
|
||||
"test"
|
||||
)
|
||||
absolute = "http://www.example.com/elsewhere/test"
|
||||
self.assertEqual(joined, absolute)
|
||||
assert joined == absolute
|
||||
|
||||
def test_follow_selector(self):
|
||||
resp = self._links_response()
|
||||
|
|
@ -728,7 +716,7 @@ class TextResponseTest(BaseResponseTest):
|
|||
"http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82",
|
||||
response=resp1,
|
||||
)
|
||||
self.assertEqual(req.encoding, "utf8")
|
||||
assert req.encoding == "utf8"
|
||||
|
||||
resp2 = self.response_class(
|
||||
"http://example.com",
|
||||
|
|
@ -742,12 +730,12 @@ class TextResponseTest(BaseResponseTest):
|
|||
"http://example.com/foo?%EF%F0%E8%E2%E5%F2",
|
||||
response=resp2,
|
||||
)
|
||||
self.assertEqual(req.encoding, "cp1251")
|
||||
assert req.encoding == "cp1251"
|
||||
|
||||
def test_follow_flags(self):
|
||||
res = self.response_class("http://example.com/")
|
||||
fol = res.follow("http://example.com/", flags=["cached", "allowed"])
|
||||
self.assertEqual(fol.flags, ["cached", "allowed"])
|
||||
assert fol.flags == ["cached", "allowed"]
|
||||
|
||||
def test_follow_all_flags(self):
|
||||
re = self.response_class("http://www.example.com/")
|
||||
|
|
@ -758,7 +746,7 @@ class TextResponseTest(BaseResponseTest):
|
|||
]
|
||||
fol = re.follow_all(urls, flags=["cached", "allowed"])
|
||||
for req in fol:
|
||||
self.assertEqual(req.flags, ["cached", "allowed"])
|
||||
assert req.flags == ["cached", "allowed"]
|
||||
|
||||
def test_follow_all_css(self):
|
||||
expected = [
|
||||
|
|
@ -767,7 +755,7 @@ class TextResponseTest(BaseResponseTest):
|
|||
]
|
||||
response = self._links_response()
|
||||
extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')]
|
||||
self.assertEqual(expected, extracted)
|
||||
assert expected == extracted
|
||||
|
||||
def test_follow_all_css_skip_invalid(self):
|
||||
expected = [
|
||||
|
|
@ -777,9 +765,9 @@ class TextResponseTest(BaseResponseTest):
|
|||
]
|
||||
response = self._links_response_no_href()
|
||||
extracted1 = [r.url for r in response.follow_all(css=".pagination a")]
|
||||
self.assertEqual(expected, extracted1)
|
||||
assert expected == extracted1
|
||||
extracted2 = [r.url for r in response.follow_all(response.css(".pagination a"))]
|
||||
self.assertEqual(expected, extracted2)
|
||||
assert expected == extracted2
|
||||
|
||||
def test_follow_all_xpath(self):
|
||||
expected = [
|
||||
|
|
@ -788,7 +776,7 @@ class TextResponseTest(BaseResponseTest):
|
|||
]
|
||||
response = self._links_response()
|
||||
extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]')
|
||||
self.assertEqual(expected, [r.url for r in extracted])
|
||||
assert expected == [r.url for r in extracted]
|
||||
|
||||
def test_follow_all_xpath_skip_invalid(self):
|
||||
expected = [
|
||||
|
|
@ -800,12 +788,12 @@ class TextResponseTest(BaseResponseTest):
|
|||
extracted1 = [
|
||||
r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')
|
||||
]
|
||||
self.assertEqual(expected, extracted1)
|
||||
assert expected == extracted1
|
||||
extracted2 = [
|
||||
r.url
|
||||
for r in response.follow_all(response.xpath('//div[@id="pagination"]/a'))
|
||||
]
|
||||
self.assertEqual(expected, extracted2)
|
||||
assert expected == extracted2
|
||||
|
||||
def test_follow_all_too_many_arguments(self):
|
||||
response = self._links_response()
|
||||
|
|
@ -820,7 +808,7 @@ class TextResponseTest(BaseResponseTest):
|
|||
def test_json_response(self):
|
||||
json_body = b"""{"ip": "109.187.217.200"}"""
|
||||
json_response = self.response_class("http://www.example.com", body=json_body)
|
||||
self.assertEqual(json_response.json(), {"ip": "109.187.217.200"})
|
||||
assert json_response.json() == {"ip": "109.187.217.200"}
|
||||
|
||||
text_body = b"""<html><body>text</body></html>"""
|
||||
text_response = self.response_class("http://www.example.com", body=text_body)
|
||||
|
|
@ -842,7 +830,7 @@ class TextResponseTest(BaseResponseTest):
|
|||
mock_json.assert_called_once_with(json_body)
|
||||
|
||||
|
||||
class HtmlResponseTest(TextResponseTest):
|
||||
class TestHtmlResponse(TestTextResponse):
|
||||
response_class = HtmlResponse
|
||||
|
||||
def test_html_encoding(self):
|
||||
|
|
@ -883,7 +871,7 @@ class HtmlResponseTest(TextResponseTest):
|
|||
self._assert_response_values(r1, "gb2312", body)
|
||||
|
||||
|
||||
class XmlResponseTest(TextResponseTest):
|
||||
class TestXmlResponse(TestTextResponse):
|
||||
response_class = XmlResponse
|
||||
|
||||
def test_xml_encoding(self):
|
||||
|
|
@ -917,20 +905,20 @@ class XmlResponseTest(TextResponseTest):
|
|||
body = b'<?xml version="1.0" encoding="utf-8"?><xml><elem>value</elem></xml>'
|
||||
response = self.response_class("http://www.example.com", body=body)
|
||||
|
||||
self.assertIsInstance(response.selector, Selector)
|
||||
self.assertEqual(response.selector.type, "xml")
|
||||
self.assertIs(response.selector, response.selector) # property is cached
|
||||
self.assertIs(response.selector.response, response)
|
||||
assert isinstance(response.selector, Selector)
|
||||
assert response.selector.type == "xml"
|
||||
assert response.selector is response.selector # property is cached
|
||||
assert response.selector.response is response
|
||||
|
||||
self.assertEqual(response.selector.xpath("//elem/text()").getall(), ["value"])
|
||||
assert response.selector.xpath("//elem/text()").getall() == ["value"]
|
||||
|
||||
def test_selector_shortcuts(self):
|
||||
body = b'<?xml version="1.0" encoding="utf-8"?><xml><elem>value</elem></xml>'
|
||||
response = self.response_class("http://www.example.com", body=body)
|
||||
|
||||
self.assertEqual(
|
||||
response.xpath("//elem/text()").getall(),
|
||||
response.selector.xpath("//elem/text()").getall(),
|
||||
assert (
|
||||
response.xpath("//elem/text()").getall()
|
||||
== response.selector.xpath("//elem/text()").getall()
|
||||
)
|
||||
|
||||
def test_selector_shortcuts_kwargs(self):
|
||||
|
|
@ -940,21 +928,21 @@ class XmlResponseTest(TextResponseTest):
|
|||
</xml>"""
|
||||
response = self.response_class("http://www.example.com", body=body)
|
||||
|
||||
self.assertEqual(
|
||||
assert (
|
||||
response.xpath(
|
||||
"//s:elem/text()", namespaces={"s": "http://scrapy.org"}
|
||||
).getall(),
|
||||
response.selector.xpath(
|
||||
).getall()
|
||||
== response.selector.xpath(
|
||||
"//s:elem/text()", namespaces={"s": "http://scrapy.org"}
|
||||
).getall(),
|
||||
).getall()
|
||||
)
|
||||
|
||||
response.selector.register_namespace("s2", "http://scrapy.org")
|
||||
self.assertEqual(
|
||||
assert (
|
||||
response.xpath(
|
||||
"//s1:elem/text()", namespaces={"s1": "http://scrapy.org"}
|
||||
).getall(),
|
||||
response.selector.xpath("//s2:elem/text()").getall(),
|
||||
).getall()
|
||||
== response.selector.xpath("//s2:elem/text()").getall()
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -968,7 +956,7 @@ class CustomResponse(TextResponse):
|
|||
super().__init__(*args, **kwargs)
|
||||
|
||||
|
||||
class CustomResponseTest(TextResponseTest):
|
||||
class TestCustomResponse(TestTextResponse):
|
||||
response_class = CustomResponse
|
||||
|
||||
def test_copy(self):
|
||||
|
|
@ -981,11 +969,11 @@ class CustomResponseTest(TextResponseTest):
|
|||
lost="lost",
|
||||
)
|
||||
r2 = r1.copy()
|
||||
self.assertIsInstance(r2, self.response_class)
|
||||
self.assertEqual(r1.foo, r2.foo)
|
||||
self.assertEqual(r1.bar, r2.bar)
|
||||
self.assertEqual(r1.lost, "lost")
|
||||
self.assertIsNone(r2.lost)
|
||||
assert isinstance(r2, self.response_class)
|
||||
assert r1.foo == r2.foo
|
||||
assert r1.bar == r2.bar
|
||||
assert r1.lost == "lost"
|
||||
assert r2.lost is None
|
||||
|
||||
def test_replace(self):
|
||||
super().test_replace()
|
||||
|
|
@ -998,31 +986,31 @@ class CustomResponseTest(TextResponseTest):
|
|||
)
|
||||
|
||||
r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost")
|
||||
self.assertIsInstance(r2, self.response_class)
|
||||
self.assertEqual(r1.foo, "foo")
|
||||
self.assertEqual(r1.bar, "bar")
|
||||
self.assertEqual(r1.lost, "lost")
|
||||
self.assertEqual(r2.foo, "new-foo")
|
||||
self.assertEqual(r2.bar, "new-bar")
|
||||
self.assertEqual(r2.lost, "new-lost")
|
||||
assert isinstance(r2, self.response_class)
|
||||
assert r1.foo == "foo"
|
||||
assert r1.bar == "bar"
|
||||
assert r1.lost == "lost"
|
||||
assert r2.foo == "new-foo"
|
||||
assert r2.bar == "new-bar"
|
||||
assert r2.lost == "new-lost"
|
||||
|
||||
r3 = r1.replace(foo="new-foo", bar="new-bar")
|
||||
self.assertIsInstance(r3, self.response_class)
|
||||
self.assertEqual(r1.foo, "foo")
|
||||
self.assertEqual(r1.bar, "bar")
|
||||
self.assertEqual(r1.lost, "lost")
|
||||
self.assertEqual(r3.foo, "new-foo")
|
||||
self.assertEqual(r3.bar, "new-bar")
|
||||
self.assertIsNone(r3.lost)
|
||||
assert isinstance(r3, self.response_class)
|
||||
assert r1.foo == "foo"
|
||||
assert r1.bar == "bar"
|
||||
assert r1.lost == "lost"
|
||||
assert r3.foo == "new-foo"
|
||||
assert r3.bar == "new-bar"
|
||||
assert r3.lost is None
|
||||
|
||||
r4 = r1.replace(foo="new-foo")
|
||||
self.assertIsInstance(r4, self.response_class)
|
||||
self.assertEqual(r1.foo, "foo")
|
||||
self.assertEqual(r1.bar, "bar")
|
||||
self.assertEqual(r1.lost, "lost")
|
||||
self.assertEqual(r4.foo, "new-foo")
|
||||
self.assertEqual(r4.bar, "bar")
|
||||
self.assertIsNone(r4.lost)
|
||||
assert isinstance(r4, self.response_class)
|
||||
assert r1.foo == "foo"
|
||||
assert r1.bar == "bar"
|
||||
assert r1.lost == "lost"
|
||||
assert r4.foo == "new-foo"
|
||||
assert r4.bar == "bar"
|
||||
assert r4.lost is None
|
||||
|
||||
with pytest.raises(
|
||||
TypeError,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import unittest
|
||||
|
||||
import attr
|
||||
import pytest
|
||||
|
|
@ -67,7 +66,7 @@ def processor_with_args(value, other=None, loader_context=None):
|
|||
return value
|
||||
|
||||
|
||||
class BasicItemLoaderTest(unittest.TestCase):
|
||||
class TestBasicItemLoader:
|
||||
def test_add_value_on_unknown_field(self):
|
||||
il = ProcessorItemLoader()
|
||||
with pytest.raises(KeyError):
|
||||
|
|
@ -80,14 +79,14 @@ class BasicItemLoaderTest(unittest.TestCase):
|
|||
il.add_value("name", "marta")
|
||||
item = il.load_item()
|
||||
assert item is i
|
||||
self.assertEqual(item["summary"], ["lala"])
|
||||
self.assertEqual(item["name"], ["marta"])
|
||||
assert item["summary"] == ["lala"]
|
||||
assert item["name"] == ["marta"]
|
||||
|
||||
def test_load_item_using_custom_loader(self):
|
||||
il = ProcessorItemLoader()
|
||||
il.add_value("name", "marta")
|
||||
item = il.load_item()
|
||||
self.assertEqual(item["name"], ["Marta"])
|
||||
assert item["name"] == ["Marta"]
|
||||
|
||||
|
||||
class InitializationTestMixin:
|
||||
|
|
@ -98,16 +97,16 @@ class InitializationTestMixin:
|
|||
input_item = self.item_class(name="foo")
|
||||
il = ItemLoader(item=input_item)
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo"]})
|
||||
assert isinstance(loaded_item, self.item_class)
|
||||
assert ItemAdapter(loaded_item).asdict() == {"name": ["foo"]}
|
||||
|
||||
def test_keep_list(self):
|
||||
"""Loaded item should contain values from the initial item"""
|
||||
input_item = self.item_class(name=["foo", "bar"])
|
||||
il = ItemLoader(item=input_item)
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]})
|
||||
assert isinstance(loaded_item, self.item_class)
|
||||
assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "bar"]}
|
||||
|
||||
def test_add_value_singlevalue_singlevalue(self):
|
||||
"""Values added after initialization should be appended"""
|
||||
|
|
@ -115,8 +114,8 @@ class InitializationTestMixin:
|
|||
il = ItemLoader(item=input_item)
|
||||
il.add_value("name", "bar")
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]})
|
||||
assert isinstance(loaded_item, self.item_class)
|
||||
assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "bar"]}
|
||||
|
||||
def test_add_value_singlevalue_list(self):
|
||||
"""Values added after initialization should be appended"""
|
||||
|
|
@ -124,10 +123,8 @@ class InitializationTestMixin:
|
|||
il = ItemLoader(item=input_item)
|
||||
il.add_value("name", ["item", "loader"])
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(
|
||||
ItemAdapter(loaded_item).asdict(), {"name": ["foo", "item", "loader"]}
|
||||
)
|
||||
assert isinstance(loaded_item, self.item_class)
|
||||
assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "item", "loader"]}
|
||||
|
||||
def test_add_value_list_singlevalue(self):
|
||||
"""Values added after initialization should be appended"""
|
||||
|
|
@ -135,10 +132,8 @@ class InitializationTestMixin:
|
|||
il = ItemLoader(item=input_item)
|
||||
il.add_value("name", "qwerty")
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(
|
||||
ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar", "qwerty"]}
|
||||
)
|
||||
assert isinstance(loaded_item, self.item_class)
|
||||
assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "bar", "qwerty"]}
|
||||
|
||||
def test_add_value_list_list(self):
|
||||
"""Values added after initialization should be appended"""
|
||||
|
|
@ -146,56 +141,55 @@ class InitializationTestMixin:
|
|||
il = ItemLoader(item=input_item)
|
||||
il.add_value("name", ["item", "loader"])
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(
|
||||
ItemAdapter(loaded_item).asdict(),
|
||||
{"name": ["foo", "bar", "item", "loader"]},
|
||||
)
|
||||
assert isinstance(loaded_item, self.item_class)
|
||||
assert ItemAdapter(loaded_item).asdict() == {
|
||||
"name": ["foo", "bar", "item", "loader"]
|
||||
}
|
||||
|
||||
def test_get_output_value_singlevalue(self):
|
||||
"""Getting output value must not remove value from item"""
|
||||
input_item = self.item_class(name="foo")
|
||||
il = ItemLoader(item=input_item)
|
||||
self.assertEqual(il.get_output_value("name"), ["foo"])
|
||||
assert il.get_output_value("name") == ["foo"]
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo"]})
|
||||
assert isinstance(loaded_item, self.item_class)
|
||||
assert ItemAdapter(loaded_item).asdict() == {"name": ["foo"]}
|
||||
|
||||
def test_get_output_value_list(self):
|
||||
"""Getting output value must not remove value from item"""
|
||||
input_item = self.item_class(name=["foo", "bar"])
|
||||
il = ItemLoader(item=input_item)
|
||||
self.assertEqual(il.get_output_value("name"), ["foo", "bar"])
|
||||
assert il.get_output_value("name") == ["foo", "bar"]
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]})
|
||||
assert isinstance(loaded_item, self.item_class)
|
||||
assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "bar"]}
|
||||
|
||||
def test_values_single(self):
|
||||
"""Values from initial item must be added to loader._values"""
|
||||
input_item = self.item_class(name="foo")
|
||||
il = ItemLoader(item=input_item)
|
||||
self.assertEqual(il._values.get("name"), ["foo"])
|
||||
assert il._values.get("name") == ["foo"]
|
||||
|
||||
def test_values_list(self):
|
||||
"""Values from initial item must be added to loader._values"""
|
||||
input_item = self.item_class(name=["foo", "bar"])
|
||||
il = ItemLoader(item=input_item)
|
||||
self.assertEqual(il._values.get("name"), ["foo", "bar"])
|
||||
assert il._values.get("name") == ["foo", "bar"]
|
||||
|
||||
|
||||
class InitializationFromDictTest(InitializationTestMixin, unittest.TestCase):
|
||||
class TestInitializationFromDict(InitializationTestMixin):
|
||||
item_class = dict
|
||||
|
||||
|
||||
class InitializationFromItemTest(InitializationTestMixin, unittest.TestCase):
|
||||
class TestInitializationFromItem(InitializationTestMixin):
|
||||
item_class = NameItem
|
||||
|
||||
|
||||
class InitializationFromAttrsItemTest(InitializationTestMixin, unittest.TestCase):
|
||||
class TestInitializationFromAttrsItem(InitializationTestMixin):
|
||||
item_class = AttrsNameItem
|
||||
|
||||
|
||||
class InitializationFromDataClassTest(InitializationTestMixin, unittest.TestCase):
|
||||
class TestInitializationFromDataClass(InitializationTestMixin):
|
||||
item_class = NameDataClass
|
||||
|
||||
|
||||
|
|
@ -212,7 +206,7 @@ class NoInputReprocessingItemLoader(BaseNoInputReprocessingLoader):
|
|||
default_item_class = NoInputReprocessingItem
|
||||
|
||||
|
||||
class NoInputReprocessingFromItemTest(unittest.TestCase):
|
||||
class TestNoInputReprocessingFromItem:
|
||||
"""
|
||||
Loaders initialized from loaded items must not reprocess fields (Item instances)
|
||||
"""
|
||||
|
|
@ -220,41 +214,41 @@ class NoInputReprocessingFromItemTest(unittest.TestCase):
|
|||
def test_avoid_reprocessing_with_initial_values_single(self):
|
||||
il = NoInputReprocessingItemLoader(item=NoInputReprocessingItem(title="foo"))
|
||||
il_loaded = il.load_item()
|
||||
self.assertEqual(il_loaded, {"title": "foo"})
|
||||
self.assertEqual(
|
||||
NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "foo"}
|
||||
)
|
||||
assert il_loaded == {"title": "foo"}
|
||||
assert NoInputReprocessingItemLoader(item=il_loaded).load_item() == {
|
||||
"title": "foo"
|
||||
}
|
||||
|
||||
def test_avoid_reprocessing_with_initial_values_list(self):
|
||||
il = NoInputReprocessingItemLoader(
|
||||
item=NoInputReprocessingItem(title=["foo", "bar"])
|
||||
)
|
||||
il_loaded = il.load_item()
|
||||
self.assertEqual(il_loaded, {"title": "foo"})
|
||||
self.assertEqual(
|
||||
NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "foo"}
|
||||
)
|
||||
assert il_loaded == {"title": "foo"}
|
||||
assert NoInputReprocessingItemLoader(item=il_loaded).load_item() == {
|
||||
"title": "foo"
|
||||
}
|
||||
|
||||
def test_avoid_reprocessing_without_initial_values_single(self):
|
||||
il = NoInputReprocessingItemLoader()
|
||||
il.add_value("title", "FOO")
|
||||
il_loaded = il.load_item()
|
||||
self.assertEqual(il_loaded, {"title": "FOO"})
|
||||
self.assertEqual(
|
||||
NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "FOO"}
|
||||
)
|
||||
assert il_loaded == {"title": "FOO"}
|
||||
assert NoInputReprocessingItemLoader(item=il_loaded).load_item() == {
|
||||
"title": "FOO"
|
||||
}
|
||||
|
||||
def test_avoid_reprocessing_without_initial_values_list(self):
|
||||
il = NoInputReprocessingItemLoader()
|
||||
il.add_value("title", ["foo", "bar"])
|
||||
il_loaded = il.load_item()
|
||||
self.assertEqual(il_loaded, {"title": "FOO"})
|
||||
self.assertEqual(
|
||||
NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "FOO"}
|
||||
)
|
||||
assert il_loaded == {"title": "FOO"}
|
||||
assert NoInputReprocessingItemLoader(item=il_loaded).load_item() == {
|
||||
"title": "FOO"
|
||||
}
|
||||
|
||||
|
||||
class TestOutputProcessorItem(unittest.TestCase):
|
||||
class TestOutputProcessorItem:
|
||||
def test_output_processor(self):
|
||||
class TempItem(Item):
|
||||
temp = Field()
|
||||
|
|
@ -270,11 +264,11 @@ class TestOutputProcessorItem(unittest.TestCase):
|
|||
|
||||
loader = TempLoader()
|
||||
item = loader.load_item()
|
||||
self.assertIsInstance(item, TempItem)
|
||||
self.assertEqual(dict(item), {"temp": 0.3})
|
||||
assert isinstance(item, TempItem)
|
||||
assert dict(item) == {"temp": 0.3}
|
||||
|
||||
|
||||
class SelectortemLoaderTest(unittest.TestCase):
|
||||
class TestSelectortemLoader:
|
||||
response = HtmlResponse(
|
||||
url="",
|
||||
encoding="utf-8",
|
||||
|
|
@ -292,7 +286,7 @@ class SelectortemLoaderTest(unittest.TestCase):
|
|||
|
||||
def test_init_method(self):
|
||||
l = ProcessorItemLoader()
|
||||
self.assertEqual(l.selector, None)
|
||||
assert l.selector is None
|
||||
|
||||
def test_init_method_errors(self):
|
||||
l = ProcessorItemLoader()
|
||||
|
|
@ -312,150 +306,149 @@ class SelectortemLoaderTest(unittest.TestCase):
|
|||
def test_init_method_with_selector(self):
|
||||
sel = Selector(text="<html><body><div>marta</div></body></html>")
|
||||
l = ProcessorItemLoader(selector=sel)
|
||||
self.assertIs(l.selector, sel)
|
||||
assert l.selector is sel
|
||||
|
||||
l.add_xpath("name", "//div/text()")
|
||||
self.assertEqual(l.get_output_value("name"), ["Marta"])
|
||||
assert l.get_output_value("name") == ["Marta"]
|
||||
|
||||
def test_init_method_with_selector_css(self):
|
||||
sel = Selector(text="<html><body><div>marta</div></body></html>")
|
||||
l = ProcessorItemLoader(selector=sel)
|
||||
self.assertIs(l.selector, sel)
|
||||
assert l.selector is sel
|
||||
|
||||
l.add_css("name", "div::text")
|
||||
self.assertEqual(l.get_output_value("name"), ["Marta"])
|
||||
assert l.get_output_value("name") == ["Marta"]
|
||||
|
||||
def test_init_method_with_base_response(self):
|
||||
"""Selector should be None after initialization"""
|
||||
response = Response("https://scrapy.org")
|
||||
l = ProcessorItemLoader(response=response)
|
||||
self.assertIs(l.selector, None)
|
||||
assert l.selector is None
|
||||
|
||||
def test_init_method_with_response(self):
|
||||
l = ProcessorItemLoader(response=self.response)
|
||||
self.assertTrue(l.selector)
|
||||
assert l.selector
|
||||
|
||||
l.add_xpath("name", "//div/text()")
|
||||
self.assertEqual(l.get_output_value("name"), ["Marta"])
|
||||
assert l.get_output_value("name") == ["Marta"]
|
||||
|
||||
def test_init_method_with_response_css(self):
|
||||
l = ProcessorItemLoader(response=self.response)
|
||||
self.assertTrue(l.selector)
|
||||
assert l.selector
|
||||
|
||||
l.add_css("name", "div::text")
|
||||
self.assertEqual(l.get_output_value("name"), ["Marta"])
|
||||
assert l.get_output_value("name") == ["Marta"]
|
||||
|
||||
l.add_css("url", "a::attr(href)")
|
||||
self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"])
|
||||
assert l.get_output_value("url") == ["http://www.scrapy.org"]
|
||||
|
||||
# combining/accumulating CSS selectors and XPath expressions
|
||||
l.add_xpath("name", "//div/text()")
|
||||
self.assertEqual(l.get_output_value("name"), ["Marta", "Marta"])
|
||||
assert l.get_output_value("name") == ["Marta", "Marta"]
|
||||
|
||||
l.add_xpath("url", "//img/@src")
|
||||
self.assertEqual(
|
||||
l.get_output_value("url"), ["http://www.scrapy.org", "/images/logo.png"]
|
||||
)
|
||||
assert l.get_output_value("url") == [
|
||||
"http://www.scrapy.org",
|
||||
"/images/logo.png",
|
||||
]
|
||||
|
||||
def test_add_xpath_re(self):
|
||||
l = ProcessorItemLoader(response=self.response)
|
||||
l.add_xpath("name", "//div/text()", re="ma")
|
||||
self.assertEqual(l.get_output_value("name"), ["Ma"])
|
||||
assert l.get_output_value("name") == ["Ma"]
|
||||
|
||||
def test_replace_xpath(self):
|
||||
l = ProcessorItemLoader(response=self.response)
|
||||
self.assertTrue(l.selector)
|
||||
assert l.selector
|
||||
l.add_xpath("name", "//div/text()")
|
||||
self.assertEqual(l.get_output_value("name"), ["Marta"])
|
||||
assert l.get_output_value("name") == ["Marta"]
|
||||
l.replace_xpath("name", "//p/text()")
|
||||
self.assertEqual(l.get_output_value("name"), ["Paragraph"])
|
||||
assert l.get_output_value("name") == ["Paragraph"]
|
||||
|
||||
l.replace_xpath("name", ["//p/text()", "//div/text()"])
|
||||
self.assertEqual(l.get_output_value("name"), ["Paragraph", "Marta"])
|
||||
assert l.get_output_value("name") == ["Paragraph", "Marta"]
|
||||
|
||||
def test_get_xpath(self):
|
||||
l = ProcessorItemLoader(response=self.response)
|
||||
self.assertEqual(l.get_xpath("//p/text()"), ["paragraph"])
|
||||
self.assertEqual(l.get_xpath("//p/text()", TakeFirst()), "paragraph")
|
||||
self.assertEqual(l.get_xpath("//p/text()", TakeFirst(), re="pa"), "pa")
|
||||
assert l.get_xpath("//p/text()") == ["paragraph"]
|
||||
assert l.get_xpath("//p/text()", TakeFirst()) == "paragraph"
|
||||
assert l.get_xpath("//p/text()", TakeFirst(), re="pa") == "pa"
|
||||
|
||||
self.assertEqual(
|
||||
l.get_xpath(["//p/text()", "//div/text()"]), ["paragraph", "marta"]
|
||||
)
|
||||
assert l.get_xpath(["//p/text()", "//div/text()"]) == ["paragraph", "marta"]
|
||||
|
||||
def test_replace_xpath_multi_fields(self):
|
||||
l = ProcessorItemLoader(response=self.response)
|
||||
l.add_xpath(None, "//div/text()", TakeFirst(), lambda x: {"name": x})
|
||||
self.assertEqual(l.get_output_value("name"), ["Marta"])
|
||||
assert l.get_output_value("name") == ["Marta"]
|
||||
l.replace_xpath(None, "//p/text()", TakeFirst(), lambda x: {"name": x})
|
||||
self.assertEqual(l.get_output_value("name"), ["Paragraph"])
|
||||
assert l.get_output_value("name") == ["Paragraph"]
|
||||
|
||||
def test_replace_xpath_re(self):
|
||||
l = ProcessorItemLoader(response=self.response)
|
||||
self.assertTrue(l.selector)
|
||||
assert l.selector
|
||||
l.add_xpath("name", "//div/text()")
|
||||
self.assertEqual(l.get_output_value("name"), ["Marta"])
|
||||
assert l.get_output_value("name") == ["Marta"]
|
||||
l.replace_xpath("name", "//div/text()", re="ma")
|
||||
self.assertEqual(l.get_output_value("name"), ["Ma"])
|
||||
assert l.get_output_value("name") == ["Ma"]
|
||||
|
||||
def test_add_css_re(self):
|
||||
l = ProcessorItemLoader(response=self.response)
|
||||
l.add_css("name", "div::text", re="ma")
|
||||
self.assertEqual(l.get_output_value("name"), ["Ma"])
|
||||
assert l.get_output_value("name") == ["Ma"]
|
||||
|
||||
l.add_css("url", "a::attr(href)", re="http://(.+)")
|
||||
self.assertEqual(l.get_output_value("url"), ["www.scrapy.org"])
|
||||
assert l.get_output_value("url") == ["www.scrapy.org"]
|
||||
|
||||
def test_replace_css(self):
|
||||
l = ProcessorItemLoader(response=self.response)
|
||||
self.assertTrue(l.selector)
|
||||
assert l.selector
|
||||
l.add_css("name", "div::text")
|
||||
self.assertEqual(l.get_output_value("name"), ["Marta"])
|
||||
assert l.get_output_value("name") == ["Marta"]
|
||||
l.replace_css("name", "p::text")
|
||||
self.assertEqual(l.get_output_value("name"), ["Paragraph"])
|
||||
assert l.get_output_value("name") == ["Paragraph"]
|
||||
|
||||
l.replace_css("name", ["p::text", "div::text"])
|
||||
self.assertEqual(l.get_output_value("name"), ["Paragraph", "Marta"])
|
||||
assert l.get_output_value("name") == ["Paragraph", "Marta"]
|
||||
|
||||
l.add_css("url", "a::attr(href)", re="http://(.+)")
|
||||
self.assertEqual(l.get_output_value("url"), ["www.scrapy.org"])
|
||||
assert l.get_output_value("url") == ["www.scrapy.org"]
|
||||
l.replace_css("url", "img::attr(src)")
|
||||
self.assertEqual(l.get_output_value("url"), ["/images/logo.png"])
|
||||
assert l.get_output_value("url") == ["/images/logo.png"]
|
||||
|
||||
def test_get_css(self):
|
||||
l = ProcessorItemLoader(response=self.response)
|
||||
self.assertEqual(l.get_css("p::text"), ["paragraph"])
|
||||
self.assertEqual(l.get_css("p::text", TakeFirst()), "paragraph")
|
||||
self.assertEqual(l.get_css("p::text", TakeFirst(), re="pa"), "pa")
|
||||
assert l.get_css("p::text") == ["paragraph"]
|
||||
assert l.get_css("p::text", TakeFirst()) == "paragraph"
|
||||
assert l.get_css("p::text", TakeFirst(), re="pa") == "pa"
|
||||
|
||||
self.assertEqual(l.get_css(["p::text", "div::text"]), ["paragraph", "marta"])
|
||||
self.assertEqual(
|
||||
l.get_css(["a::attr(href)", "img::attr(src)"]),
|
||||
["http://www.scrapy.org", "/images/logo.png"],
|
||||
)
|
||||
assert l.get_css(["p::text", "div::text"]) == ["paragraph", "marta"]
|
||||
assert l.get_css(["a::attr(href)", "img::attr(src)"]) == [
|
||||
"http://www.scrapy.org",
|
||||
"/images/logo.png",
|
||||
]
|
||||
|
||||
def test_replace_css_multi_fields(self):
|
||||
l = ProcessorItemLoader(response=self.response)
|
||||
l.add_css(None, "div::text", TakeFirst(), lambda x: {"name": x})
|
||||
self.assertEqual(l.get_output_value("name"), ["Marta"])
|
||||
assert l.get_output_value("name") == ["Marta"]
|
||||
l.replace_css(None, "p::text", TakeFirst(), lambda x: {"name": x})
|
||||
self.assertEqual(l.get_output_value("name"), ["Paragraph"])
|
||||
assert l.get_output_value("name") == ["Paragraph"]
|
||||
|
||||
l.add_css(None, "a::attr(href)", TakeFirst(), lambda x: {"url": x})
|
||||
self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"])
|
||||
assert l.get_output_value("url") == ["http://www.scrapy.org"]
|
||||
l.replace_css(None, "img::attr(src)", TakeFirst(), lambda x: {"url": x})
|
||||
self.assertEqual(l.get_output_value("url"), ["/images/logo.png"])
|
||||
assert l.get_output_value("url") == ["/images/logo.png"]
|
||||
|
||||
def test_replace_css_re(self):
|
||||
l = ProcessorItemLoader(response=self.response)
|
||||
self.assertTrue(l.selector)
|
||||
assert l.selector
|
||||
l.add_css("url", "a::attr(href)")
|
||||
self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"])
|
||||
assert l.get_output_value("url") == ["http://www.scrapy.org"]
|
||||
l.replace_css("url", "a::attr(href)", re=r"http://www\.(.+)")
|
||||
self.assertEqual(l.get_output_value("url"), ["scrapy.org"])
|
||||
assert l.get_output_value("url") == ["scrapy.org"]
|
||||
|
||||
|
||||
class SubselectorLoaderTest(unittest.TestCase):
|
||||
class TestSubselectorLoader:
|
||||
response = HtmlResponse(
|
||||
url="",
|
||||
encoding="utf-8",
|
||||
|
|
@ -483,17 +476,13 @@ class SubselectorLoaderTest(unittest.TestCase):
|
|||
nl.add_css("name_div", "#id")
|
||||
nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall())
|
||||
|
||||
self.assertEqual(l.get_output_value("name"), ["marta"])
|
||||
self.assertEqual(l.get_output_value("name_div"), ['<div id="id">marta</div>'])
|
||||
self.assertEqual(l.get_output_value("name_value"), ["marta"])
|
||||
assert l.get_output_value("name") == ["marta"]
|
||||
assert l.get_output_value("name_div") == ['<div id="id">marta</div>']
|
||||
assert l.get_output_value("name_value") == ["marta"]
|
||||
|
||||
self.assertEqual(l.get_output_value("name"), nl.get_output_value("name"))
|
||||
self.assertEqual(
|
||||
l.get_output_value("name_div"), nl.get_output_value("name_div")
|
||||
)
|
||||
self.assertEqual(
|
||||
l.get_output_value("name_value"), nl.get_output_value("name_value")
|
||||
)
|
||||
assert l.get_output_value("name") == nl.get_output_value("name")
|
||||
assert l.get_output_value("name_div") == nl.get_output_value("name_div")
|
||||
assert l.get_output_value("name_value") == nl.get_output_value("name_value")
|
||||
|
||||
def test_nested_css(self):
|
||||
l = NestedItemLoader(response=self.response)
|
||||
|
|
@ -502,17 +491,13 @@ class SubselectorLoaderTest(unittest.TestCase):
|
|||
nl.add_css("name_div", "#id")
|
||||
nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall())
|
||||
|
||||
self.assertEqual(l.get_output_value("name"), ["marta"])
|
||||
self.assertEqual(l.get_output_value("name_div"), ['<div id="id">marta</div>'])
|
||||
self.assertEqual(l.get_output_value("name_value"), ["marta"])
|
||||
assert l.get_output_value("name") == ["marta"]
|
||||
assert l.get_output_value("name_div") == ['<div id="id">marta</div>']
|
||||
assert l.get_output_value("name_value") == ["marta"]
|
||||
|
||||
self.assertEqual(l.get_output_value("name"), nl.get_output_value("name"))
|
||||
self.assertEqual(
|
||||
l.get_output_value("name_div"), nl.get_output_value("name_div")
|
||||
)
|
||||
self.assertEqual(
|
||||
l.get_output_value("name_value"), nl.get_output_value("name_value")
|
||||
)
|
||||
assert l.get_output_value("name") == nl.get_output_value("name")
|
||||
assert l.get_output_value("name_div") == nl.get_output_value("name_div")
|
||||
assert l.get_output_value("name_value") == nl.get_output_value("name_value")
|
||||
|
||||
def test_nested_replace(self):
|
||||
l = NestedItemLoader(response=self.response)
|
||||
|
|
@ -520,11 +505,11 @@ class SubselectorLoaderTest(unittest.TestCase):
|
|||
nl2 = nl1.nested_xpath("a")
|
||||
|
||||
l.add_xpath("url", "//footer/a/@href")
|
||||
self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"])
|
||||
assert l.get_output_value("url") == ["http://www.scrapy.org"]
|
||||
nl1.replace_xpath("url", "img/@src")
|
||||
self.assertEqual(l.get_output_value("url"), ["/images/logo.png"])
|
||||
assert l.get_output_value("url") == ["/images/logo.png"]
|
||||
nl2.replace_xpath("url", "@href")
|
||||
self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"])
|
||||
assert l.get_output_value("url") == ["http://www.scrapy.org"]
|
||||
|
||||
def test_nested_ordering(self):
|
||||
l = NestedItemLoader(response=self.response)
|
||||
|
|
@ -536,15 +521,12 @@ class SubselectorLoaderTest(unittest.TestCase):
|
|||
nl2.add_xpath("url", "text()")
|
||||
l.add_xpath("url", "//footer/a/@href")
|
||||
|
||||
self.assertEqual(
|
||||
l.get_output_value("url"),
|
||||
[
|
||||
"/images/logo.png",
|
||||
"http://www.scrapy.org",
|
||||
"homepage",
|
||||
"http://www.scrapy.org",
|
||||
],
|
||||
)
|
||||
assert l.get_output_value("url") == [
|
||||
"/images/logo.png",
|
||||
"http://www.scrapy.org",
|
||||
"homepage",
|
||||
"http://www.scrapy.org",
|
||||
]
|
||||
|
||||
def test_nested_load_item(self):
|
||||
l = NestedItemLoader(response=self.response)
|
||||
|
|
@ -561,9 +543,9 @@ class SubselectorLoaderTest(unittest.TestCase):
|
|||
assert item is nl1.item
|
||||
assert item is nl2.item
|
||||
|
||||
self.assertEqual(item["name"], ["marta"])
|
||||
self.assertEqual(item["url"], ["http://www.scrapy.org"])
|
||||
self.assertEqual(item["image"], ["/images/logo.png"])
|
||||
assert item["name"] == ["marta"]
|
||||
assert item["url"] == ["http://www.scrapy.org"]
|
||||
assert item["image"] == ["/images/logo.png"]
|
||||
|
||||
|
||||
# Functions as processors
|
||||
|
|
@ -588,9 +570,9 @@ class FunctionProcessorItemLoader(ItemLoader):
|
|||
default_item_class = FunctionProcessorItem
|
||||
|
||||
|
||||
class FunctionProcessorTestCase(unittest.TestCase):
|
||||
class TestFunctionProcessor:
|
||||
def test_processor_defined_in_item(self):
|
||||
lo = FunctionProcessorItemLoader()
|
||||
lo.add_value("foo", " bar ")
|
||||
lo.add_value("foo", [" asdf ", " qwerty "])
|
||||
self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]})
|
||||
assert dict(lo.load_item()) == {"foo": ["BAR", "ASDF", "QWERTY"]}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
import unittest
|
||||
# pylint: disable=unsubscriptable-object,unsupported-membership-test,use-implicit-booleaness-not-comparison
|
||||
# (too many false positives)
|
||||
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
|
@ -14,31 +16,31 @@ from scrapy.settings import (
|
|||
from . import default_settings
|
||||
|
||||
|
||||
class SettingsGlobalFuncsTest(unittest.TestCase):
|
||||
class TestSettingsGlobalFuncs:
|
||||
def test_get_settings_priority(self):
|
||||
for prio_str, prio_num in SETTINGS_PRIORITIES.items():
|
||||
self.assertEqual(get_settings_priority(prio_str), prio_num)
|
||||
self.assertEqual(get_settings_priority(99), 99)
|
||||
assert get_settings_priority(prio_str) == prio_num
|
||||
assert get_settings_priority(99) == 99
|
||||
|
||||
|
||||
class SettingsAttributeTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
class TestSettingsAttribute:
|
||||
def setup_method(self):
|
||||
self.attribute = SettingsAttribute("value", 10)
|
||||
|
||||
def test_set_greater_priority(self):
|
||||
self.attribute.set("value2", 20)
|
||||
self.assertEqual(self.attribute.value, "value2")
|
||||
self.assertEqual(self.attribute.priority, 20)
|
||||
assert self.attribute.value == "value2"
|
||||
assert self.attribute.priority == 20
|
||||
|
||||
def test_set_equal_priority(self):
|
||||
self.attribute.set("value2", 10)
|
||||
self.assertEqual(self.attribute.value, "value2")
|
||||
self.assertEqual(self.attribute.priority, 10)
|
||||
assert self.attribute.value == "value2"
|
||||
assert self.attribute.priority == 10
|
||||
|
||||
def test_set_less_priority(self):
|
||||
self.attribute.set("value2", 0)
|
||||
self.assertEqual(self.attribute.value, "value")
|
||||
self.assertEqual(self.attribute.priority, 10)
|
||||
assert self.attribute.value == "value"
|
||||
assert self.attribute.priority == 10
|
||||
|
||||
def test_overwrite_basesettings(self):
|
||||
original_dict = {"one": 10, "two": 20}
|
||||
|
|
@ -47,61 +49,59 @@ class SettingsAttributeTest(unittest.TestCase):
|
|||
|
||||
new_dict = {"three": 11, "four": 21}
|
||||
attribute.set(new_dict, 10)
|
||||
self.assertIsInstance(attribute.value, BaseSettings)
|
||||
self.assertCountEqual(attribute.value, new_dict)
|
||||
self.assertCountEqual(original_settings, original_dict)
|
||||
assert isinstance(attribute.value, BaseSettings)
|
||||
assert set(attribute.value) == set(new_dict)
|
||||
assert set(original_settings) == set(original_dict)
|
||||
|
||||
new_settings = BaseSettings({"five": 12}, 0)
|
||||
attribute.set(new_settings, 0) # Insufficient priority
|
||||
self.assertCountEqual(attribute.value, new_dict)
|
||||
assert set(attribute.value) == set(new_dict)
|
||||
attribute.set(new_settings, 10)
|
||||
self.assertCountEqual(attribute.value, new_settings)
|
||||
assert set(attribute.value) == set(new_settings)
|
||||
|
||||
def test_repr(self):
|
||||
self.assertEqual(
|
||||
repr(self.attribute), "<SettingsAttribute value='value' priority=10>"
|
||||
)
|
||||
assert repr(self.attribute) == "<SettingsAttribute value='value' priority=10>"
|
||||
|
||||
|
||||
class BaseSettingsTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
class TestBaseSettings:
|
||||
def setup_method(self):
|
||||
self.settings = BaseSettings()
|
||||
|
||||
def test_setdefault_not_existing_value(self):
|
||||
settings = BaseSettings()
|
||||
value = settings.setdefault("TEST_OPTION", "value")
|
||||
self.assertEqual(settings["TEST_OPTION"], "value")
|
||||
self.assertEqual(value, "value")
|
||||
self.assertIsNotNone(value)
|
||||
assert settings["TEST_OPTION"] == "value"
|
||||
assert value == "value"
|
||||
assert value is not None
|
||||
|
||||
def test_setdefault_existing_value(self):
|
||||
settings = BaseSettings({"TEST_OPTION": "value"})
|
||||
value = settings.setdefault("TEST_OPTION", None)
|
||||
self.assertEqual(settings["TEST_OPTION"], "value")
|
||||
self.assertEqual(value, "value")
|
||||
assert settings["TEST_OPTION"] == "value"
|
||||
assert value == "value"
|
||||
|
||||
def test_set_new_attribute(self):
|
||||
self.settings.set("TEST_OPTION", "value", 0)
|
||||
self.assertIn("TEST_OPTION", self.settings.attributes)
|
||||
assert "TEST_OPTION" in self.settings.attributes
|
||||
|
||||
attr = self.settings.attributes["TEST_OPTION"]
|
||||
self.assertIsInstance(attr, SettingsAttribute)
|
||||
self.assertEqual(attr.value, "value")
|
||||
self.assertEqual(attr.priority, 0)
|
||||
assert isinstance(attr, SettingsAttribute)
|
||||
assert attr.value == "value"
|
||||
assert attr.priority == 0
|
||||
|
||||
def test_set_settingsattribute(self):
|
||||
myattr = SettingsAttribute(0, 30) # Note priority 30
|
||||
self.settings.set("TEST_ATTR", myattr, 10)
|
||||
self.assertEqual(self.settings.get("TEST_ATTR"), 0)
|
||||
self.assertEqual(self.settings.getpriority("TEST_ATTR"), 30)
|
||||
assert self.settings.get("TEST_ATTR") == 0
|
||||
assert self.settings.getpriority("TEST_ATTR") == 30
|
||||
|
||||
def test_set_instance_identity_on_update(self):
|
||||
attr = SettingsAttribute("value", 0)
|
||||
self.settings.attributes = {"TEST_OPTION": attr}
|
||||
self.settings.set("TEST_OPTION", "othervalue", 10)
|
||||
|
||||
self.assertIn("TEST_OPTION", self.settings.attributes)
|
||||
self.assertIs(attr, self.settings.attributes["TEST_OPTION"])
|
||||
assert "TEST_OPTION" in self.settings.attributes
|
||||
assert attr is self.settings.attributes["TEST_OPTION"]
|
||||
|
||||
def test_set_calls_settings_attributes_methods_on_update(self):
|
||||
attr = SettingsAttribute("value", 10)
|
||||
|
|
@ -114,7 +114,7 @@ class BaseSettingsTest(unittest.TestCase):
|
|||
for priority in (0, 10, 20):
|
||||
self.settings.set("TEST_OPTION", "othervalue", priority)
|
||||
mock_set.assert_called_once_with("othervalue", priority)
|
||||
self.assertFalse(mock_setattr.called)
|
||||
assert not mock_setattr.called
|
||||
mock_set.reset_mock()
|
||||
mock_setattr.reset_mock()
|
||||
|
||||
|
|
@ -122,19 +122,19 @@ class BaseSettingsTest(unittest.TestCase):
|
|||
settings = BaseSettings()
|
||||
settings.set("key", "a", "default")
|
||||
settings["key"] = "b"
|
||||
self.assertEqual(settings["key"], "b")
|
||||
self.assertEqual(settings.getpriority("key"), 20)
|
||||
assert settings["key"] == "b"
|
||||
assert settings.getpriority("key") == 20
|
||||
settings["key"] = "c"
|
||||
self.assertEqual(settings["key"], "c")
|
||||
assert settings["key"] == "c"
|
||||
settings["key2"] = "x"
|
||||
self.assertIn("key2", settings)
|
||||
self.assertEqual(settings["key2"], "x")
|
||||
self.assertEqual(settings.getpriority("key2"), 20)
|
||||
assert "key2" in settings
|
||||
assert settings["key2"] == "x"
|
||||
assert settings.getpriority("key2") == 20
|
||||
|
||||
def test_setdict_alias(self):
|
||||
with mock.patch.object(self.settings, "set") as mock_set:
|
||||
self.settings.setdict({"TEST_1": "value1", "TEST_2": "value2"}, 10)
|
||||
self.assertEqual(mock_set.call_count, 2)
|
||||
assert mock_set.call_count == 2
|
||||
calls = [
|
||||
mock.call("TEST_1", "value1", 10),
|
||||
mock.call("TEST_2", "value2", 10),
|
||||
|
|
@ -149,10 +149,10 @@ class BaseSettingsTest(unittest.TestCase):
|
|||
|
||||
self.settings.attributes = {}
|
||||
self.settings.setmodule(ModuleMock(), 10)
|
||||
self.assertIn("UPPERCASE_VAR", self.settings.attributes)
|
||||
self.assertNotIn("MIXEDcase_VAR", self.settings.attributes)
|
||||
self.assertNotIn("lowercase_var", self.settings.attributes)
|
||||
self.assertEqual(len(self.settings.attributes), 1)
|
||||
assert "UPPERCASE_VAR" in self.settings.attributes
|
||||
assert "MIXEDcase_VAR" not in self.settings.attributes
|
||||
assert "lowercase_var" not in self.settings.attributes
|
||||
assert len(self.settings.attributes) == 1
|
||||
|
||||
def test_setmodule_alias(self):
|
||||
with mock.patch.object(self.settings, "set") as mock_set:
|
||||
|
|
@ -168,13 +168,13 @@ class BaseSettingsTest(unittest.TestCase):
|
|||
self.settings.attributes = {}
|
||||
self.settings.setmodule("tests.test_settings.default_settings", 10)
|
||||
|
||||
self.assertCountEqual(self.settings.attributes.keys(), ctrl_attributes.keys())
|
||||
assert set(self.settings.attributes) == set(ctrl_attributes)
|
||||
|
||||
for key in ctrl_attributes:
|
||||
attr = self.settings.attributes[key]
|
||||
ctrl_attr = ctrl_attributes[key]
|
||||
self.assertEqual(attr.value, ctrl_attr.value)
|
||||
self.assertEqual(attr.priority, ctrl_attr.priority)
|
||||
assert attr.value == ctrl_attr.value
|
||||
assert attr.priority == ctrl_attr.priority
|
||||
|
||||
def test_update(self):
|
||||
settings = BaseSettings({"key_lowprio": 0}, priority=0)
|
||||
|
|
@ -186,21 +186,21 @@ class BaseSettingsTest(unittest.TestCase):
|
|||
custom_dict = {"key_lowprio": 2, "key_highprio": 12, "newkey_two": None}
|
||||
|
||||
settings.update(custom_dict, priority=20)
|
||||
self.assertEqual(settings["key_lowprio"], 2)
|
||||
self.assertEqual(settings.getpriority("key_lowprio"), 20)
|
||||
self.assertEqual(settings["key_highprio"], 10)
|
||||
self.assertIn("newkey_two", settings)
|
||||
self.assertEqual(settings.getpriority("newkey_two"), 20)
|
||||
assert settings["key_lowprio"] == 2
|
||||
assert settings.getpriority("key_lowprio") == 20
|
||||
assert settings["key_highprio"] == 10
|
||||
assert "newkey_two" in settings
|
||||
assert settings.getpriority("newkey_two") == 20
|
||||
|
||||
settings.update(custom_settings)
|
||||
self.assertEqual(settings["key_lowprio"], 1)
|
||||
self.assertEqual(settings.getpriority("key_lowprio"), 30)
|
||||
self.assertEqual(settings["key_highprio"], 10)
|
||||
self.assertIn("newkey_one", settings)
|
||||
self.assertEqual(settings.getpriority("newkey_one"), 50)
|
||||
assert settings["key_lowprio"] == 1
|
||||
assert settings.getpriority("key_lowprio") == 30
|
||||
assert settings["key_highprio"] == 10
|
||||
assert "newkey_one" in settings
|
||||
assert settings.getpriority("newkey_one") == 50
|
||||
|
||||
settings.update({"key_lowprio": 3}, priority=20)
|
||||
self.assertEqual(settings["key_lowprio"], 1)
|
||||
assert settings["key_lowprio"] == 1
|
||||
|
||||
@pytest.mark.xfail(
|
||||
raises=TypeError, reason="BaseSettings.update doesn't support kwargs input"
|
||||
|
|
@ -220,21 +220,21 @@ class BaseSettingsTest(unittest.TestCase):
|
|||
def test_update_jsonstring(self):
|
||||
settings = BaseSettings({"number": 0, "dict": BaseSettings({"key": "val"})})
|
||||
settings.update('{"number": 1, "newnumber": 2}')
|
||||
self.assertEqual(settings["number"], 1)
|
||||
self.assertEqual(settings["newnumber"], 2)
|
||||
assert settings["number"] == 1
|
||||
assert settings["newnumber"] == 2
|
||||
settings.set("dict", '{"key": "newval", "newkey": "newval2"}')
|
||||
self.assertEqual(settings["dict"]["key"], "newval")
|
||||
self.assertEqual(settings["dict"]["newkey"], "newval2")
|
||||
assert settings["dict"]["key"] == "newval"
|
||||
assert settings["dict"]["newkey"] == "newval2"
|
||||
|
||||
def test_delete(self):
|
||||
settings = BaseSettings({"key": None})
|
||||
settings.set("key_highprio", None, priority=50)
|
||||
settings.delete("key")
|
||||
settings.delete("key_highprio")
|
||||
self.assertNotIn("key", settings)
|
||||
self.assertIn("key_highprio", settings)
|
||||
assert "key" not in settings
|
||||
assert "key_highprio" in settings
|
||||
del settings["key_highprio"]
|
||||
self.assertNotIn("key_highprio", settings)
|
||||
assert "key_highprio" not in settings
|
||||
with pytest.raises(KeyError):
|
||||
settings.delete("notkey")
|
||||
with pytest.raises(KeyError):
|
||||
|
|
@ -271,40 +271,40 @@ class BaseSettingsTest(unittest.TestCase):
|
|||
for key, value in test_configuration.items()
|
||||
}
|
||||
|
||||
self.assertTrue(settings.getbool("TEST_ENABLED1"))
|
||||
self.assertTrue(settings.getbool("TEST_ENABLED2"))
|
||||
self.assertTrue(settings.getbool("TEST_ENABLED3"))
|
||||
self.assertTrue(settings.getbool("TEST_ENABLED4"))
|
||||
self.assertTrue(settings.getbool("TEST_ENABLED5"))
|
||||
self.assertFalse(settings.getbool("TEST_ENABLEDx"))
|
||||
self.assertTrue(settings.getbool("TEST_ENABLEDx", True))
|
||||
self.assertFalse(settings.getbool("TEST_DISABLED1"))
|
||||
self.assertFalse(settings.getbool("TEST_DISABLED2"))
|
||||
self.assertFalse(settings.getbool("TEST_DISABLED3"))
|
||||
self.assertFalse(settings.getbool("TEST_DISABLED4"))
|
||||
self.assertFalse(settings.getbool("TEST_DISABLED5"))
|
||||
self.assertEqual(settings.getint("TEST_INT1"), 123)
|
||||
self.assertEqual(settings.getint("TEST_INT2"), 123)
|
||||
self.assertEqual(settings.getint("TEST_INTx"), 0)
|
||||
self.assertEqual(settings.getint("TEST_INTx", 45), 45)
|
||||
self.assertEqual(settings.getfloat("TEST_FLOAT1"), 123.45)
|
||||
self.assertEqual(settings.getfloat("TEST_FLOAT2"), 123.45)
|
||||
self.assertEqual(settings.getfloat("TEST_FLOATx"), 0.0)
|
||||
self.assertEqual(settings.getfloat("TEST_FLOATx", 55.0), 55.0)
|
||||
self.assertEqual(settings.getlist("TEST_LIST1"), ["one", "two"])
|
||||
self.assertEqual(settings.getlist("TEST_LIST2"), ["one", "two"])
|
||||
self.assertEqual(settings.getlist("TEST_LIST3"), [])
|
||||
self.assertEqual(settings.getlist("TEST_LISTx"), [])
|
||||
self.assertEqual(settings.getlist("TEST_LISTx", ["default"]), ["default"])
|
||||
self.assertEqual(settings["TEST_STR"], "value")
|
||||
self.assertEqual(settings.get("TEST_STR"), "value")
|
||||
self.assertEqual(settings["TEST_STRx"], None)
|
||||
self.assertEqual(settings.get("TEST_STRx"), None)
|
||||
self.assertEqual(settings.get("TEST_STRx", "default"), "default")
|
||||
self.assertEqual(settings.getdict("TEST_DICT1"), {"key1": "val1", "ke2": 3})
|
||||
self.assertEqual(settings.getdict("TEST_DICT2"), {"key1": "val1", "ke2": 3})
|
||||
self.assertEqual(settings.getdict("TEST_DICT3"), {})
|
||||
self.assertEqual(settings.getdict("TEST_DICT3", {"key1": 5}), {"key1": 5})
|
||||
assert settings.getbool("TEST_ENABLED1")
|
||||
assert settings.getbool("TEST_ENABLED2")
|
||||
assert settings.getbool("TEST_ENABLED3")
|
||||
assert settings.getbool("TEST_ENABLED4")
|
||||
assert settings.getbool("TEST_ENABLED5")
|
||||
assert not settings.getbool("TEST_ENABLEDx")
|
||||
assert settings.getbool("TEST_ENABLEDx", True)
|
||||
assert not settings.getbool("TEST_DISABLED1")
|
||||
assert not settings.getbool("TEST_DISABLED2")
|
||||
assert not settings.getbool("TEST_DISABLED3")
|
||||
assert not settings.getbool("TEST_DISABLED4")
|
||||
assert not settings.getbool("TEST_DISABLED5")
|
||||
assert settings.getint("TEST_INT1") == 123
|
||||
assert settings.getint("TEST_INT2") == 123
|
||||
assert settings.getint("TEST_INTx") == 0
|
||||
assert settings.getint("TEST_INTx", 45) == 45
|
||||
assert settings.getfloat("TEST_FLOAT1") == 123.45
|
||||
assert settings.getfloat("TEST_FLOAT2") == 123.45
|
||||
assert settings.getfloat("TEST_FLOATx") == 0.0
|
||||
assert settings.getfloat("TEST_FLOATx", 55.0) == 55.0
|
||||
assert settings.getlist("TEST_LIST1") == ["one", "two"]
|
||||
assert settings.getlist("TEST_LIST2") == ["one", "two"]
|
||||
assert settings.getlist("TEST_LIST3") == []
|
||||
assert settings.getlist("TEST_LISTx") == []
|
||||
assert settings.getlist("TEST_LISTx", ["default"]) == ["default"]
|
||||
assert settings["TEST_STR"] == "value"
|
||||
assert settings.get("TEST_STR") == "value"
|
||||
assert settings["TEST_STRx"] is None
|
||||
assert settings.get("TEST_STRx") is None
|
||||
assert settings.get("TEST_STRx", "default") == "default"
|
||||
assert settings.getdict("TEST_DICT1") == {"key1": "val1", "ke2": 3}
|
||||
assert settings.getdict("TEST_DICT2") == {"key1": "val1", "ke2": 3}
|
||||
assert settings.getdict("TEST_DICT3") == {}
|
||||
assert settings.getdict("TEST_DICT3", {"key1": 5}) == {"key1": 5}
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="dictionary update sequence element #0 has length 3; 2 is required|sequence of pairs expected",
|
||||
|
|
@ -321,8 +321,8 @@ class BaseSettingsTest(unittest.TestCase):
|
|||
|
||||
def test_getpriority(self):
|
||||
settings = BaseSettings({"key": "value"}, priority=99)
|
||||
self.assertEqual(settings.getpriority("key"), 99)
|
||||
self.assertEqual(settings.getpriority("nonexistentkey"), None)
|
||||
assert settings.getpriority("key") == 99
|
||||
assert settings.getpriority("nonexistentkey") is None
|
||||
|
||||
def test_getwithbase(self):
|
||||
s = BaseSettings(
|
||||
|
|
@ -333,16 +333,16 @@ class BaseSettingsTest(unittest.TestCase):
|
|||
}
|
||||
)
|
||||
s["TEST"].set(2, 200, "cmdline")
|
||||
self.assertCountEqual(s.getwithbase("TEST"), {1: 1, 2: 200, 3: 30})
|
||||
self.assertCountEqual(s.getwithbase("HASNOBASE"), s["HASNOBASE"])
|
||||
self.assertEqual(s.getwithbase("NONEXISTENT"), {})
|
||||
assert set(s.getwithbase("TEST")) == {1, 2, 3}
|
||||
assert set(s.getwithbase("HASNOBASE")) == set(s["HASNOBASE"])
|
||||
assert s.getwithbase("NONEXISTENT") == {}
|
||||
|
||||
def test_maxpriority(self):
|
||||
# Empty settings should return 'default'
|
||||
self.assertEqual(self.settings.maxpriority(), 0)
|
||||
assert self.settings.maxpriority() == 0
|
||||
self.settings.set("A", 0, 10)
|
||||
self.settings.set("B", 0, 30)
|
||||
self.assertEqual(self.settings.maxpriority(), 30)
|
||||
assert self.settings.maxpriority() == 30
|
||||
|
||||
def test_copy(self):
|
||||
values = {
|
||||
|
|
@ -356,17 +356,15 @@ class BaseSettingsTest(unittest.TestCase):
|
|||
self.settings.setdict(values)
|
||||
copy = self.settings.copy()
|
||||
self.settings.set("TEST_BOOL", False)
|
||||
self.assertTrue(copy.get("TEST_BOOL"))
|
||||
assert copy.get("TEST_BOOL")
|
||||
|
||||
test_list = self.settings.get("TEST_LIST")
|
||||
test_list.append("three")
|
||||
self.assertListEqual(copy.get("TEST_LIST"), ["one", "two"])
|
||||
assert copy.get("TEST_LIST") == ["one", "two"]
|
||||
|
||||
test_list_of_lists = self.settings.get("TEST_LIST_OF_LISTS")
|
||||
test_list_of_lists[0].append("first_three")
|
||||
self.assertListEqual(
|
||||
copy.get("TEST_LIST_OF_LISTS")[0], ["first_one", "first_two"]
|
||||
)
|
||||
assert copy.get("TEST_LIST_OF_LISTS")[0] == ["first_one", "first_two"]
|
||||
|
||||
def test_copy_to_dict(self):
|
||||
s = BaseSettings(
|
||||
|
|
@ -379,17 +377,14 @@ class BaseSettingsTest(unittest.TestCase):
|
|||
"HASNOBASE": BaseSettings({3: 3000}, "default"),
|
||||
}
|
||||
)
|
||||
self.assertDictEqual(
|
||||
s.copy_to_dict(),
|
||||
{
|
||||
"HASNOBASE": {3: 3000},
|
||||
"TEST": {1: 10, 3: 30},
|
||||
"TEST_BASE": {1: 1, 2: 2},
|
||||
"TEST_LIST": [1, 2],
|
||||
"TEST_BOOLEAN": False,
|
||||
"TEST_STRING": "a string",
|
||||
},
|
||||
)
|
||||
assert s.copy_to_dict() == {
|
||||
"HASNOBASE": {3: 3000},
|
||||
"TEST": {1: 10, 3: 30},
|
||||
"TEST_BASE": {1: 1, 2: 2},
|
||||
"TEST_LIST": [1, 2],
|
||||
"TEST_BOOLEAN": False,
|
||||
"TEST_STRING": "a string",
|
||||
}
|
||||
|
||||
def test_freeze(self):
|
||||
self.settings.freeze()
|
||||
|
|
@ -400,55 +395,55 @@ class BaseSettingsTest(unittest.TestCase):
|
|||
|
||||
def test_frozencopy(self):
|
||||
frozencopy = self.settings.frozencopy()
|
||||
self.assertTrue(frozencopy.frozen)
|
||||
self.assertIsNot(frozencopy, self.settings)
|
||||
assert frozencopy.frozen
|
||||
assert frozencopy is not self.settings
|
||||
|
||||
|
||||
class SettingsTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
class TestSettings:
|
||||
def setup_method(self):
|
||||
self.settings = Settings()
|
||||
|
||||
@mock.patch.dict("scrapy.settings.SETTINGS_PRIORITIES", {"default": 10})
|
||||
@mock.patch("scrapy.settings.default_settings", default_settings)
|
||||
def test_initial_defaults(self):
|
||||
settings = Settings()
|
||||
self.assertEqual(len(settings.attributes), 2)
|
||||
self.assertIn("TEST_DEFAULT", settings.attributes)
|
||||
assert len(settings.attributes) == 2
|
||||
assert "TEST_DEFAULT" in settings.attributes
|
||||
|
||||
attr = settings.attributes["TEST_DEFAULT"]
|
||||
self.assertIsInstance(attr, SettingsAttribute)
|
||||
self.assertEqual(attr.value, "defvalue")
|
||||
self.assertEqual(attr.priority, 10)
|
||||
assert isinstance(attr, SettingsAttribute)
|
||||
assert attr.value == "defvalue"
|
||||
assert attr.priority == 10
|
||||
|
||||
@mock.patch.dict("scrapy.settings.SETTINGS_PRIORITIES", {})
|
||||
@mock.patch("scrapy.settings.default_settings", {})
|
||||
def test_initial_values(self):
|
||||
settings = Settings({"TEST_OPTION": "value"}, 10)
|
||||
self.assertEqual(len(settings.attributes), 1)
|
||||
self.assertIn("TEST_OPTION", settings.attributes)
|
||||
assert len(settings.attributes) == 1
|
||||
assert "TEST_OPTION" in settings.attributes
|
||||
|
||||
attr = settings.attributes["TEST_OPTION"]
|
||||
self.assertIsInstance(attr, SettingsAttribute)
|
||||
self.assertEqual(attr.value, "value")
|
||||
self.assertEqual(attr.priority, 10)
|
||||
assert isinstance(attr, SettingsAttribute)
|
||||
assert attr.value == "value"
|
||||
assert attr.priority == 10
|
||||
|
||||
@mock.patch("scrapy.settings.default_settings", default_settings)
|
||||
def test_autopromote_dicts(self):
|
||||
settings = Settings()
|
||||
mydict = settings.get("TEST_DICT")
|
||||
self.assertIsInstance(mydict, BaseSettings)
|
||||
self.assertIn("key", mydict)
|
||||
self.assertEqual(mydict["key"], "val") # pylint: disable=unsubscriptable-object
|
||||
self.assertEqual(mydict.getpriority("key"), 0)
|
||||
assert isinstance(mydict, BaseSettings)
|
||||
assert "key" in mydict
|
||||
assert mydict["key"] == "val"
|
||||
assert mydict.getpriority("key") == 0
|
||||
|
||||
@mock.patch("scrapy.settings.default_settings", default_settings)
|
||||
def test_getdict_autodegrade_basesettings(self):
|
||||
settings = Settings()
|
||||
mydict = settings.getdict("TEST_DICT")
|
||||
self.assertIsInstance(mydict, dict)
|
||||
self.assertEqual(len(mydict), 1)
|
||||
self.assertIn("key", mydict)
|
||||
self.assertEqual(mydict["key"], "val")
|
||||
assert isinstance(mydict, dict)
|
||||
assert len(mydict) == 1
|
||||
assert "key" in mydict
|
||||
assert mydict["key"] == "val"
|
||||
|
||||
def test_passing_objects_as_values(self):
|
||||
from scrapy.core.downloader.handlers.file import FileDownloadHandler
|
||||
|
|
@ -470,19 +465,19 @@ class SettingsTest(unittest.TestCase):
|
|||
}
|
||||
)
|
||||
|
||||
self.assertIn("ITEM_PIPELINES", settings.attributes)
|
||||
assert "ITEM_PIPELINES" in settings.attributes
|
||||
|
||||
mypipeline, priority = settings.getdict("ITEM_PIPELINES").popitem()
|
||||
self.assertEqual(priority, 800)
|
||||
self.assertEqual(mypipeline, TestPipeline)
|
||||
self.assertIsInstance(mypipeline(), TestPipeline)
|
||||
self.assertEqual(mypipeline().process_item("item", None), "item")
|
||||
assert priority == 800
|
||||
assert mypipeline == TestPipeline
|
||||
assert isinstance(mypipeline(), TestPipeline)
|
||||
assert mypipeline().process_item("item", None) == "item"
|
||||
|
||||
myhandler = settings.getdict("DOWNLOAD_HANDLERS").pop("ftp")
|
||||
self.assertEqual(myhandler, FileDownloadHandler)
|
||||
assert myhandler == FileDownloadHandler
|
||||
myhandler_instance = build_from_crawler(myhandler, get_crawler())
|
||||
self.assertIsInstance(myhandler_instance, FileDownloadHandler)
|
||||
self.assertTrue(hasattr(myhandler_instance, "download_request"))
|
||||
assert isinstance(myhandler_instance, FileDownloadHandler)
|
||||
assert hasattr(myhandler_instance, "download_request")
|
||||
|
||||
def test_pop_item_with_default_value(self):
|
||||
settings = Settings()
|
||||
|
|
@ -491,14 +486,14 @@ class SettingsTest(unittest.TestCase):
|
|||
settings.pop("DUMMY_CONFIG")
|
||||
|
||||
dummy_config_value = settings.pop("DUMMY_CONFIG", "dummy_value")
|
||||
self.assertEqual(dummy_config_value, "dummy_value")
|
||||
assert dummy_config_value == "dummy_value"
|
||||
|
||||
def test_pop_item_with_immutable_settings(self):
|
||||
settings = Settings(
|
||||
{"DUMMY_CONFIG": "dummy_value", "OTHER_DUMMY_CONFIG": "other_dummy_value"}
|
||||
)
|
||||
|
||||
self.assertEqual(settings.pop("DUMMY_CONFIG"), "dummy_value")
|
||||
assert settings.pop("DUMMY_CONFIG") == "dummy_value"
|
||||
|
||||
settings.freeze()
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue