RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Switch to pytest.raises(). (#6680) 

* Switch to pytest.raises().

* Add matches= to broad pytest.raises().

* Adjust the test_nonserializable_object() regex for Python <= 3.11.

* Adjust the test_nonserializable_object() regex for PyPy.

* Adjust other test exception regexes for PyPy.

* Cleanup.
This commit is contained in:
Andrey Rakhmatullin 2025-02-27 23:13:04 +05:00 committed by GitHub
parent 391af6afcc
commit 8d92c28a16
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
47 changed files with 482 additions and 394 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pyproject.toml
View File

@ -378,8 +378,6 @@ ignore = [
# Temporarily silenced PT rules
# Use a regular `assert` instead of unittest-style `assertEqual`
"PT009",
# Use `pytest.raises` instead of unittest-style `assertRaises`
"PT027",
]
[tool.ruff.lint.per-file-ignores]

2
scrapy/pqueues.py
View File

@ -214,7 +214,7 @@ class DownloaderAwarePriorityQueue:
"DownloaderAwarePriorityQueue accepts "
"``slot_startprios`` as a dict; "
f"{slot_startprios.__class__!r} instance "
"is passed. Most likely, it means the state is"
"is passed. Most likely, it means the state is "
"created by an incompatible priority queue. "
"Only a crawl started with the same priority "
"queue class can be resumed."

33
tests/test_crawler.py
View File

@ -185,9 +185,8 @@ class CrawlerTestCase(BaseCrawlerTest):
def test_get_downloader_middleware_not_crawling(self):
crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
self.assertRaises(
RuntimeError, crawler.get_downloader_middleware, DefaultSpider
)
with pytest.raises(RuntimeError):
crawler.get_downloader_middleware(DefaultSpider)
@inlineCallbacks
def test_get_downloader_middleware_no_engine(self):
@ -266,7 +265,8 @@ class CrawlerTestCase(BaseCrawlerTest):
def test_get_extension_not_crawling(self):
crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
self.assertRaises(RuntimeError, crawler.get_extension, DefaultSpider)
with pytest.raises(RuntimeError):
crawler.get_extension(DefaultSpider)
@inlineCallbacks
def test_get_extension_no_engine(self):
@ -345,7 +345,8 @@ class CrawlerTestCase(BaseCrawlerTest):
def test_get_item_pipeline_not_crawling(self):
crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
self.assertRaises(RuntimeError, crawler.get_item_pipeline, DefaultSpider)
with pytest.raises(RuntimeError):
crawler.get_item_pipeline(DefaultSpider)
@inlineCallbacks
def test_get_item_pipeline_no_engine(self):
@ -424,7 +425,8 @@ class CrawlerTestCase(BaseCrawlerTest):
def test_get_spider_middleware_not_crawling(self):
crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
self.assertRaises(RuntimeError, crawler.get_spider_middleware, DefaultSpider)
with pytest.raises(RuntimeError):
crawler.get_spider_middleware(DefaultSpider)
@inlineCallbacks
def test_get_spider_middleware_no_engine(self):
@ -537,7 +539,8 @@ class CrawlerRunnerTestCase(BaseCrawlerTest):
"SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
}
)
self.assertRaises(MultipleInvalid, CrawlerRunner, settings)
with pytest.raises(MultipleInvalid):
CrawlerRunner(settings)
def test_crawler_runner_accepts_dict(self):
runner = CrawlerRunner({"foo": "bar"})
@ -630,13 +633,15 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
}
)
else:
msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
with self.assertRaisesRegex(Exception, msg):
runner = CrawlerRunner(
settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
}
)
runner = CrawlerRunner(
settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
}
)
with pytest.raises(
Exception,
match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)",
):
yield runner.crawl(NoRequestsSpider)

2
tests/test_downloader_handlers_http2.py
View File

@ -248,5 +248,5 @@ class Https2ProxyTestCase(BaseTestClasses.Http11ProxyTestCase):
@defer.inlineCallbacks
def test_download_with_proxy_https_timeout(self):
with self.assertRaises(NotImplementedError):
with pytest.raises(NotImplementedError):
yield super().test_download_with_proxy_https_timeout()

4
tests/test_downloadermiddleware.py
View File

@ -1,4 +1,5 @@
import asyncio
from gzip import BadGzipFile
from unittest import mock
import pytest
@ -106,7 +107,8 @@ class DefaultsTest(ManagerTestCase):
"Location": "http://example.com/login",
},
)
self.assertRaises(OSError, self._download, request=req, response=resp)
with pytest.raises(BadGzipFile):
self._download(request=req, response=resp)
class ResponseFromProcessRequestTest(ManagerTestCase):

9
tests/test_downloadermiddleware_cookies.py
View File

@ -83,11 +83,10 @@ class CookiesMiddlewareTest(TestCase):
self.assertEqual(req2.headers.get("Cookie"), b"C1=value1")
def test_setting_false_cookies_enabled(self):
self.assertRaises(
NotConfigured,
CookiesMiddleware.from_crawler,
get_crawler(settings_dict={"COOKIES_ENABLED": False}),
)
with pytest.raises(NotConfigured):
CookiesMiddleware.from_crawler(
get_crawler(settings_dict={"COOKIES_ENABLED": False})
)
def test_setting_default_cookies_enabled(self):
self.assertIsInstance(

5
tests/test_downloadermiddleware_httpauth.py
View File

@ -1,5 +1,6 @@
import unittest
import pytest
from w3lib.http import basic_auth_header
from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
@ -29,8 +30,8 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
self.spider = LegacySpider("foo")
def test_auth(self):
with self.assertRaises(AttributeError):
mw = HttpAuthMiddleware()
mw = HttpAuthMiddleware()
with pytest.raises(AttributeError):
mw.spider_opened(self.spider)

7
tests/test_downloadermiddleware_httpcache.py
View File

@ -5,6 +5,8 @@ import time
import unittest
from contextlib import contextmanager
import pytest
from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware
from scrapy.exceptions import IgnoreRequest
from scrapy.http import HtmlResponse, Request, Response
@ -192,9 +194,8 @@ class DummyPolicyTest(_BaseTest):
def test_middleware_ignore_missing(self):
with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw:
self.assertRaises(
IgnoreRequest, mw.process_request, self.request, self.spider
)
with pytest.raises(IgnoreRequest):
mw.process_request(self.request, self.spider)
mw.process_response(self.request, self.response, self.spider)
response = mw.process_request(self.request, self.spider)
assert isinstance(response, HtmlResponse)

37
tests/test_downloadermiddleware_httpcompression.py
View File

@ -4,6 +4,7 @@ from logging import WARNING
from pathlib import Path
from unittest import SkipTest, TestCase
import pytest
from testfixtures import LogCapture
from w3lib.encoding import resolve_encoding
@ -87,11 +88,10 @@ class HttpCompressionTest(TestCase):
)
def test_setting_false_compression_enabled(self):
self.assertRaises(
NotConfigured,
HttpCompressionMiddleware.from_crawler,
get_crawler(settings_dict={"COMPRESSION_ENABLED": False}),
)
with pytest.raises(NotConfigured):
HttpCompressionMiddleware.from_crawler(
get_crawler(settings_dict={"COMPRESSION_ENABLED": False})
)
def test_setting_default_compression_enabled(self):
self.assertIsInstance(
@ -520,13 +520,8 @@ class HttpCompressionTest(TestCase):
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
self.assertRaises(
IgnoreRequest,
mw.process_response,
response.request,
response,
spider,
)
with pytest.raises(IgnoreRequest):
mw.process_response(response.request, response, spider)
def test_compression_bomb_setting_br(self):
try:
@ -561,13 +556,8 @@ class HttpCompressionTest(TestCase):
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
self.assertRaises(
IgnoreRequest,
mw.process_response,
response.request,
response,
spider,
)
with pytest.raises(IgnoreRequest):
mw.process_response(response.request, response, spider)
def test_compression_bomb_spider_attr_br(self):
try:
@ -600,13 +590,8 @@ class HttpCompressionTest(TestCase):
response = self._getresponse(f"bomb-{compression_id}")
response.meta["download_maxsize"] = 10_000_000
self.assertRaises(
IgnoreRequest,
mw.process_response,
response.request,
response,
spider,
)
with pytest.raises(IgnoreRequest):
mw.process_response(response.request, response, spider)
def test_compression_bomb_request_meta_br(self):
try:

10
tests/test_downloadermiddleware_redirect.py
View File

@ -72,9 +72,8 @@ class Base:
assert isinstance(req, Request)
assert "redirect_times" in req.meta
self.assertEqual(req.meta["redirect_times"], 1)
self.assertRaises(
IgnoreRequest, self.mw.process_response, req, rsp, self.spider
)
with pytest.raises(IgnoreRequest):
self.mw.process_response(req, rsp, self.spider)
def test_ttl(self):
self.mw.max_redirect_times = 100
@ -83,9 +82,8 @@ class Base:
req = self.mw.process_response(req, rsp, self.spider)
assert isinstance(req, Request)
self.assertRaises(
IgnoreRequest, self.mw.process_response, req, rsp, self.spider
)
with pytest.raises(IgnoreRequest):
self.mw.process_response(req, rsp, self.spider)
def test_redirect_urls(self):
req1 = Request("http://scrapytest.org/first")

3
tests/test_downloadermiddleware_retry.py
View File

@ -1,6 +1,7 @@
import logging
import unittest
import pytest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.error import (
@ -407,7 +408,7 @@ class GetRetryRequestTest(unittest.TestCase):
def test_no_spider(self):
request = Request("https://example.com")
with self.assertRaises(TypeError):
with pytest.raises(TypeError):
get_retry_request(request) # pylint: disable=missing-kwoa
def test_max_retry_times_setting(self):

4
tests/test_downloadermiddleware_robotstxt.py
View File

@ -1,5 +1,6 @@
from unittest import mock
import pytest
from twisted.internet import error, reactor
from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
from twisted.python import failure
@ -26,7 +27,8 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
def test_robotstxt_settings(self):
self.crawler.settings = Settings()
self.crawler.settings.set("USER_AGENT", "CustomAgent")
self.assertRaises(NotConfigured, RobotsTxtMiddleware, self.crawler)
with pytest.raises(NotConfigured):
RobotsTxtMiddleware(self.crawler)
def _get_successful_crawler(self):
crawler = self.crawler

14
tests/test_exporters.py
View File

@ -10,6 +10,7 @@ from io import BytesIO
from typing import Any
import lxml.etree
import pytest
from itemadapter import ItemAdapter
from scrapy.exporters import (
@ -147,7 +148,7 @@ class PythonItemExporterTest(BaseItemExporterTest):
return PythonItemExporter(**kwargs)
def test_invalid_option(self):
with self.assertRaisesRegex(TypeError, "Unexpected options: invalid_option"):
with pytest.raises(TypeError, match="Unexpected options: invalid_option"):
PythonItemExporter(invalid_option="something")
def test_nested_item(self):
@ -388,7 +389,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
)
def test_errors_default(self):
with self.assertRaises(UnicodeEncodeError):
with pytest.raises(UnicodeEncodeError):
self.assertExportResult(
item={"text": "W\u0275\u200brd"},
expected=None,
@ -549,7 +550,8 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
self.ie = self._get_exporter(sort_keys=True)
self.test_export_item()
self._check_output()
self.assertRaises(TypeError, self._get_exporter, foo_unknown_keyword_bar=True)
with pytest.raises(TypeError):
self._get_exporter(foo_unknown_keyword_bar=True)
def test_nonstring_types_item(self):
item = self._get_nonstring_types_item()
@ -602,7 +604,8 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
i3 = MyItem(name="Jesus", age="44")
self.ie.start_exporting()
self.ie.export_item(i1)
self.assertRaises(TypeError, self.ie.export_item, i2)
with pytest.raises(TypeError):
self.ie.export_item(i2)
self.ie.export_item(i3)
self.ie.finish_exporting()
exported = json.loads(to_unicode(self.output.getvalue()))
@ -657,7 +660,8 @@ class JsonItemExporterToBytesTest(BaseItemExporterTest):
i3 = MyItem(name="Jesus", age="44")
self.ie.start_exporting()
self.ie.export_item(i1)
self.assertRaises(UnicodeEncodeError, self.ie.export_item, i2)
with pytest.raises(UnicodeEncodeError):
self.ie.export_item(i2)
self.ie.export_item(i3)
self.ie.finish_exporting()
exported = json.loads(to_unicode(self.output.getvalue(), encoding="latin"))

12
tests/test_feedexport.py
View File

@ -233,7 +233,8 @@ class BlockingFeedStorageTest(unittest.TestCase):
invalid_path = tests_path / "invalid_path"
spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)})
self.assertRaises(OSError, b.open, spider=spider)
with pytest.raises(OSError, match="Not a Directory:"):
b.open(spider=spider)
@pytest.mark.requires_boto3
@ -2437,7 +2438,8 @@ class BatchDeliveriesTest(FeedExportTestBase):
"FEED_EXPORT_BATCH_ITEM_COUNT": 1,
}
crawler = get_crawler(settings_dict=settings)
self.assertRaises(NotConfigured, FeedExporter, crawler)
with pytest.raises(NotConfigured):
FeedExporter(crawler)
@defer.inlineCallbacks
def test_export_no_items_not_store_empty(self):
@ -2758,7 +2760,7 @@ class FeedExportInitTest(unittest.TestCase):
},
}
crawler = get_crawler(settings_dict=settings)
with self.assertRaises(NotConfigured):
with pytest.raises(NotConfigured):
FeedExporter.from_crawler(crawler)
def test_unsupported_format(self):
@ -2770,7 +2772,7 @@ class FeedExportInitTest(unittest.TestCase):
},
}
crawler = get_crawler(settings_dict=settings)
with self.assertRaises(NotConfigured):
with pytest.raises(NotConfigured):
FeedExporter.from_crawler(crawler)
def test_absolute_pathlib_as_uri(self):
@ -2863,7 +2865,7 @@ class URIParamsTest:
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
with self.assertRaises(KeyError):
with pytest.raises(KeyError):
feed_exporter.open_spider(spider)
def test_params_as_is(self):

5
tests/test_http2_client_protocol.py
View File

@ -12,6 +12,7 @@ from typing import TYPE_CHECKING
from unittest import mock, skipIf
from urllib.parse import urlencode
import pytest
from twisted.internet import reactor
from twisted.internet.defer import (
CancelledError,
@ -406,7 +407,7 @@ class Https2ClientProtocolTestCase(TestCase):
"scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2"
):
request = Request(url=self.get_url("/status?n=200"))
with self.assertRaises(ResponseFailed):
with pytest.raises(ResponseFailed):
yield self.make_request(request)
def test_cancel_request(self):
@ -560,7 +561,7 @@ class Https2ClientProtocolTestCase(TestCase):
return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True)
def test_invalid_request_type(self):
with self.assertRaises(TypeError):
with pytest.raises(TypeError):
self.make_request("https://InvalidDataTypePassed.com")
def test_query_parameters(self):

25
tests/test_http_headers.py
View File

@ -1,6 +1,8 @@
import copy
import unittest
import pytest
from scrapy.http import Headers
@ -13,7 +15,8 @@ class HeadersTest(unittest.TestCase):
assert h["Content-Type"]
assert h["Content-Length"]
self.assertRaises(KeyError, h.__getitem__, "Accept")
with pytest.raises(KeyError):
h["Accept"]
self.assertEqual(h.get("Accept"), None)
self.assertEqual(h.getlist("Accept"), [])
@ -152,15 +155,11 @@ class HeadersTest(unittest.TestCase):
self.assertEqual(h1.getlist("hey"), [b"5"])
def test_invalid_value(self):
self.assertRaisesRegex(
TypeError, "Unsupported value type", Headers, {"foo": object()}
)
self.assertRaisesRegex(
TypeError, "Unsupported value type", Headers().__setitem__, "foo", object()
)
self.assertRaisesRegex(
TypeError, "Unsupported value type", Headers().setdefault, "foo", object()
)
self.assertRaisesRegex(
TypeError, "Unsupported value type", Headers().setlist, "foo", [object()]
)
with pytest.raises(TypeError, match="Unsupported value type"):
Headers({"foo": object()})
with pytest.raises(TypeError, match="Unsupported value type"):
Headers()["foo"] = object()
with pytest.raises(TypeError, match="Unsupported value type"):
Headers().setdefault("foo", object())
with pytest.raises(TypeError, match="Unsupported value type"):
Headers().setlist("foo", [object()])

149
tests/test_http_request.py
View File

@ -7,6 +7,8 @@ from typing import Any
from unittest import mock
from urllib.parse import parse_qs, unquote_to_bytes
import pytest
from scrapy.http import (
FormRequest,
Headers,
@ -28,10 +30,12 @@ class RequestTest(unittest.TestCase):
def test_init(self):
# Request requires url in the __init__ method
self.assertRaises(Exception, self.request_class)
with pytest.raises(TypeError):
self.request_class()
# url argument must be basestring
self.assertRaises(TypeError, self.request_class, 123)
with pytest.raises(TypeError):
self.request_class(123)
r = self.request_class("http://www.example.com")
r = self.request_class("http://www.example.com")
@ -64,9 +68,13 @@ class RequestTest(unittest.TestCase):
self.request_class("data:,Hello%2C%20World!")
def test_url_no_scheme(self):
self.assertRaises(ValueError, self.request_class, "foo")
self.assertRaises(ValueError, self.request_class, "/foo/")
self.assertRaises(ValueError, self.request_class, "/foo:bar")
msg = "Missing scheme in request url:"
with pytest.raises(ValueError, match=msg):
self.request_class("foo")
with pytest.raises(ValueError, match=msg):
self.request_class("/foo/")
with pytest.raises(ValueError, match=msg):
self.request_class("/foo:bar")
def test_headers(self):
# Different ways of setting headers attribute
@ -273,8 +281,10 @@ class RequestTest(unittest.TestCase):
def test_immutable_attributes(self):
r = self.request_class("http://example.com")
self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com")
self.assertRaises(AttributeError, setattr, r, "body", "xxx")
with pytest.raises(AttributeError):
r.url = "http://example2.com"
with pytest.raises(AttributeError):
r.body = "xxx"
def test_callback_and_errback(self):
def a_function():
@ -309,11 +319,11 @@ class RequestTest(unittest.TestCase):
self.assertIs(r5.errback, NO_CALLBACK)
def test_callback_and_errback_type(self):
with self.assertRaises(TypeError):
with pytest.raises(TypeError):
self.request_class("http://example.com", callback="a_function")
with self.assertRaises(TypeError):
with pytest.raises(TypeError):
self.request_class("http://example.com", errback="a_function")
with self.assertRaises(TypeError):
with pytest.raises(TypeError):
self.request_class(
url="http://example.com",
callback="a_function",
@ -321,7 +331,7 @@ class RequestTest(unittest.TestCase):
)
def test_no_callback(self):
with self.assertRaises(RuntimeError):
with pytest.raises(RuntimeError):
NO_CALLBACK()
def test_from_curl(self):
@ -403,13 +413,11 @@ class RequestTest(unittest.TestCase):
# If `ignore_unknown_options` is set to `False` it raises an error with
# the unknown options: --foo and -z
self.assertRaises(
ValueError,
lambda: self.request_class.from_curl(
with pytest.raises(ValueError, match="Unrecognized options:"):
self.request_class.from_curl(
'curl -X PATCH "http://example.org" --foo -z',
ignore_unknown_options=False,
),
)
)
class FormRequestTest(RequestTest):
@ -428,7 +436,7 @@ class FormRequestTest(RequestTest):
data = (("a", "one"), ("a", "two"), ("b", "2"))
url = self.request_class(
"http://www.example.com/?a=0&b=1&c=3#fragment", method="GET", formdata=data
).url.split("#")[0]
).url.split("#", maxsplit=1)[0]
fs = _qs(self.request_class(url, method="GET", formdata=data))
self.assertEqual(set(fs[b"a"]), {b"one", b"two"})
self.assertEqual(fs[b"b"], [b"2"])
@ -897,12 +905,11 @@ class FormRequestTest(RequestTest):
<input type="submit" name="clickable2" value="clicked2">
</form>"""
)
self.assertRaises(
with pytest.raises(
ValueError,
self.request_class.from_response,
response,
clickdata={"type": "submit"},
)
match="Multiple elements found .* matching the criteria in clickdata",
):
self.request_class.from_response(response, clickdata={"type": "submit"})
def test_from_response_non_matching_clickdata(self):
response = _buildresponse(
@ -910,12 +917,12 @@ class FormRequestTest(RequestTest):
<input type="submit" name="clickable" value="clicked">
</form>"""
)
self.assertRaises(
ValueError,
self.request_class.from_response,
response,
clickdata={"nonexistent": "notme"},
)
with pytest.raises(
ValueError, match="No clickable element matching clickdata:"
):
self.request_class.from_response(
response, clickdata={"nonexistent": "notme"}
)
def test_from_response_nr_index_clickdata(self):
response = _buildresponse(
@ -937,13 +944,15 @@ class FormRequestTest(RequestTest):
</form>
"""
)
self.assertRaises(
ValueError, self.request_class.from_response, response, clickdata={"nr": 1}
)
with pytest.raises(
ValueError, match="No clickable element matching clickdata:"
):
self.request_class.from_response(response, clickdata={"nr": 1})
def test_from_response_errors_noform(self):
response = _buildresponse("""<html></html>""")
self.assertRaises(ValueError, self.request_class.from_response, response)
with pytest.raises(ValueError, match="No <form> element found in"):
self.request_class.from_response(response)
def test_from_response_invalid_html5(self):
response = _buildresponse(
@ -963,9 +972,8 @@ class FormRequestTest(RequestTest):
<input type="hidden" name="test2" value="xxx">
</form>"""
)
self.assertRaises(
IndexError, self.request_class.from_response, response, formnumber=1
)
with pytest.raises(IndexError):
self.request_class.from_response(response, formnumber=1)
def test_from_response_noformname(self):
response = _buildresponse(
@ -1021,13 +1029,8 @@ class FormRequestTest(RequestTest):
<input type="hidden" name="two" value="2">
</form>"""
)
self.assertRaises(
IndexError,
self.request_class.from_response,
response,
formname="form3",
formnumber=2,
)
with pytest.raises(IndexError):
self.request_class.from_response(response, formname="form3", formnumber=2)
def test_from_response_formid_exists(self):
response = _buildresponse(
@ -1086,13 +1089,8 @@ class FormRequestTest(RequestTest):
<input type="hidden" name="two" value="2">
</form>"""
)
self.assertRaises(
IndexError,
self.request_class.from_response,
response,
formid="form3",
formnumber=2,
)
with pytest.raises(IndexError):
self.request_class.from_response(response, formid="form3", formnumber=2)
def test_from_response_select(self):
res = _buildresponse(
@ -1245,12 +1243,10 @@ class FormRequestTest(RequestTest):
fs = _qs(r1)
self.assertEqual(fs[b"three"], [b"3"])
self.assertRaises(
ValueError,
self.request_class.from_response,
response,
formxpath="//form/input[@name='abc']",
)
with pytest.raises(ValueError, match="No <form> element found with"):
self.request_class.from_response(
response, formxpath="//form/input[@name='abc']"
)
def test_from_response_unicode_xpath(self):
response = _buildresponse(b'<form name="\xd1\x8a"></form>')
@ -1261,13 +1257,8 @@ class FormRequestTest(RequestTest):
self.assertEqual(fs, {})
xpath = "//form[@name='\u03b1']"
self.assertRaisesRegex(
ValueError,
re.escape(xpath),
self.request_class.from_response,
response,
formxpath=xpath,
)
with pytest.raises(ValueError, match=re.escape(xpath)):
self.request_class.from_response(response, formxpath=xpath)
def test_from_response_button_submit(self):
response = _buildresponse(
@ -1393,12 +1384,8 @@ class FormRequestTest(RequestTest):
fs = _qs(r1)
self.assertEqual(fs[b"three"], [b"3"])
self.assertRaises(
ValueError,
self.request_class.from_response,
response,
formcss="input[name='abc']",
)
with pytest.raises(ValueError, match="No <form> element found with"):
self.request_class.from_response(response, formcss="input[name='abc']")
def test_from_response_valid_form_methods(self):
form_methods = [
@ -1424,13 +1411,11 @@ class FormRequestTest(RequestTest):
</form>
</body></html>"""
)
with self.assertRaises(ValueError) as context:
with pytest.raises(
ValueError, match="formdata should be a dict or iterable of tuples"
):
FormRequest.from_response(response, formdata=123)
self.assertIn(
"formdata should be a dict or iterable of tuples", str(context.exception)
)
def test_form_response_with_custom_invalid_formdata_value_error(self):
"""Test that a ValueError is raised for fault-inducing iterable formdata input"""
response = _buildresponse(
@ -1441,13 +1426,11 @@ class FormRequestTest(RequestTest):
</body></html>"""
)
with self.assertRaises(ValueError) as context:
with pytest.raises(
ValueError, match="formdata should be a dict or iterable of tuples"
):
FormRequest.from_response(response, formdata=("a",))
self.assertIn(
"formdata should be a dict or iterable of tuples", str(context.exception)
)
def test_get_form_with_xpath_no_form_parent(self):
"""Test that _get_from raised a ValueError when an XPath selects an element
not nested within a <form> and no <form> parent is found"""
@ -1462,11 +1445,9 @@ class FormRequestTest(RequestTest):
</body></html>"""
)
with self.assertRaises(ValueError) as context:
with pytest.raises(ValueError, match="No <form> element found with"):
FormRequest.from_response(response, formxpath='//div[@id="outside-form"]/p')
self.assertIn("No <form> element found with", str(context.exception))
def _buildresponse(body, **kwargs):
kwargs.setdefault("body", body)
@ -1507,8 +1488,10 @@ class XmlRpcRequestTest(RequestTest):
self._test_request(params=("response",), methodresponse="login")
self._test_request(params=("pas£",), encoding="utf-8")
self._test_request(params=(None,), allow_none=1)
self.assertRaises(TypeError, self._test_request)
self.assertRaises(TypeError, self._test_request, params=(None,))
with pytest.raises(TypeError):
self._test_request()
with pytest.raises(TypeError):
self._test_request(params=(None,))
def test_latin1(self):
self._test_request(params=("pas£",), encoding="latin1")

113
tests/test_http_response.py
View File

@ -27,14 +27,15 @@ class BaseResponseTest(unittest.TestCase):
def test_init(self):
# Response requires url in the constructor
self.assertRaises(Exception, self.response_class)
with pytest.raises(TypeError):
self.response_class()
self.assertTrue(
isinstance(self.response_class("http://example.com/"), self.response_class)
)
self.assertRaises(TypeError, self.response_class, b"http://example.com")
self.assertRaises(
TypeError, self.response_class, url="http://example.com", body={}
)
with pytest.raises(TypeError):
self.response_class(b"http://example.com")
with pytest.raises(TypeError):
self.response_class(url="http://example.com", body={})
# body can be str or None
self.assertTrue(
isinstance(
@ -77,12 +78,8 @@ class BaseResponseTest(unittest.TestCase):
self.assertEqual(r.status, 301)
r = self.response_class("http://www.example.com", status="301")
self.assertEqual(r.status, 301)
self.assertRaises(
ValueError,
self.response_class,
"http://example.com",
status="lala200",
)
with pytest.raises(ValueError, match=r"invalid literal for int\(\)"):
self.response_class("http://example.com", status="lala200")
def test_copy(self):
"""Test Response copy"""
@ -122,14 +119,12 @@ class BaseResponseTest(unittest.TestCase):
def test_unavailable_meta(self):
r1 = self.response_class("http://www.example.com", body=b"Some body")
with self.assertRaisesRegex(AttributeError, r"Response\.meta not available"):
with pytest.raises(AttributeError, match=r"Response\.meta not available"):
r1.meta
def test_unavailable_cb_kwargs(self):
r1 = self.response_class("http://www.example.com", body=b"Some body")
with self.assertRaisesRegex(
AttributeError, r"Response\.cb_kwargs not available"
):
with pytest.raises(AttributeError, match=r"Response\.cb_kwargs not available"):
r1.cb_kwargs
def test_copy_inherited_classes(self):
@ -179,8 +174,10 @@ class BaseResponseTest(unittest.TestCase):
def test_immutable_attributes(self):
r = self.response_class("http://example.com")
self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com")
self.assertRaises(AttributeError, setattr, r, "body", "xxx")
with pytest.raises(AttributeError):
r.url = "http://example2.com"
with pytest.raises(AttributeError):
r.body = "xxx"
def test_urljoin(self):
"""Test urljoin shortcut (only for existence, since behavior equals urljoin)"""
@ -192,10 +189,14 @@ class BaseResponseTest(unittest.TestCase):
r = self.response_class("http://example.com", body=b"hello")
if self.response_class == Response:
msg = "Response content isn't text"
self.assertRaisesRegex(AttributeError, msg, getattr, r, "text")
self.assertRaisesRegex(NotSupported, msg, r.css, "body")
self.assertRaisesRegex(NotSupported, msg, r.xpath, "//body")
self.assertRaisesRegex(NotSupported, msg, r.jmespath, "body")
with pytest.raises(AttributeError, match=msg):
r.text
with pytest.raises(NotSupported, match=msg):
r.css("body")
with pytest.raises(NotSupported, match=msg):
r.xpath("//body")
with pytest.raises(NotSupported, match=msg):
r.jmespath("body")
else:
r.text
r.css("body")
@ -216,7 +217,8 @@ class BaseResponseTest(unittest.TestCase):
def test_follow_None_url(self):
r = self.response_class("http://example.com")
self.assertRaises(ValueError, r.follow, None)
with pytest.raises(ValueError, match="url can't be None"):
r.follow(None)
@pytest.mark.xfail(
parse_version(w3lib_version) < parse_version("2.1.1"),
@ -279,18 +281,20 @@ class BaseResponseTest(unittest.TestCase):
def test_follow_all_invalid(self):
r = self.response_class("http://example.com")
if self.response_class == Response:
with self.assertRaises(TypeError):
with pytest.raises(TypeError):
list(r.follow_all(urls=None))
with self.assertRaises(TypeError):
with pytest.raises(TypeError):
list(r.follow_all(urls=12345))
with self.assertRaises(ValueError):
with pytest.raises(ValueError, match="url can't be None"):
list(r.follow_all(urls=[None]))
else:
with self.assertRaises(ValueError):
with pytest.raises(
ValueError, match="Please supply exactly one of the following arguments"
):
list(r.follow_all(urls=None))
with self.assertRaises(TypeError):
with pytest.raises(TypeError):
list(r.follow_all(urls=12345))
with self.assertRaises(ValueError):
with pytest.raises(ValueError, match="url can't be None"):
list(r.follow_all(urls=[None]))
def test_follow_all_whitespace(self):
@ -399,12 +403,8 @@ class TextResponseTest(BaseResponseTest):
"\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 "
"\u0442\u0435\u043a\u0441\u0442"
)
self.assertRaises(
TypeError,
self.response_class,
"http://www.example.com",
body="unicode body",
)
with pytest.raises(TypeError):
self.response_class("http://www.example.com", body="unicode body")
original_string = unicode_string.encode("cp1251")
r1 = self.response_class(
@ -483,12 +483,8 @@ class TextResponseTest(BaseResponseTest):
self._assert_response_values(r9, "cp1252", "€")
# TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies
self.assertRaises(
TypeError,
self.response_class,
"http://www.example.com",
body="\xa3",
)
with pytest.raises(TypeError):
self.response_class("http://www.example.com", body="\xa3")
def test_declared_encoding_invalid(self):
"""Check that unknown declared encodings are ignored"""
@ -679,20 +675,20 @@ class TextResponseTest(BaseResponseTest):
self._assert_followed_url(sel, url, response=resp)
# non-a elements are not supported
self.assertRaises(ValueError, resp.follow, resp.css("div")[0])
with pytest.raises(
ValueError, match="Only <a> and <link> elements are supported"
):
resp.follow(resp.css("div")[0])
def test_follow_selector_list(self):
resp = self._links_response()
self.assertRaisesRegex(ValueError, "SelectorList", resp.follow, resp.css("a"))
with pytest.raises(ValueError, match="SelectorList"):
resp.follow(resp.css("a"))
def test_follow_selector_invalid(self):
resp = self._links_response()
self.assertRaisesRegex(
ValueError,
"Unsupported",
resp.follow,
resp.xpath("count(//div)")[0],
)
with pytest.raises(ValueError, match="Unsupported"):
resp.follow(resp.xpath("count(//div)")[0])
def test_follow_selector_attribute(self):
resp = self._links_response()
@ -704,7 +700,8 @@ class TextResponseTest(BaseResponseTest):
url="http://example.com",
body=b"<html><body><a name=123>click me</a></body></html>",
)
self.assertRaisesRegex(ValueError, "no href", resp.follow, resp.css("a")[0])
with pytest.raises(ValueError, match="no href"):
resp.follow(resp.css("a")[0])
def test_follow_whitespace_selector(self):
resp = self.response_class(
@ -812,7 +809,9 @@ class TextResponseTest(BaseResponseTest):
def test_follow_all_too_many_arguments(self):
response = self._links_response()
with self.assertRaises(ValueError):
with pytest.raises(
ValueError, match="Please supply exactly one of the following arguments"
):
response.follow_all(
css='a[href*="example.com"]',
xpath='//a[contains(@href, "example.com")]',
@ -825,7 +824,9 @@ class TextResponseTest(BaseResponseTest):
text_body = b"""<html><body>text</body></html>"""
text_response = self.response_class("http://www.example.com", body=text_body)
with self.assertRaises(ValueError):
with pytest.raises(
ValueError, match="(Expecting value|Unexpected '<'): line 1"
):
text_response.json()
def test_cache_json_response(self):
@ -1023,10 +1024,8 @@ class CustomResponseTest(TextResponseTest):
self.assertEqual(r4.bar, "bar")
self.assertIsNone(r4.lost)
with self.assertRaises(TypeError) as ctx:
with pytest.raises(
TypeError,
match=r"__init__\(\) got an unexpected keyword argument 'unknown'",
):
r1.replace(unknown="unknown")
self.assertTrue(
str(ctx.exception).endswith(
"__init__() got an unexpected keyword argument 'unknown'"
)
)

29
tests/test_item.py
View File

@ -2,6 +2,8 @@ import unittest
from abc import ABCMeta
from unittest import mock
import pytest
from scrapy.item import Field, Item, ItemMeta
@ -22,7 +24,8 @@ class ItemTest(unittest.TestCase):
name = Field()
i = TestItem()
self.assertRaises(KeyError, i.__getitem__, "name")
with pytest.raises(KeyError):
i["name"]
i2 = TestItem(name="john doe")
self.assertEqual(i2["name"], "john doe")
@ -33,15 +36,18 @@ class ItemTest(unittest.TestCase):
i4 = TestItem(i3)
self.assertEqual(i4["name"], "john doe")
self.assertRaises(KeyError, TestItem, {"name": "john doe", "other": "foo"})
with pytest.raises(KeyError):
TestItem({"name": "john doe", "other": "foo"})
def test_invalid_field(self):
class TestItem(Item):
pass
i = TestItem()
self.assertRaises(KeyError, i.__setitem__, "field", "text")
self.assertRaises(KeyError, i.__getitem__, "field")
with pytest.raises(KeyError):
i["field"] = "text"
with pytest.raises(KeyError):
i["field"]
def test_repr(self):
class TestItem(Item):
@ -72,14 +78,16 @@ class ItemTest(unittest.TestCase):
name = Field()
i = TestItem()
self.assertRaises(AttributeError, getattr, i, "name")
with pytest.raises(AttributeError):
i.name
def test_raise_setattr(self):
class TestItem(Item):
name = Field()
i = TestItem()
self.assertRaises(AttributeError, setattr, i, "name", "john")
with pytest.raises(AttributeError):
i.name = "john"
def test_custom_methods(self):
class TestItem(Item):
@ -92,7 +100,8 @@ class ItemTest(unittest.TestCase):
self["name"] = name
i = TestItem()
self.assertRaises(KeyError, i.get_name)
with pytest.raises(KeyError):
i.get_name()
i["name"] = "lala"
self.assertEqual(i.get_name(), "lala")
i.change_name("other")
@ -223,7 +232,8 @@ class ItemTest(unittest.TestCase):
class D(B, C):
pass
self.assertRaises(KeyError, D, not_allowed="value")
with pytest.raises(KeyError):
D(not_allowed="value")
self.assertEqual(D(save="X")["save"], "X")
self.assertEqual(D.fields, {"save": {"default": "A"}, "load": {"default": "A"}})
@ -231,7 +241,8 @@ class ItemTest(unittest.TestCase):
class E(C, B):
pass
self.assertRaises(KeyError, E, not_allowed="value")
with pytest.raises(KeyError):
E(not_allowed="value")
self.assertEqual(E(save="X")["save"], "X")
self.assertEqual(E.fields, {"save": {"default": "A"}, "load": {"default": "A"}})

4
tests/test_link.py
View File

@ -1,5 +1,7 @@
import unittest
import pytest
from scrapy.link import Link
@ -53,5 +55,5 @@ class LinkTest(unittest.TestCase):
self._assert_same_links(l1, l2)
def test_bytes_url(self):
with self.assertRaises(TypeError):
with pytest.raises(TypeError):
Link(b"http://www.example.com/\xc2\xa3")

22
tests/test_loader.py
View File

@ -4,6 +4,7 @@ import dataclasses
import unittest
import attr
import pytest
from itemadapter import ItemAdapter
from itemloaders.processors import Compose, Identity, MapCompose, TakeFirst
@ -69,7 +70,8 @@ def processor_with_args(value, other=None, loader_context=None):
class BasicItemLoaderTest(unittest.TestCase):
def test_add_value_on_unknown_field(self):
il = ProcessorItemLoader()
self.assertRaises(KeyError, il.add_value, "wrong_field", ["lala", "lolo"])
with pytest.raises(KeyError):
il.add_value("wrong_field", ["lala", "lolo"])
def test_load_item_using_default_loader(self):
i = SummaryItem()
@ -294,12 +296,18 @@ class SelectortemLoaderTest(unittest.TestCase):
def test_init_method_errors(self):
l = ProcessorItemLoader()
self.assertRaises(RuntimeError, l.add_xpath, "url", "//a/@href")
self.assertRaises(RuntimeError, l.replace_xpath, "url", "//a/@href")
self.assertRaises(RuntimeError, l.get_xpath, "//a/@href")
self.assertRaises(RuntimeError, l.add_css, "name", "#name::text")
self.assertRaises(RuntimeError, l.replace_css, "name", "#name::text")
self.assertRaises(RuntimeError, l.get_css, "#name::text")
with pytest.raises(RuntimeError):
l.add_xpath("url", "//a/@href")
with pytest.raises(RuntimeError):
l.replace_xpath("url", "//a/@href")
with pytest.raises(RuntimeError):
l.get_xpath("//a/@href")
with pytest.raises(RuntimeError):
l.add_css("name", "#name::text")
with pytest.raises(RuntimeError):
l.replace_css("name", "#name::text")
with pytest.raises(RuntimeError):
l.get_css("#name::text")
def test_init_method_with_selector(self):
sel = Selector(text="<html><body><div>marta</div></body></html>")

53
tests/test_loader_deprecated.py
View File

@ -6,6 +6,7 @@ Once we remove the references from scrapy, we can remove these tests.
import unittest
from functools import partial
import pytest
from itemloaders.processors import (
Compose,
Identity,
@ -435,7 +436,13 @@ class BasicItemLoaderTest(unittest.TestCase):
name_in = MapCompose(float)
il = TestItemLoader()
self.assertRaises(ValueError, il.add_value, "name", ["marta", "other"])
with pytest.raises(
ValueError,
match="Error with input processor MapCompose: .* "
"error='ValueError: Error in MapCompose .* "
"error='ValueError: could not convert",
):
il.add_value("name", ["marta", "other"])
def test_error_output_processor(self):
class TestItem(Item):
@ -447,7 +454,12 @@ class BasicItemLoaderTest(unittest.TestCase):
il = TestItemLoader()
il.add_value("name", "marta")
with self.assertRaises(ValueError):
with pytest.raises(
ValueError,
match="Error with output processor: .* "
"error='ValueError: Error in Compose .* "
"error='ValueError: could not convert",
):
il.load_item()
def test_error_processor_as_argument(self):
@ -458,9 +470,13 @@ class BasicItemLoaderTest(unittest.TestCase):
default_item_class = TestItem
il = TestItemLoader()
self.assertRaises(
ValueError, il.add_value, "name", ["marta", "other"], Compose(float)
)
with pytest.raises(
ValueError,
match=r"Error with processor Compose .* "
r"error='ValueError: Error in Compose .* "
r"error='TypeError: float\(\) argument",
):
il.add_value("name", ["marta", "other"], Compose(float))
class InitializationFromDictTest(unittest.TestCase):
@ -630,7 +646,8 @@ class ProcessorsTest(unittest.TestCase):
def test_join(self):
proc = Join()
self.assertRaises(TypeError, proc, [None, "", "hello", "world"])
with pytest.raises(TypeError):
proc([None, "", "hello", "world"])
self.assertEqual(proc(["", "hello", "world"]), " hello world")
self.assertEqual(proc(["hello", "world"]), "hello world")
self.assertIsInstance(proc(["hello", "world"]), str)
@ -641,9 +658,17 @@ class ProcessorsTest(unittest.TestCase):
proc = Compose(str.upper)
self.assertEqual(proc(None), None)
proc = Compose(str.upper, stop_on_none=False)
self.assertRaises(ValueError, proc, None)
with pytest.raises(
ValueError,
match="Error in Compose with .* error='TypeError: (descriptor 'upper'|'str' object expected)",
):
proc(None)
proc = Compose(str.upper, lambda x: x + 1)
self.assertRaises(ValueError, proc, "hello")
with pytest.raises(
ValueError,
match="Error in Compose with .* error='TypeError: (can only|unsupported operand)",
):
proc("hello")
def test_mapcompose(self):
def filter_world(x):
@ -657,9 +682,17 @@ class ProcessorsTest(unittest.TestCase):
proc = MapCompose(filter_world, str.upper)
self.assertEqual(proc(None), [])
proc = MapCompose(filter_world, str.upper)
self.assertRaises(ValueError, proc, [1])
with pytest.raises(
ValueError,
match="Error in MapCompose with .* error='TypeError: (descriptor 'upper'|'str' object expected)",
):
proc([1])
proc = MapCompose(filter_world, lambda x: x + 1)
self.assertRaises(ValueError, proc, "hello")
with pytest.raises(
ValueError,
match="Error in MapCompose with .* error='TypeError: (can only|unsupported operand)",
):
proc("hello")
class SelectJmesTestCase(unittest.TestCase):

5
tests/test_logstats.py
View File

@ -1,6 +1,8 @@
import unittest
from datetime import datetime
import pytest
from scrapy.extensions.logstats import LogStats
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
@ -18,8 +20,9 @@ class TestLogStats(unittest.TestCase):
def test_stats_calculations(self):
logstats = LogStats.from_crawler(self.crawler)
with self.assertRaises(AttributeError):
with pytest.raises(AttributeError):
logstats.pagesprev
with pytest.raises(AttributeError):
logstats.itemsprev
logstats.spider_opened(self.spider)

7
tests/test_pipeline_images.py
View File

@ -7,6 +7,7 @@ from shutil import rmtree
from tempfile import mkdtemp
import attr
import pytest
from itemadapter import ItemAdapter
from twisted.trial import unittest
@ -146,11 +147,11 @@ class ImagesPipelineTestCase(unittest.TestCase):
resp3 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf3.getvalue())
req = Request(url="https://dev.mydeco.com/mydeco.gif")
with self.assertRaises(ImageException):
with pytest.raises(ImageException):
next(self.pipeline.get_images(response=resp1, request=req, info=object()))
with self.assertRaises(ImageException):
with pytest.raises(ImageException):
next(self.pipeline.get_images(response=resp2, request=req, info=object()))
with self.assertRaises(ImageException):
with pytest.raises(ImageException):
next(self.pipeline.get_images(response=resp3, request=req, info=object()))
def test_get_images(self):

9
tests/test_pqueues.py
View File

@ -1,6 +1,7 @@
import tempfile
import unittest
import pytest
import queuelib
from scrapy.http.request import Request
@ -40,9 +41,9 @@ class PriorityQueueTest(unittest.TestCase):
self.crawler, FifoMemoryQueue, temp_dir
)
queue.push(Request("https://example.org"))
with self.assertRaises(
with pytest.raises(
NotImplementedError,
msg="The underlying queue class does not implement 'peek'",
match="The underlying queue class does not implement 'peek'",
):
queue.peek()
queue.close()
@ -129,9 +130,9 @@ class DownloaderAwarePriorityQueueTest(unittest.TestCase):
if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined")
self.queue.push(Request("https://example.org"))
with self.assertRaises(
with pytest.raises(
NotImplementedError,
msg="The underlying queue class does not implement 'peek'",
match="The underlying queue class does not implement 'peek'",
):
self.queue.peek()

18
tests/test_request_dict.py
View File

@ -1,5 +1,7 @@
import unittest
import pytest
from scrapy import Request, Spider
from scrapy.http import FormRequest, JsonRequest
from scrapy.utils.request import request_from_dict
@ -134,11 +136,15 @@ class RequestSerializationTest(unittest.TestCase):
def test_unserializable_callback1(self):
r = Request("http://www.example.com", callback=lambda x: x)
self.assertRaises(ValueError, r.to_dict, spider=self.spider)
with pytest.raises(
ValueError, match="is not an instance method in: <MethodsSpider"
):
r.to_dict(spider=self.spider)
def test_unserializable_callback2(self):
r = Request("http://www.example.com", callback=self.spider.parse_item)
self.assertRaises(ValueError, r.to_dict, spider=None)
with pytest.raises(ValueError, match="is not an instance method in: None"):
r.to_dict(spider=None)
def test_unserializable_callback3(self):
"""Parser method is removed or replaced dynamically."""
@ -152,14 +158,18 @@ class RequestSerializationTest(unittest.TestCase):
spider = MySpider()
r = Request("http://www.example.com", callback=spider.parse)
spider.parse = None
self.assertRaises(ValueError, r.to_dict, spider=spider)
with pytest.raises(ValueError, match="is not an instance method in: <MySpider"):
r.to_dict(spider=spider)
def test_callback_not_available(self):
"""Callback method is not available in the spider passed to from_dict"""
spider = SpiderDelegation()
r = Request("http://www.example.com", callback=spider.delegated_callback)
d = r.to_dict(spider=spider)
self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo"))
with pytest.raises(
ValueError, match="Method 'delegated_callback' not found in: <Spider"
):
request_from_dict(d, spider=Spider("foo"))
class SpiderMixin:

10
tests/test_scheduler.py
View File

@ -5,6 +5,7 @@ import tempfile
import unittest
from typing import Any, NamedTuple
import pytest
from twisted.internet import defer
from twisted.trial.unittest import TestCase
@ -229,7 +230,10 @@ class TestMigration(unittest.TestCase):
next_scheduler_handler.create_scheduler()
def test_migration(self):
with self.assertRaises(ValueError):
with pytest.raises(
ValueError,
match="DownloaderAwarePriorityQueue accepts ``slot_startprios`` as a dict",
):
self._migration(self.tmpdir)
@ -351,5 +355,7 @@ class TestIncompatibility(unittest.TestCase):
scheduler.open(spider)
def test_incompatibility(self):
with self.assertRaises(ValueError):
with pytest.raises(
ValueError, match="does not support CONCURRENT_REQUESTS_PER_IP"
):
self._incompatible()

22
tests/test_scheduler_base.py
View File

@ -3,6 +3,7 @@ from __future__ import annotations
from unittest import TestCase
from urllib.parse import urljoin
import pytest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial.unittest import TestCase as TwistedTestCase
@ -75,13 +76,12 @@ class BaseSchedulerTest(TestCase, InterfaceCheckMixin):
def test_methods(self):
self.assertIsNone(self.scheduler.open(Spider("foo")))
self.assertIsNone(self.scheduler.close("finished"))
self.assertRaises(NotImplementedError, self.scheduler.has_pending_requests)
self.assertRaises(
NotImplementedError,
self.scheduler.enqueue_request,
Request("https://example.org"),
)
self.assertRaises(NotImplementedError, self.scheduler.next_request)
with pytest.raises(NotImplementedError):
self.scheduler.has_pending_requests()
with pytest.raises(NotImplementedError):
self.scheduler.enqueue_request(Request("https://example.org"))
with pytest.raises(NotImplementedError):
self.scheduler.next_request()
class MinimalSchedulerTest(TestCase, InterfaceCheckMixin):
@ -89,15 +89,15 @@ class MinimalSchedulerTest(TestCase, InterfaceCheckMixin):
self.scheduler = MinimalScheduler()
def test_open_close(self):
with self.assertRaises(AttributeError):
with pytest.raises(AttributeError):
self.scheduler.open(Spider("foo"))
with self.assertRaises(AttributeError):
with pytest.raises(AttributeError):
self.scheduler.close("finished")
def test_len(self):
with self.assertRaises(AttributeError):
with pytest.raises(AttributeError):
self.scheduler.__len__()
with self.assertRaises(TypeError):
with pytest.raises(TypeError):
len(self.scheduler)
def test_enqueue_dequeue(self):

2
tests/test_selector.py
View File

@ -112,7 +112,7 @@ class SelectorTestCase(unittest.TestCase):
)
def test_selector_bad_args(self):
with self.assertRaisesRegex(ValueError, "received both response and text"):
with pytest.raises(ValueError, match="received both response and text"):
Selector(TextResponse(url="http://example.com", body=b""), text="")

37
tests/test_settings/__init__.py
View File

@ -235,9 +235,9 @@ class BaseSettingsTest(unittest.TestCase):
self.assertIn("key_highprio", settings)
del settings["key_highprio"]
self.assertNotIn("key_highprio", settings)
with self.assertRaises(KeyError):
with pytest.raises(KeyError):
settings.delete("notkey")
with self.assertRaises(KeyError):
with pytest.raises(KeyError):
del settings["notkey"]
def test_get(self):
@ -303,9 +303,19 @@ class BaseSettingsTest(unittest.TestCase):
self.assertEqual(settings.getdict("TEST_DICT2"), {"key1": "val1", "ke2": 3})
self.assertEqual(settings.getdict("TEST_DICT3"), {})
self.assertEqual(settings.getdict("TEST_DICT3", {"key1": 5}), {"key1": 5})
self.assertRaises(ValueError, settings.getdict, "TEST_LIST1")
self.assertRaises(ValueError, settings.getbool, "TEST_ENABLED_WRONG")
self.assertRaises(ValueError, settings.getbool, "TEST_DISABLED_WRONG")
with pytest.raises(
ValueError,
match="dictionary update sequence element #0 has length 3; 2 is required|sequence of pairs expected",
):
settings.getdict("TEST_LIST1")
with pytest.raises(
ValueError, match="Supported values for boolean settings are"
):
settings.getbool("TEST_ENABLED_WRONG")
with pytest.raises(
ValueError, match="Supported values for boolean settings are"
):
settings.getbool("TEST_DISABLED_WRONG")
def test_getpriority(self):
settings = BaseSettings({"key": "value"}, priority=99)
@ -381,11 +391,10 @@ class BaseSettingsTest(unittest.TestCase):
def test_freeze(self):
self.settings.freeze()
with self.assertRaises(TypeError) as cm:
with pytest.raises(
TypeError, match="Trying to modify an immutable Settings object"
):
self.settings.set("TEST_BOOL", False)
self.assertEqual(
str(cm.exception), "Trying to modify an immutable Settings object"
)
def test_frozencopy(self):
frozencopy = self.settings.frozencopy()
@ -476,7 +485,7 @@ class SettingsTest(unittest.TestCase):
def test_pop_item_with_default_value(self):
settings = Settings()
with self.assertRaises(KeyError):
with pytest.raises(KeyError):
settings.pop("DUMMY_CONFIG")
dummy_config_value = settings.pop("DUMMY_CONFIG", "dummy_value")
@ -491,9 +500,7 @@ class SettingsTest(unittest.TestCase):
settings.freeze()
with self.assertRaises(TypeError) as error:
with pytest.raises(
TypeError, match="Trying to modify an immutable Settings object"
):
settings.pop("OTHER_DUMMY_CONFIG")
self.assertEqual(
str(error.exception), "Trying to modify an immutable Settings object"
)

12
tests/test_spider.py
View File

@ -7,6 +7,7 @@ from pathlib import Path
from typing import Any
from unittest import mock
import pytest
from testfixtures import LogCapture
from twisted.internet.defer import inlineCallbacks
from twisted.trial import unittest
@ -57,8 +58,11 @@ class SpiderTest(unittest.TestCase):
def test_spider_without_name(self):
"""``__init__`` method arguments are assigned to spider attributes"""
self.assertRaises(ValueError, self.spider_class)
self.assertRaises(ValueError, self.spider_class, somearg="foo")
msg = "must have a name"
with pytest.raises(ValueError, match=msg):
self.spider_class()
with pytest.raises(ValueError, match=msg):
self.spider_class(somearg="foo")
def test_from_crawler_crawler_and_settings_population(self):
crawler = get_crawler()
@ -475,7 +479,7 @@ class CrawlSpiderTest(SpiderTest):
spider = self.spider_class("example.com")
spider.start_url = "https://www.example.com"
with self.assertRaisesRegex(AttributeError, r"^Crawling could not start.*$"):
with pytest.raises(AttributeError, match=r"^Crawling could not start.*$"):
list(spider.start_requests())
@ -825,5 +829,5 @@ class NoParseMethodSpiderTest(unittest.TestCase):
resp = TextResponse(url="http://www.example.com/random_url", body=text)
exc_msg = "Spider.parse callback is not defined"
with self.assertRaisesRegex(NotImplementedError, exc_msg):
with pytest.raises(NotImplementedError, match=exc_msg):
spider.parse(resp)

12
tests/test_spiderloader/__init__.py
View File

@ -7,6 +7,7 @@ from pathlib import Path
from tempfile import mkdtemp
from unittest import mock
import pytest
from twisted.trial import unittest
from zope.interface.verify import verifyObject
@ -124,9 +125,8 @@ class SpiderLoaderTest(unittest.TestCase):
}
)
self.assertRaisesRegex(
KeyError, "Spider not found", runner.create_crawler, "spider2"
)
with pytest.raises(KeyError, match="Spider not found"):
runner.create_crawler("spider2")
crawler = runner.create_crawler("spider1")
self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
@ -135,7 +135,8 @@ class SpiderLoaderTest(unittest.TestCase):
def test_bad_spider_modules_exception(self):
module = "tests.test_spiderloader.test_spiders.doesnotexist"
settings = Settings({"SPIDER_MODULES": [module]})
self.assertRaises(ImportError, SpiderLoader.from_settings, settings)
with pytest.raises(ImportError):
SpiderLoader.from_settings(settings)
def test_bad_spider_modules_warning(self):
with warnings.catch_warnings(record=True) as w:
@ -159,7 +160,8 @@ class SpiderLoaderTest(unittest.TestCase):
with mock.patch.object(SpiderLoader, "_load_spiders") as m:
m.side_effect = SyntaxError
settings = Settings({"SPIDER_MODULES": [module]})
self.assertRaises(SyntaxError, SpiderLoader.from_settings, settings)
with pytest.raises(SyntaxError):
SpiderLoader.from_settings(settings)
def test_syntax_error_warning(self):
with (

16
tests/test_spidermiddleware.py
View File

@ -3,6 +3,7 @@ from __future__ import annotations
from collections.abc import AsyncIterator, Iterable
from unittest import mock
import pytest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.python.failure import Failure
@ -299,12 +300,9 @@ class ProcessSpiderOutputCoroutineMiddleware:
class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
@defer.inlineCallbacks
def test_non_iterable(self):
with self.assertRaisesRegex(
with pytest.raises(
_InvalidOutput,
(
r"\.process_spider_output must return an iterable, got <class "
r"'NoneType'>"
),
match=r"\.process_spider_output must return an iterable, got <class 'NoneType'>",
):
yield self._get_middleware_result(
ProcessSpiderOutputNonIterableMiddleware,
@ -312,9 +310,9 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
@defer.inlineCallbacks
def test_coroutine(self):
with self.assertRaisesRegex(
with pytest.raises(
_InvalidOutput,
r"\.process_spider_output must be an asynchronous generator",
match=r"\.process_spider_output must be an asynchronous generator",
):
yield self._get_middleware_result(
ProcessSpiderOutputCoroutineMiddleware,
@ -518,8 +516,8 @@ class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase):
@defer.inlineCallbacks
def _test_asyncgen_nodowngrade(self, *mw_classes):
with self.assertRaisesRegex(
_InvalidOutput, "Async iterable returned from .+ cannot be downgraded"
with pytest.raises(
_InvalidOutput, match="Async iterable returned from .+ cannot be downgraded"
):
yield self._get_middleware_result(*mw_classes)

25
tests/test_spidermiddleware_httperror.py
View File

@ -1,6 +1,7 @@
import logging
from unittest import TestCase
import pytest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial.unittest import TestCase as TrialTestCase
@ -68,9 +69,8 @@ class TestHttpErrorMiddleware(TestCase):
def test_process_spider_input(self):
self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
self.assertRaises(
HttpError, self.mw.process_spider_input, self.res404, self.spider
)
with pytest.raises(HttpError):
self.mw.process_spider_input(self.res404, self.spider)
def test_process_spider_exception(self):
self.assertEqual(
@ -105,9 +105,8 @@ class TestHttpErrorMiddlewareSettings(TestCase):
def test_process_spider_input(self):
self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
self.assertRaises(
HttpError, self.mw.process_spider_input, self.res404, self.spider
)
with pytest.raises(HttpError):
self.mw.process_spider_input(self.res404, self.spider)
self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider))
def test_meta_overrides_settings(self):
@ -120,14 +119,14 @@ class TestHttpErrorMiddlewareSettings(TestCase):
res402.request = request
self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)
with pytest.raises(HttpError):
self.mw.process_spider_input(res402, self.spider)
def test_spider_override_settings(self):
self.spider.handle_httpstatus_list = [404]
self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
self.assertRaises(
HttpError, self.mw.process_spider_input, self.res402, self.spider
)
with pytest.raises(HttpError):
self.mw.process_spider_input(self.res402, self.spider)
class TestHttpErrorMiddlewareHandleAll(TestCase):
@ -151,7 +150,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
res402.request = request
self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)
with pytest.raises(HttpError):
self.mw.process_spider_input(res402, self.spider)
def test_httperror_allow_all_false(self):
crawler = get_crawler(_HttpErrorSpider)
@ -167,7 +167,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
res402 = self.res402.copy()
res402.request = request_httpstatus_true
self.assertRaises(HttpError, mw.process_spider_input, res404, self.spider)
with pytest.raises(HttpError):
mw.process_spider_input(res404, self.spider)
self.assertIsNone(mw.process_spider_input(res402, self.spider))

6
tests/test_spidermiddleware_referer.py
View File

@ -5,6 +5,8 @@ from typing import Any
from unittest import TestCase
from urllib.parse import urlparse
import pytest
from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
from scrapy.http import Request, Response
from scrapy.settings import Settings
@ -884,7 +886,7 @@ class TestSettingsPolicyByName(TestCase):
def test_invalid_name(self):
settings = Settings({"REFERRER_POLICY": "some-custom-unknown-policy"})
with self.assertRaises(RuntimeError):
with pytest.raises(RuntimeError):
RefererMiddleware(settings)
def test_multiple_policy_tokens(self):
@ -925,7 +927,7 @@ class TestSettingsPolicyByName(TestCase):
)
}
)
with self.assertRaises(RuntimeError):
with pytest.raises(RuntimeError):
RefererMiddleware(settings)

4
tests/test_spiderstate.py
View File

@ -2,6 +2,7 @@ import shutil
from datetime import datetime, timezone
from tempfile import mkdtemp
import pytest
from twisted.trial import unittest
from scrapy.exceptions import NotConfigured
@ -42,4 +43,5 @@ class SpiderStateTest(unittest.TestCase):
def test_not_configured(self):
crawler = get_crawler(Spider)
self.assertRaises(NotConfigured, SpiderState.from_crawler, crawler)
with pytest.raises(NotConfigured):
SpiderState.from_crawler(crawler)

12
tests/test_squeues.py
View File

@ -1,6 +1,7 @@
import pickle
import sys
import pytest
from queuelib.tests import test_queue as t
from scrapy.http import Request
@ -30,10 +31,17 @@ class MyLoader(ItemLoader):
def nonserializable_object_test(self):
q = self.queue()
self.assertRaises(ValueError, q.push, lambda x: x)
with pytest.raises(
ValueError,
match="unmarshallable object|Can't (get|pickle) local object|Can't pickle .*: it's not found as",
):
q.push(lambda x: x)
# Selectors should fail (lxml.html.HtmlElement objects can't be pickled)
sel = Selector(text="<html><body><p>some text</p></body></html>")
self.assertRaises(ValueError, q.push, sel)
with pytest.raises(
ValueError, match="unmarshallable object|can't pickle Selector objects"
):
q.push(sel)
class FifoDiskQueueTestMixin:

13
tests/test_squeues_request.py
View File

@ -6,6 +6,7 @@ import shutil
import tempfile
import unittest
import pytest
import queuelib
from scrapy.http import Request
@ -69,9 +70,9 @@ class RequestQueueTestMixin:
req = Request("http://www.example.com")
q.push(req)
self.assertEqual(len(q), 1)
with self.assertRaises(
with pytest.raises(
NotImplementedError,
msg="The underlying queue class does not implement 'peek'",
match="The underlying queue class does not implement 'peek'",
):
q.peek()
self.assertEqual(q.pop().url, req.url)
@ -120,9 +121,9 @@ class FifoQueueMixin(RequestQueueTestMixin):
q.push(req1)
q.push(req2)
q.push(req3)
with self.assertRaises(
with pytest.raises(
NotImplementedError,
msg="The underlying queue class does not implement 'peek'",
match="The underlying queue class does not implement 'peek'",
):
q.peek()
self.assertEqual(len(q), 3)
@ -176,9 +177,9 @@ class LifoQueueMixin(RequestQueueTestMixin):
q.push(req1)
q.push(req2)
q.push(req3)
with self.assertRaises(
with pytest.raises(
NotImplementedError,
msg="The underlying queue class does not implement 'peek'",
match="The underlying queue class does not implement 'peek'",
):
q.peek()
self.assertEqual(len(q), 3)

28
tests/test_utils_conf.py
View File

@ -1,5 +1,7 @@
import unittest
import pytest
from scrapy.exceptions import UsageError
from scrapy.settings import BaseSettings, Settings
from scrapy.utils.conf import (
@ -32,7 +34,9 @@ class BuildComponentListTest(unittest.TestCase):
)
# Same priority raises ValueError
duplicate_bs.set("ONE", duplicate_bs["ONE"], priority=20)
with self.assertRaises(ValueError):
with pytest.raises(
ValueError, match="Some paths in .* convert to the same object"
):
build_component_list(duplicate_bs, convert=lambda x: x.lower())
def test_valid_numbers(self):
@ -58,21 +62,13 @@ class UtilsConfTestCase(unittest.TestCase):
class FeedExportConfigTestCase(unittest.TestCase):
def test_feed_export_config_invalid_format(self):
settings = Settings()
self.assertRaises(
UsageError,
feed_process_params_from_cli,
settings,
["items.dat"],
)
with pytest.raises(UsageError):
feed_process_params_from_cli(settings, ["items.dat"])
def test_feed_export_config_mismatch(self):
settings = Settings()
self.assertRaises(
UsageError,
feed_process_params_from_cli,
settings,
["items1.dat", "items2.dat"],
)
with pytest.raises(UsageError):
feed_process_params_from_cli(settings, ["items1.dat", "items2.dat"])
def test_feed_export_config_explicit_formats(self):
settings = Settings()
@ -117,11 +113,9 @@ class FeedExportConfigTestCase(unittest.TestCase):
)
def test_output_and_overwrite_output(self):
with self.assertRaises(UsageError):
with pytest.raises(UsageError):
feed_process_params_from_cli(
Settings(),
["output1.json"],
overwrite_output=["output2.json"],
Settings(), ["output1.json"], overwrite_output=["output2.json"]
)
def test_feed_complete_default_values_from_settings_empty(self):

24
tests/test_utils_curl.py
View File

@ -1,6 +1,7 @@
import unittest
import warnings
import pytest
from w3lib.http import basic_auth_header
from scrapy import Request
@ -205,11 +206,11 @@ class CurlToRequestKwargsTest(unittest.TestCase):
self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
def test_too_few_arguments_error(self):
self.assertRaisesRegex(
with pytest.raises(
ValueError,
r"too few arguments|the following arguments are required:\s*url",
lambda: curl_to_request_kwargs("curl"),
)
match=r"too few arguments|the following arguments are required:\s*url",
):
curl_to_request_kwargs("curl")
def test_ignore_unknown_options(self):
# case 1: ignore_unknown_options=True:
@ -220,16 +221,11 @@ class CurlToRequestKwargsTest(unittest.TestCase):
self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
# case 2: ignore_unknown_options=False (raise exception):
self.assertRaisesRegex(
ValueError,
"Unrecognized options:.*--bar.*--baz",
lambda: curl_to_request_kwargs(
with pytest.raises(ValueError, match="Unrecognized options:.*--bar.*--baz"):
curl_to_request_kwargs(
"curl --bar --baz http://www.example.com", ignore_unknown_options=False
),
)
)
def test_must_start_with_curl_error(self):
self.assertRaises(
ValueError,
lambda: curl_to_request_kwargs("carl -X POST http://example.org"),
)
with pytest.raises(ValueError, match="A curl command must start"):
curl_to_request_kwargs("carl -X POST http://example.org")

13
tests/test_utils_datatypes.py
View File

@ -87,8 +87,10 @@ class CaseInsensitiveDictMixin:
def test_delete(self):
d = self.dict_class({"key_lower": 1})
del d["key_LOWER"]
self.assertRaises(KeyError, d.__getitem__, "key_LOWER")
self.assertRaises(KeyError, d.__getitem__, "key_lower")
with pytest.raises(KeyError):
d["key_LOWER"]
with pytest.raises(KeyError):
d["key_lower"]
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_getdefault(self):
@ -138,7 +140,8 @@ class CaseInsensitiveDictMixin:
d = self.dict_class()
d["a"] = 1
self.assertEqual(d.pop("A"), 1)
self.assertRaises(KeyError, d.pop, "A")
with pytest.raises(KeyError):
d.pop("A")
def test_normkey(self):
class MyDict(self.dict_class):
@ -279,8 +282,8 @@ class SequenceExcludeTest(unittest.TestCase):
self.assertIn(set("bar"), d)
# supplied sequence is a set, so checking for list (non)inclusion fails
self.assertRaises(TypeError, (0, 1, 2) in d)
self.assertRaises(TypeError, d.__contains__, ["a", "b", "c"])
with pytest.raises(TypeError):
["a", "b", "c"] in d # noqa: B015
for v in [-3, "test", 1.1]:
self.assertNotIn(v, d)

5
tests/test_utils_deprecate.py
View File

@ -3,6 +3,8 @@ import unittest
import warnings
from unittest import mock
import pytest
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.deprecate import create_deprecated_class, update_classpath
@ -181,7 +183,8 @@ class WarnWhenSubclassedTest(unittest.TestCase):
assert not issubclass(OutdatedUserClass1, OutdatedUserClass1a)
assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1)
self.assertRaises(TypeError, issubclass, object(), DeprecatedName)
with pytest.raises(TypeError):
issubclass(object(), DeprecatedName)
def test_isinstance(self):
with warnings.catch_warnings():

7
tests/test_utils_gz.py
View File

@ -1,6 +1,8 @@
import unittest
from gzip import BadGzipFile
from pathlib import Path
import pytest
from w3lib.encoding import html_to_unicode
from scrapy.http import Response
@ -27,9 +29,8 @@ class GunzipTest(unittest.TestCase):
assert text.endswith(b"</html")
def test_gunzip_no_gzip_file_raises(self):
self.assertRaises(
OSError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes()
)
with pytest.raises(BadGzipFile):
gunzip((SAMPLEDIR / "feed-sample1.xml").read_bytes())
def test_gunzip_truncated_short(self):
r1 = Response(

16
tests/test_utils_iterators.py
View File

@ -215,7 +215,7 @@ class XmliterBaseTestCase:
"""
response = XmlResponse(url="http://mydummycompany.com", body=body)
my_iter = self.xmliter(response, "g:link_image")
with self.assertRaises(StopIteration):
with pytest.raises(StopIteration):
next(my_iter)
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
@ -228,13 +228,14 @@ class XmliterBaseTestCase:
iter = self.xmliter(body, "product")
next(iter)
next(iter)
self.assertRaises(StopIteration, next, iter)
with pytest.raises(StopIteration):
next(iter)
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_objtype_exception(self):
i = self.xmliter(42, "product")
self.assertRaises(TypeError, next, i)
with pytest.raises(TypeError):
next(i)
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_encoding(self):
@ -344,7 +345,8 @@ class LxmlXmliterTestCase(XmliterBaseTestCase, unittest.TestCase):
def test_xmliter_objtype_exception(self):
i = self.xmliter(42, "product")
self.assertRaises(TypeError, next, i)
with pytest.raises(TypeError):
next(i)
class UtilsCsvTestCase(unittest.TestCase):
@ -491,8 +493,8 @@ class UtilsCsvTestCase(unittest.TestCase):
next(iter)
next(iter)
next(iter)
self.assertRaises(StopIteration, next, iter)
with pytest.raises(StopIteration):
next(iter)
def test_csviter_encoding(self):
body1 = get_testdata("feeds", "feed-sample4.csv")

20
tests/test_utils_misc/__init__.py
View File

@ -32,9 +32,12 @@ class UtilsMiscTestCase(unittest.TestCase):
self.assertIs(obj, load_object)
def test_load_object_exceptions(self):
self.assertRaises(ImportError, load_object, "nomodule999.mod.function")
self.assertRaises(NameError, load_object, "scrapy.utils.misc.load_object999")
self.assertRaises(TypeError, load_object, {})
with pytest.raises(ImportError):
load_object("nomodule999.mod.function")
with pytest.raises(NameError):
load_object("scrapy.utils.misc.load_object999")
with pytest.raises(TypeError):
load_object({})
def test_walk_modules(self):
mods = walk_modules("tests.test_utils_misc.test_walk_modules")
@ -59,7 +62,8 @@ class UtilsMiscTestCase(unittest.TestCase):
]
self.assertEqual({m.__name__ for m in mods}, set(expected))
self.assertRaises(ImportError, walk_modules, "nomodule999")
with pytest.raises(ImportError):
walk_modules("nomodule999")
def test_walk_modules_egg(self):
egg = str(Path(__file__).parent / "test.egg")
@ -148,11 +152,13 @@ class UtilsMiscTestCase(unittest.TestCase):
create_instance(m, None, crawler, *args, **kwargs)
m.from_settings.assert_called_once_with(crawler.settings, *args, **kwargs)
with self.assertRaises(ValueError):
with pytest.raises(
ValueError, match="Specify at least one of settings and crawler"
):
create_instance(m, None, None)
m.from_settings.return_value = None
with self.assertRaises(TypeError):
with pytest.raises(TypeError):
create_instance(m, settings, None)
def test_build_from_crawler(self):
@ -191,7 +197,7 @@ class UtilsMiscTestCase(unittest.TestCase):
# Check adoption of crawler
m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"])
m.from_crawler.return_value = None
with self.assertRaises(TypeError):
with pytest.raises(TypeError):
build_from_crawler(m, crawler, *args, **kwargs)
def test_set_environ(self):

6
tests/test_utils_python.py
View File

@ -87,7 +87,8 @@ class ToUnicodeTest(unittest.TestCase):
self.assertEqual(to_unicode("\xf1e\xf1e\xf1e"), "\xf1e\xf1e\xf1e")
def test_converting_a_strange_object_should_raise_TypeError(self):
self.assertRaises(TypeError, to_unicode, 423)
with pytest.raises(TypeError):
to_unicode(423)
def test_errors_argument(self):
self.assertEqual(to_unicode(b"a\xedb", "utf-8", errors="replace"), "a\ufffdb")
@ -104,7 +105,8 @@ class ToBytesTest(unittest.TestCase):
self.assertEqual(to_bytes(b"lel\xf1e"), b"lel\xf1e")
def test_converting_a_strange_object_should_raise_TypeError(self):
self.assertRaises(TypeError, to_bytes, unittest)
with pytest.raises(TypeError):
to_bytes(pytest)
def test_errors_argument(self):
self.assertEqual(to_bytes("a\ufffdb", "latin-1", errors="replace"), b"a?b")

3
tests/test_utils_response.py
View File

@ -35,7 +35,8 @@ class ResponseUtilsTest(unittest.TestCase):
assert open_in_browser(response, _openfunc=browser_open), "Browser not called"
resp = Response(url, body=body)
self.assertRaises(TypeError, open_in_browser, resp, debug=True)
with pytest.raises(TypeError):
open_in_browser(resp, debug=True) # pylint: disable=unexpected-keyword-arg
def test_get_meta_refresh(self):
r1 = HtmlResponse(