diff --git a/tests/test_spider.py b/tests/test_spider.py
index af29872a8..05f1c59d0 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -31,7 +31,7 @@ from scrapy.utils.test import get_crawler
from tests import get_testdata, tests_datadir
-class SpiderTest(unittest.TestCase):
+class TestSpider(unittest.TestCase):
spider_class = Spider
def setUp(self):
@@ -42,19 +42,19 @@ class SpiderTest(unittest.TestCase):
def test_base_spider(self):
spider = self.spider_class("example.com")
- self.assertEqual(spider.name, "example.com")
- self.assertEqual(spider.start_urls, [])
+ assert spider.name == "example.com"
+ assert spider.start_urls == [] # pylint: disable=use-implicit-booleaness-not-comparison
def test_start_requests(self):
spider = self.spider_class("example.com")
start_requests = spider.start_requests()
- self.assertTrue(inspect.isgenerator(start_requests))
- self.assertEqual(list(start_requests), [])
+ assert inspect.isgenerator(start_requests)
+ assert not list(start_requests)
def test_spider_args(self):
"""``__init__`` method arguments are assigned to spider attributes"""
spider = self.spider_class("example.com", foo="bar")
- self.assertEqual(spider.foo, "bar")
+ assert spider.foo == "bar"
def test_spider_without_name(self):
"""``__init__`` method arguments are assigned to spider attributes"""
@@ -67,10 +67,10 @@ class SpiderTest(unittest.TestCase):
def test_from_crawler_crawler_and_settings_population(self):
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, "example.com")
- self.assertTrue(hasattr(spider, "crawler"))
- self.assertIs(spider.crawler, crawler)
- self.assertTrue(hasattr(spider, "settings"))
- self.assertIs(spider.settings, crawler.settings)
+ assert hasattr(spider, "crawler")
+ assert spider.crawler is crawler
+ assert hasattr(spider, "settings")
+ assert spider.settings is crawler.settings
def test_from_crawler_init_call(self):
with mock.patch.object(
@@ -92,7 +92,7 @@ class SpiderTest(unittest.TestCase):
crawler.signals.send_catch_log(
signal=signals.spider_closed, spider=spider, reason=None
)
- self.assertTrue(spider.closed_called)
+ assert spider.closed_called
def test_update_settings(self):
spider_settings = {"TEST1": "spider", "TEST2": "spider"}
@@ -101,9 +101,9 @@ class SpiderTest(unittest.TestCase):
settings = Settings(project_settings, priority="project")
self.spider_class.update_settings(settings)
- self.assertEqual(settings.get("TEST1"), "spider")
- self.assertEqual(settings.get("TEST2"), "spider")
- self.assertEqual(settings.get("TEST3"), "project")
+ assert settings.get("TEST1") == "spider"
+ assert settings.get("TEST2") == "spider"
+ assert settings.get("TEST3") == "project"
@inlineCallbacks
def test_settings_in_from_crawler(self):
@@ -121,11 +121,11 @@ class SpiderTest(unittest.TestCase):
return spider
crawler = Crawler(TestSpider, project_settings)
- self.assertEqual(crawler.settings.get("TEST1"), "spider")
- self.assertEqual(crawler.settings.get("TEST2"), "spider")
- self.assertEqual(crawler.settings.get("TEST3"), "project")
+ assert crawler.settings.get("TEST1") == "spider"
+ assert crawler.settings.get("TEST2") == "spider"
+ assert crawler.settings.get("TEST3") == "project"
yield crawler.crawl()
- self.assertEqual(crawler.settings.get("TEST1"), "spider_instance")
+ assert crawler.settings.get("TEST1") == "spider_instance"
def test_logger(self):
spider = self.spider_class("example.com")
@@ -134,8 +134,8 @@ class SpiderTest(unittest.TestCase):
lc.check(("example.com", "INFO", "test log msg"))
record = lc.records[0]
- self.assertIn("spider", record.__dict__)
- self.assertIs(record.spider, spider)
+ assert "spider" in record.__dict__
+ assert record.spider is spider
def test_log(self):
spider = self.spider_class("example.com")
@@ -144,11 +144,11 @@ class SpiderTest(unittest.TestCase):
mock_logger.log.assert_called_once_with("INFO", "test log msg")
-class InitSpiderTest(SpiderTest):
+class TestInitSpider(TestSpider):
spider_class = InitSpider
-class XMLFeedSpiderTest(SpiderTest):
+class TestXMLFeedSpider(TestSpider):
spider_class = XMLFeedSpider
def test_register_namespace(self):
@@ -180,28 +180,24 @@ class XMLFeedSpiderTest(SpiderTest):
for iterator in ("iternodes", "xml"):
spider = _XMLSpider("example", iterator=iterator)
output = list(spider._parse(response))
- self.assertEqual(len(output), 2, iterator)
- self.assertEqual(
- output,
- [
- {
- "loc": ["http://www.example.com/Special-Offers.html"],
- "updated": ["2009-08-16"],
- "custom": ["fuu"],
- "other": ["bar"],
- },
- {
- "loc": [],
- "updated": ["2009-08-16"],
- "other": ["foo"],
- "custom": [],
- },
- ],
- iterator,
- )
+ assert len(output) == 2, iterator
+ assert output == [
+ {
+ "loc": ["http://www.example.com/Special-Offers.html"],
+ "updated": ["2009-08-16"],
+ "custom": ["fuu"],
+ "other": ["bar"],
+ },
+ {
+ "loc": [],
+ "updated": ["2009-08-16"],
+ "other": ["foo"],
+ "custom": [],
+ },
+ ], iterator
-class CSVFeedSpiderTest(SpiderTest):
+class TestCSVFeedSpider(TestSpider):
spider_class = CSVFeedSpider
def test_parse_rows(self):
@@ -222,7 +218,7 @@ class CSVFeedSpiderTest(SpiderTest):
assert len(rows) == 4
-class CrawlSpiderTest(SpiderTest):
+class TestCrawlSpider(TestSpider):
test_body = b"""
Page title
Item 12
@@ -247,16 +243,13 @@ class CrawlSpiderTest(SpiderTest):
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
- self.assertEqual(len(output), 3)
- self.assertTrue(all(isinstance(r, Request) for r in output))
- self.assertEqual(
- [r.url for r in output],
- [
- "http://example.org/somepage/item/12.html",
- "http://example.org/about.html",
- "http://example.org/nofollow.html",
- ],
- )
+ assert len(output) == 3
+ assert all(isinstance(r, Request) for r in output)
+ assert [r.url for r in output] == [
+ "http://example.org/somepage/item/12.html",
+ "http://example.org/about.html",
+ "http://example.org/nofollow.html",
+ ]
def test_process_links(self):
response = HtmlResponse(
@@ -273,16 +266,13 @@ class CrawlSpiderTest(SpiderTest):
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
- self.assertEqual(len(output), 3)
- self.assertTrue(all(isinstance(r, Request) for r in output))
- self.assertEqual(
- [r.url for r in output],
- [
- "http://example.org/somepage/item/12.html",
- "http://example.org/about.html",
- "http://example.org/nofollow.html",
- ],
- )
+ assert len(output) == 3
+ assert all(isinstance(r, Request) for r in output)
+ assert [r.url for r in output] == [
+ "http://example.org/somepage/item/12.html",
+ "http://example.org/about.html",
+ "http://example.org/nofollow.html",
+ ]
def test_process_links_filter(self):
response = HtmlResponse(
@@ -302,15 +292,12 @@ class CrawlSpiderTest(SpiderTest):
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
- self.assertEqual(len(output), 2)
- self.assertTrue(all(isinstance(r, Request) for r in output))
- self.assertEqual(
- [r.url for r in output],
- [
- "http://example.org/somepage/item/12.html",
- "http://example.org/about.html",
- ],
- )
+ assert len(output) == 2
+ assert all(isinstance(r, Request) for r in output)
+ assert [r.url for r in output] == [
+ "http://example.org/somepage/item/12.html",
+ "http://example.org/about.html",
+ ]
def test_process_links_generator(self):
response = HtmlResponse(
@@ -327,16 +314,13 @@ class CrawlSpiderTest(SpiderTest):
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
- self.assertEqual(len(output), 3)
- self.assertTrue(all(isinstance(r, Request) for r in output))
- self.assertEqual(
- [r.url for r in output],
- [
- "http://example.org/somepage/item/12.html",
- "http://example.org/about.html",
- "http://example.org/nofollow.html",
- ],
- )
+ assert len(output) == 3
+ assert all(isinstance(r, Request) for r in output)
+ assert [r.url for r in output] == [
+ "http://example.org/somepage/item/12.html",
+ "http://example.org/about.html",
+ "http://example.org/nofollow.html",
+ ]
def test_process_request(self):
response = HtmlResponse(
@@ -355,16 +339,13 @@ class CrawlSpiderTest(SpiderTest):
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
- self.assertEqual(len(output), 3)
- self.assertTrue(all(isinstance(r, Request) for r in output))
- self.assertEqual(
- [r.url for r in output],
- [
- "http://example.com/somepage/item/12.html",
- "http://example.com/about.html",
- "http://example.com/nofollow.html",
- ],
- )
+ assert len(output) == 3
+ assert all(isinstance(r, Request) for r in output)
+ assert [r.url for r in output] == [
+ "http://example.com/somepage/item/12.html",
+ "http://example.com/about.html",
+ "http://example.com/nofollow.html",
+ ]
def test_process_request_with_response(self):
response = HtmlResponse(
@@ -386,20 +367,18 @@ class CrawlSpiderTest(SpiderTest):
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
- self.assertEqual(len(output), 3)
- self.assertTrue(all(isinstance(r, Request) for r in output))
- self.assertEqual(
- [r.url for r in output],
- [
- "http://example.org/somepage/item/12.html",
- "http://example.org/about.html",
- "http://example.org/nofollow.html",
- ],
- )
- self.assertEqual(
- [r.meta["response_class"] for r in output],
- ["HtmlResponse", "HtmlResponse", "HtmlResponse"],
- )
+ assert len(output) == 3
+ assert all(isinstance(r, Request) for r in output)
+ assert [r.url for r in output] == [
+ "http://example.org/somepage/item/12.html",
+ "http://example.org/about.html",
+ "http://example.org/nofollow.html",
+ ]
+ assert [r.meta["response_class"] for r in output] == [
+ "HtmlResponse",
+ "HtmlResponse",
+ "HtmlResponse",
+ ]
def test_process_request_instance_method(self):
response = HtmlResponse(
@@ -416,16 +395,13 @@ class CrawlSpiderTest(SpiderTest):
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
- self.assertEqual(len(output), 3)
- self.assertTrue(all(isinstance(r, Request) for r in output))
- self.assertEqual(
- [r.url for r in output],
- [
- safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"),
- safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"),
- safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"),
- ],
- )
+ assert len(output) == 3
+ assert all(isinstance(r, Request) for r in output)
+ assert [r.url for r in output] == [
+ safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"),
+ safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"),
+ safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"),
+ ]
def test_process_request_instance_method_with_response(self):
response = HtmlResponse(
@@ -448,32 +424,30 @@ class CrawlSpiderTest(SpiderTest):
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
- self.assertEqual(len(output), 3)
- self.assertTrue(all(isinstance(r, Request) for r in output))
- self.assertEqual(
- [r.url for r in output],
- [
- "http://example.org/somepage/item/12.html",
- "http://example.org/about.html",
- "http://example.org/nofollow.html",
- ],
- )
- self.assertEqual(
- [r.meta["response_class"] for r in output],
- ["HtmlResponse", "HtmlResponse", "HtmlResponse"],
- )
+ assert len(output) == 3
+ assert all(isinstance(r, Request) for r in output)
+ assert [r.url for r in output] == [
+ "http://example.org/somepage/item/12.html",
+ "http://example.org/about.html",
+ "http://example.org/nofollow.html",
+ ]
+ assert [r.meta["response_class"] for r in output] == [
+ "HtmlResponse",
+ "HtmlResponse",
+ "HtmlResponse",
+ ]
def test_follow_links_attribute_population(self):
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, "example.com")
- self.assertTrue(hasattr(spider, "_follow_links"))
- self.assertTrue(spider._follow_links)
+ assert hasattr(spider, "_follow_links")
+ assert spider._follow_links
settings_dict = {"CRAWLSPIDER_FOLLOW_LINKS": False}
crawler = get_crawler(settings_dict=settings_dict)
spider = self.spider_class.from_crawler(crawler, "example.com")
- self.assertTrue(hasattr(spider, "_follow_links"))
- self.assertFalse(spider._follow_links)
+ assert hasattr(spider, "_follow_links")
+ assert not spider._follow_links
def test_start_url(self):
spider = self.spider_class("example.com")
@@ -483,7 +457,7 @@ class CrawlSpiderTest(SpiderTest):
list(spider.start_requests())
-class SitemapSpiderTest(SpiderTest):
+class TestSitemapSpider(TestSpider):
spider_class = SitemapSpider
BODY = b"SITEMAP"
@@ -496,7 +470,7 @@ class SitemapSpiderTest(SpiderTest):
def assertSitemapBody(self, response, body):
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, "example.com")
- self.assertEqual(spider._get_sitemap_body(response), body)
+ assert spider._get_sitemap_body(response) == body
def test_get_sitemap_body(self):
r = XmlResponse(url="http://www.example.com/", body=self.BODY)
@@ -543,15 +517,12 @@ Sitemap: /sitemap-relative-url.xml
r = TextResponse(url="http://www.example.com/robots.txt", body=robots)
spider = self.spider_class("example.com")
- self.assertEqual(
- [req.url for req in spider._parse_sitemap(r)],
- [
- "http://example.com/sitemap.xml",
- "http://example.com/sitemap-product-index.xml",
- "http://example.com/sitemap-uppercase.xml",
- "http://www.example.com/sitemap-relative-url.xml",
- ],
- )
+ assert [req.url for req in spider._parse_sitemap(r)] == [
+ "http://example.com/sitemap.xml",
+ "http://example.com/sitemap-product-index.xml",
+ "http://example.com/sitemap-uppercase.xml",
+ "http://www.example.com/sitemap-relative-url.xml",
+ ]
def test_alternate_url_locs(self):
sitemap = b"""
@@ -570,21 +541,17 @@ Sitemap: /sitemap-relative-url.xml
"""
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
spider = self.spider_class("example.com")
- self.assertEqual(
- [req.url for req in spider._parse_sitemap(r)],
- ["http://www.example.com/english/"],
- )
+ assert [req.url for req in spider._parse_sitemap(r)] == [
+ "http://www.example.com/english/"
+ ]
spider.sitemap_alternate_links = True
- self.assertEqual(
- [req.url for req in spider._parse_sitemap(r)],
- [
- "http://www.example.com/english/",
- "http://www.example.com/deutsch/",
- "http://www.example.com/schweiz-deutsch/",
- "http://www.example.com/italiano/",
- ],
- )
+ assert [req.url for req in spider._parse_sitemap(r)] == [
+ "http://www.example.com/english/",
+ "http://www.example.com/deutsch/",
+ "http://www.example.com/schweiz-deutsch/",
+ "http://www.example.com/italiano/",
+ ]
def test_sitemap_filter(self):
sitemap = b"""
@@ -611,16 +578,15 @@ Sitemap: /sitemap-relative-url.xml
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
spider = self.spider_class("example.com")
- self.assertEqual(
- [req.url for req in spider._parse_sitemap(r)],
- ["http://www.example.com/english/", "http://www.example.com/portuguese/"],
- )
+ assert [req.url for req in spider._parse_sitemap(r)] == [
+ "http://www.example.com/english/",
+ "http://www.example.com/portuguese/",
+ ]
spider = FilteredSitemapSpider("example.com")
- self.assertEqual(
- [req.url for req in spider._parse_sitemap(r)],
- ["http://www.example.com/english/"],
- )
+ assert [req.url for req in spider._parse_sitemap(r)] == [
+ "http://www.example.com/english/"
+ ]
def test_sitemap_filter_with_alternate_links(self):
sitemap = b"""
@@ -649,19 +615,15 @@ Sitemap: /sitemap-relative-url.xml
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
spider = self.spider_class("example.com")
- self.assertEqual(
- [req.url for req in spider._parse_sitemap(r)],
- [
- "http://www.example.com/english/article_1/",
- "http://www.example.com/english/article_2/",
- ],
- )
+ assert [req.url for req in spider._parse_sitemap(r)] == [
+ "http://www.example.com/english/article_1/",
+ "http://www.example.com/english/article_2/",
+ ]
spider = FilteredSitemapSpider("example.com")
- self.assertEqual(
- [req.url for req in spider._parse_sitemap(r)],
- ["http://www.example.com/deutsch/article_1/"],
- )
+ assert [req.url for req in spider._parse_sitemap(r)] == [
+ "http://www.example.com/deutsch/article_1/"
+ ]
def test_sitemapindex_filter(self):
sitemap = b"""
@@ -689,19 +651,15 @@ Sitemap: /sitemap-relative-url.xml
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
spider = self.spider_class("example.com")
- self.assertEqual(
- [req.url for req in spider._parse_sitemap(r)],
- [
- "http://www.example.com/sitemap1.xml",
- "http://www.example.com/sitemap2.xml",
- ],
- )
+ assert [req.url for req in spider._parse_sitemap(r)] == [
+ "http://www.example.com/sitemap1.xml",
+ "http://www.example.com/sitemap2.xml",
+ ]
spider = FilteredSitemapSpider("example.com")
- self.assertEqual(
- [req.url for req in spider._parse_sitemap(r)],
- ["http://www.example.com/sitemap2.xml"],
- )
+ assert [req.url for req in spider._parse_sitemap(r)] == [
+ "http://www.example.com/sitemap2.xml"
+ ]
def test_compression_bomb_setting(self):
settings = {"DOWNLOAD_MAXSIZE": 10_000_000}
@@ -711,7 +669,7 @@ Sitemap: /sitemap-relative-url.xml
body = body_path.read_bytes()
request = Request(url="https://example.com")
response = Response(url="https://example.com", body=body, request=request)
- self.assertIsNone(spider._get_sitemap_body(response))
+ assert spider._get_sitemap_body(response) is None
def test_compression_bomb_spider_attr(self):
class DownloadMaxSizeSpider(self.spider_class):
@@ -723,7 +681,7 @@ Sitemap: /sitemap-relative-url.xml
body = body_path.read_bytes()
request = Request(url="https://example.com")
response = Response(url="https://example.com", body=body, request=request)
- self.assertIsNone(spider._get_sitemap_body(response))
+ assert spider._get_sitemap_body(response) is None
def test_compression_bomb_request_meta(self):
crawler = get_crawler()
@@ -734,7 +692,7 @@ Sitemap: /sitemap-relative-url.xml
url="https://example.com", meta={"download_maxsize": 10_000_000}
)
response = Response(url="https://example.com", body=body, request=request)
- self.assertIsNone(spider._get_sitemap_body(response))
+ assert spider._get_sitemap_body(response) is None
def test_download_warnsize_setting(self):
settings = {"DOWNLOAD_WARNSIZE": 10_000_000}
@@ -814,13 +772,13 @@ Sitemap: /sitemap-relative-url.xml
)
-class DeprecationTest(unittest.TestCase):
+class TestDeprecation:
def test_crawl_spider(self):
assert issubclass(CrawlSpider, Spider)
assert isinstance(CrawlSpider(name="foo"), Spider)
-class NoParseMethodSpiderTest(unittest.TestCase):
+class TestNoParseMethodSpider:
spider_class = Spider
def test_undefined_parse_method(self):
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index b103e9ed0..476487a04 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -8,7 +8,6 @@ from tempfile import mkdtemp
from unittest import mock
import pytest
-from twisted.trial import unittest
from zope.interface.verify import verifyObject
# ugly hack to avoid cyclic imports of scrapy.spiders when running this test
@@ -28,8 +27,8 @@ def _copytree(source: Path, target: Path):
shutil.copytree(source, target)
-class SpiderLoaderTest(unittest.TestCase):
- def setUp(self):
+class TestSpiderLoader:
+ def setup_method(self):
orig_spiders_dir = module_dir / "test_spiders"
self.tmpdir = Path(tempfile.mkdtemp())
self.spiders_dir = self.tmpdir / "test_spiders_xxx"
@@ -38,7 +37,7 @@ class SpiderLoaderTest(unittest.TestCase):
settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})
self.spider_loader = SpiderLoader.from_settings(settings)
- def tearDown(self):
+ def teardown_method(self):
del self.spider_loader
del sys.modules["test_spiders_xxx"]
sys.path.remove(str(self.tmpdir))
@@ -47,37 +46,35 @@ class SpiderLoaderTest(unittest.TestCase):
verifyObject(ISpiderLoader, self.spider_loader)
def test_list(self):
- self.assertEqual(
- set(self.spider_loader.list()), {"spider1", "spider2", "spider3", "spider4"}
- )
+ assert set(self.spider_loader.list()) == {
+ "spider1",
+ "spider2",
+ "spider3",
+ "spider4",
+ }
def test_load(self):
spider1 = self.spider_loader.load("spider1")
- self.assertEqual(spider1.__name__, "Spider1")
+ assert spider1.__name__ == "Spider1"
def test_find_by_request(self):
- self.assertEqual(
- self.spider_loader.find_by_request(Request("http://scrapy1.org/test")),
- ["spider1"],
- )
- self.assertEqual(
- self.spider_loader.find_by_request(Request("http://scrapy2.org/test")),
- ["spider2"],
- )
- self.assertEqual(
- set(self.spider_loader.find_by_request(Request("http://scrapy3.org/test"))),
- {"spider1", "spider2"},
- )
- self.assertEqual(
- self.spider_loader.find_by_request(Request("http://scrapy999.org/test")), []
- )
- self.assertEqual(
- self.spider_loader.find_by_request(Request("http://spider3.com")), []
- )
- self.assertEqual(
- self.spider_loader.find_by_request(Request("http://spider3.com/onlythis")),
- ["spider3"],
+ assert self.spider_loader.find_by_request(
+ Request("http://scrapy1.org/test")
+ ) == ["spider1"]
+ assert self.spider_loader.find_by_request(
+ Request("http://scrapy2.org/test")
+ ) == ["spider2"]
+ assert set(
+ self.spider_loader.find_by_request(Request("http://scrapy3.org/test"))
+ ) == {"spider1", "spider2"}
+ assert (
+ self.spider_loader.find_by_request(Request("http://scrapy999.org/test"))
+ == []
)
+ assert self.spider_loader.find_by_request(Request("http://spider3.com")) == []
+ assert self.spider_loader.find_by_request(
+ Request("http://spider3.com/onlythis")
+ ) == ["spider3"]
def test_load_spider_module(self):
module = "tests.test_spiderloader.test_spiders.spider1"
@@ -113,9 +110,9 @@ class SpiderLoaderTest(unittest.TestCase):
runner = CrawlerRunner({"ADDONS": {SpiderModuleAddon: 1}})
crawler = runner.create_crawler("spider_from_addon")
- self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
- self.assertEqual(crawler.spidercls.name, "spider_from_addon")
- self.assertTrue(len(crawler.settings["SPIDER_MODULES"]) == 1)
+ assert issubclass(crawler.spidercls, scrapy.Spider)
+ assert crawler.spidercls.name == "spider_from_addon"
+ assert len(crawler.settings["SPIDER_MODULES"]) == 1
def test_crawler_runner_loading(self):
module = "tests.test_spiderloader.test_spiders.spider1"
@@ -129,8 +126,8 @@ class SpiderLoaderTest(unittest.TestCase):
runner.create_crawler("spider2")
crawler = runner.create_crawler("spider1")
- self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
- self.assertEqual(crawler.spidercls.name, "spider1")
+ assert issubclass(crawler.spidercls, scrapy.Spider)
+ assert crawler.spidercls.name == "spider1"
def test_bad_spider_modules_exception(self):
module = "tests.test_spiderloader.test_spiders.doesnotexist"
@@ -150,10 +147,10 @@ class SpiderLoaderTest(unittest.TestCase):
# at least until all six versions we can import (including botocore.vendored.six)
# are updated to 1.16.0+
w.pop(0)
- self.assertIn("Could not load spiders from module", str(w[0].message))
+ assert "Could not load spiders from module" in str(w[0].message)
spiders = spider_loader.list()
- self.assertEqual(spiders, [])
+ assert not spiders
def test_syntax_error_exception(self):
module = "tests.test_spiderloader.test_spiders.spider1"
@@ -179,14 +176,14 @@ class SpiderLoaderTest(unittest.TestCase):
# at least until all six versions we can import (including botocore.vendored.six)
# are updated to 1.16.0+
w.pop(0)
- self.assertIn("Could not load spiders from module", str(w[0].message))
+ assert "Could not load spiders from module" in str(w[0].message)
spiders = spider_loader.list()
- self.assertEqual(spiders, [])
+ assert not spiders
-class DuplicateSpiderNameLoaderTest(unittest.TestCase):
- def setUp(self):
+class TestDuplicateSpiderNameLoader:
+ def setup_method(self):
orig_spiders_dir = module_dir / "test_spiders"
self.tmpdir = Path(mkdtemp())
self.spiders_dir = self.tmpdir / "test_spiders_xxx"
@@ -194,7 +191,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
sys.path.append(str(self.tmpdir))
self.settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})
- def tearDown(self):
+ def teardown_method(self):
del sys.modules["test_spiders_xxx"]
sys.path.remove(str(self.tmpdir))
@@ -208,18 +205,18 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
with warnings.catch_warnings(record=True) as w:
spider_loader = SpiderLoader.from_settings(self.settings)
- self.assertEqual(len(w), 1)
+ assert len(w) == 1
msg = str(w[0].message)
- self.assertIn("several spiders with the same name", msg)
- self.assertIn("'spider3'", msg)
- self.assertTrue(msg.count("'spider3'") == 2)
+ assert "several spiders with the same name" in msg
+ assert "'spider3'" in msg
+ assert msg.count("'spider3'") == 2
- self.assertNotIn("'spider1'", msg)
- self.assertNotIn("'spider2'", msg)
- self.assertNotIn("'spider4'", msg)
+ assert "'spider1'" not in msg
+ assert "'spider2'" not in msg
+ assert "'spider4'" not in msg
spiders = set(spider_loader.list())
- self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"})
+ assert spiders == {"spider1", "spider2", "spider3", "spider4"}
def test_multiple_dupename_warning(self):
# copy 2 spider modules so as to have duplicate spider name
@@ -236,17 +233,17 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
with warnings.catch_warnings(record=True) as w:
spider_loader = SpiderLoader.from_settings(self.settings)
- self.assertEqual(len(w), 1)
+ assert len(w) == 1
msg = str(w[0].message)
- self.assertIn("several spiders with the same name", msg)
- self.assertIn("'spider1'", msg)
- self.assertTrue(msg.count("'spider1'") == 2)
+ assert "several spiders with the same name" in msg
+ assert "'spider1'" in msg
+ assert msg.count("'spider1'") == 2
- self.assertIn("'spider2'", msg)
- self.assertTrue(msg.count("'spider2'") == 2)
+ assert "'spider2'" in msg
+ assert msg.count("'spider2'") == 2
- self.assertNotIn("'spider3'", msg)
- self.assertNotIn("'spider4'", msg)
+ assert "'spider3'" not in msg
+ assert "'spider4'" not in msg
spiders = set(spider_loader.list())
- self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"})
+ assert spiders == {"spider1", "spider2", "spider3", "spider4"}
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index a9f3876bb..ddc9b5206 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -18,7 +18,7 @@ from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from scrapy.utils.test import get_crawler
-class SpiderMiddlewareTestCase(TestCase):
+class TestSpiderMiddleware(TestCase):
def setUp(self):
self.request = Request("http://example.com/index.html")
self.response = Response(self.request.url, request=self.request)
@@ -41,7 +41,7 @@ class SpiderMiddlewareTestCase(TestCase):
return results[0]
-class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase):
+class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware):
"""Invalid return value for process_spider_input method"""
def test_invalid_process_spider_input(self):
@@ -51,11 +51,11 @@ class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase):
self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware())
result = self._scrape_response()
- self.assertIsInstance(result, Failure)
- self.assertIsInstance(result.value, _InvalidOutput)
+ assert isinstance(result, Failure)
+ assert isinstance(result.value, _InvalidOutput)
-class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase):
+class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware):
"""Invalid return value for process_spider_output method"""
def test_invalid_process_spider_output(self):
@@ -65,11 +65,11 @@ class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase):
self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware())
result = self._scrape_response()
- self.assertIsInstance(result, Failure)
- self.assertIsInstance(result.value, _InvalidOutput)
+ assert isinstance(result, Failure)
+ assert isinstance(result.value, _InvalidOutput)
-class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase):
+class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware):
"""Invalid return value for process_spider_exception method"""
def test_invalid_process_spider_exception(self):
@@ -84,11 +84,11 @@ class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase):
self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware())
self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
result = self._scrape_response()
- self.assertIsInstance(result, Failure)
- self.assertIsInstance(result.value, _InvalidOutput)
+ assert isinstance(result, Failure)
+ assert isinstance(result.value, _InvalidOutput)
-class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
+class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):
"""Re raise the exception by returning None"""
def test_process_spider_exception_return_none(self):
@@ -103,11 +103,11 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware())
self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
result = self._scrape_response()
- self.assertIsInstance(result, Failure)
- self.assertIsInstance(result.value, ZeroDivisionError)
+ assert isinstance(result, Failure)
+ assert isinstance(result.value, ZeroDivisionError)
-class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
+class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
"""Helpers for testing sync, async and mixed middlewares.
Should work for process_spider_output and, when it's supported, process_start_requests.
@@ -148,14 +148,13 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
result = yield self._get_middleware_result(
*mw_classes, start_index=start_index
)
- self.assertIsInstance(result, Iterable)
+ assert isinstance(result, Iterable)
result_list = list(result)
- self.assertEqual(len(result_list), self.RESULT_COUNT)
- self.assertIsInstance(result_list[0], self.ITEM_TYPE)
- self.assertEqual("downgraded to a non-async" in str(log), downgrade)
- self.assertEqual(
- "doesn't support asynchronous spider output" in str(log),
- ProcessSpiderOutputSimpleMiddleware in mw_classes,
+ assert len(result_list) == self.RESULT_COUNT
+ assert isinstance(result_list[0], self.ITEM_TYPE)
+ assert ("downgraded to a non-async" in str(log)) == downgrade
+ assert ("doesn't support asynchronous spider output" in str(log)) == (
+ ProcessSpiderOutputSimpleMiddleware in mw_classes
)
@defer.inlineCallbacks
@@ -166,11 +165,11 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
result = yield self._get_middleware_result(
*mw_classes, start_index=start_index
)
- self.assertIsInstance(result, AsyncIterator)
+ assert isinstance(result, AsyncIterator)
result_list = yield deferred_from_coro(collect_asyncgen(result))
- self.assertEqual(len(result_list), self.RESULT_COUNT)
- self.assertIsInstance(result_list[0], self.ITEM_TYPE)
- self.assertEqual("downgraded to a non-async" in str(log), downgrade)
+ assert len(result_list) == self.RESULT_COUNT
+ assert isinstance(result_list[0], self.ITEM_TYPE)
+ assert ("downgraded to a non-async" in str(log)) == downgrade
class ProcessSpiderOutputSimpleMiddleware:
@@ -212,7 +211,7 @@ class ProcessSpiderExceptionAsyncIterableMiddleware:
yield {"foo": 3}
-class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
+class TestProcessSpiderOutputSimple(TestBaseAsyncSpiderMiddleware):
"""process_spider_output tests for simple callbacks"""
ITEM_TYPE = dict
@@ -257,7 +256,7 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
return self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN)
-class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple):
+class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple):
"""process_spider_output tests for async generator callbacks"""
async def _scrape_func(self, *args, **kwargs):
@@ -297,7 +296,7 @@ class ProcessSpiderOutputCoroutineMiddleware:
return result
-class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
+class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware):
@defer.inlineCallbacks
def test_non_iterable(self):
with pytest.raises(
@@ -324,7 +323,7 @@ class ProcessStartRequestsSimpleMiddleware:
yield from start_requests
-class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase):
+class TestProcessStartRequestsSimple(TestBaseAsyncSpiderMiddleware):
"""process_start_requests tests for simple start_requests"""
ITEM_TYPE = (Request, dict)
@@ -373,67 +372,65 @@ class UniversalMiddlewareBothAsync:
yield
-class UniversalMiddlewareManagerTest(TestCase):
- def setUp(self):
+class TestUniversalMiddlewareManager:
+ def setup_method(self):
self.mwman = SpiderMiddlewareManager()
def test_simple_mw(self):
mw = ProcessSpiderOutputSimpleMiddleware()
self.mwman._add_middleware(mw)
- self.assertEqual(
- self.mwman.methods["process_spider_output"][0], mw.process_spider_output
+ assert (
+ self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable
)
def test_async_mw(self):
mw = ProcessSpiderOutputAsyncGenMiddleware()
self.mwman._add_middleware(mw)
- self.assertEqual(
- self.mwman.methods["process_spider_output"][0], mw.process_spider_output
+ assert (
+ self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable
)
def test_universal_mw(self):
mw = ProcessSpiderOutputUniversalMiddleware()
self.mwman._add_middleware(mw)
- self.assertEqual(
- self.mwman.methods["process_spider_output"][0],
- (mw.process_spider_output, mw.process_spider_output_async),
+ assert self.mwman.methods["process_spider_output"][0] == (
+ mw.process_spider_output,
+ mw.process_spider_output_async,
)
def test_universal_mw_no_sync(self):
with LogCapture() as log:
self.mwman._add_middleware(UniversalMiddlewareNoSync())
- self.assertIn(
+ assert (
"UniversalMiddlewareNoSync has process_spider_output_async"
- " without process_spider_output",
- str(log),
+ " without process_spider_output" in str(log)
)
- self.assertEqual(self.mwman.methods["process_spider_output"][0], None)
+ assert self.mwman.methods["process_spider_output"][0] is None
def test_universal_mw_both_sync(self):
mw = UniversalMiddlewareBothSync()
with LogCapture() as log:
self.mwman._add_middleware(mw)
- self.assertIn(
+ assert (
"UniversalMiddlewareBothSync.process_spider_output_async "
- "is not an async generator function",
- str(log),
+ "is not an async generator function" in str(log)
)
- self.assertEqual(
- self.mwman.methods["process_spider_output"][0], mw.process_spider_output
+ assert (
+ self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable
)
def test_universal_mw_both_async(self):
with LogCapture() as log:
self.mwman._add_middleware(UniversalMiddlewareBothAsync())
- self.assertIn(
+ assert (
"UniversalMiddlewareBothAsync.process_spider_output "
- "is an async generator function while process_spider_output_async exists",
- str(log),
+ "is an async generator function while process_spider_output_async exists"
+ in str(log)
)
- self.assertEqual(self.mwman.methods["process_spider_output"][0], None)
+ assert self.mwman.methods["process_spider_output"][0] is None
-class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase):
+class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware):
ITEM_TYPE = dict
MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware
MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
@@ -474,7 +471,7 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase):
return self._test_simple_base(self.MW_UNIVERSAL)
-class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest):
+class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple):
async def _scrape_func(self, *args, **kwargs):
for item in super()._scrape_func():
yield item
@@ -503,7 +500,7 @@ class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest):
return self._test_asyncgen_base(self.MW_UNIVERSAL)
-class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase):
+class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):
ITEM_TYPE = dict
MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware
MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py
index e359d9cfc..dfcc141c3 100644
--- a/tests/test_spidermiddleware_depth.py
+++ b/tests/test_spidermiddleware_depth.py
@@ -1,5 +1,3 @@
-from unittest import TestCase
-
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.depth import DepthMiddleware
from scrapy.spiders import Spider
@@ -7,8 +5,8 @@ from scrapy.statscollectors import StatsCollector
from scrapy.utils.test import get_crawler
-class TestDepthMiddleware(TestCase):
- def setUp(self):
+class TestDepthMiddleware:
+ def setup_method(self):
crawler = get_crawler(Spider)
self.spider = crawler._create_spider("scrapytest.org")
@@ -24,18 +22,18 @@ class TestDepthMiddleware(TestCase):
result = [Request("http://scrapytest.org")]
out = list(self.mw.process_spider_output(resp, result, self.spider))
- self.assertEqual(out, result)
+ assert out == result
rdc = self.stats.get_value("request_depth_count/1", spider=self.spider)
- self.assertEqual(rdc, 1)
+ assert rdc == 1
req.meta["depth"] = 1
out2 = list(self.mw.process_spider_output(resp, result, self.spider))
- self.assertEqual(out2, [])
+ assert not out2
rdm = self.stats.get_value("request_depth_max", spider=self.spider)
- self.assertEqual(rdm, 1)
+ assert rdm == 1
- def tearDown(self):
+ def teardown_method(self):
self.stats.close_spider(self.spider, "")
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index f9eb93d6b..e306579fa 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -1,10 +1,9 @@
import logging
-from unittest import TestCase
import pytest
from testfixtures import LogCapture
from twisted.internet import defer
-from twisted.trial.unittest import TestCase as TrialTestCase
+from twisted.trial.unittest import TestCase
from scrapy.http import Request, Response
from scrapy.settings import Settings
@@ -59,8 +58,8 @@ def _responses(request, status_codes):
return responses
-class TestHttpErrorMiddleware(TestCase):
- def setUp(self):
+class TestHttpErrorMiddleware:
+ def setup_method(self):
crawler = get_crawler(Spider)
self.spider = Spider.from_crawler(crawler, name="foo")
self.mw = HttpErrorMiddleware(Settings({}))
@@ -68,19 +67,20 @@ class TestHttpErrorMiddleware(TestCase):
self.res200, self.res404 = _responses(self.req, [200, 404])
def test_process_spider_input(self):
- self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
+ assert self.mw.process_spider_input(self.res200, self.spider) is None
with pytest.raises(HttpError):
self.mw.process_spider_input(self.res404, self.spider)
def test_process_spider_exception(self):
- self.assertEqual(
- [],
+ assert (
self.mw.process_spider_exception(
self.res404, HttpError(self.res404), self.spider
- ),
+ )
+ == []
)
- self.assertIsNone(
+ assert (
self.mw.process_spider_exception(self.res404, Exception(), self.spider)
+ is None
)
def test_handle_httpstatus_list(self):
@@ -88,26 +88,26 @@ class TestHttpErrorMiddleware(TestCase):
res.request = Request(
"http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
)
- self.assertIsNone(self.mw.process_spider_input(res, self.spider))
+ assert self.mw.process_spider_input(res, self.spider) is None
self.spider.handle_httpstatus_list = [404]
- self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
+ assert self.mw.process_spider_input(self.res404, self.spider) is None
-class TestHttpErrorMiddlewareSettings(TestCase):
+class TestHttpErrorMiddlewareSettings:
"""Similar test, but with settings"""
- def setUp(self):
+ def setup_method(self):
self.spider = Spider("foo")
self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)}))
self.req = Request("http://scrapytest.org")
self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
def test_process_spider_input(self):
- self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
+ assert self.mw.process_spider_input(self.res200, self.spider) is None
with pytest.raises(HttpError):
self.mw.process_spider_input(self.res404, self.spider)
- self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider))
+ assert self.mw.process_spider_input(self.res402, self.spider) is None
def test_meta_overrides_settings(self):
request = Request(
@@ -118,27 +118,27 @@ class TestHttpErrorMiddlewareSettings(TestCase):
res402 = self.res402.copy()
res402.request = request
- self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
+ assert self.mw.process_spider_input(res404, self.spider) is None
with pytest.raises(HttpError):
self.mw.process_spider_input(res402, self.spider)
def test_spider_override_settings(self):
self.spider.handle_httpstatus_list = [404]
- self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
+ assert self.mw.process_spider_input(self.res404, self.spider) is None
with pytest.raises(HttpError):
self.mw.process_spider_input(self.res402, self.spider)
-class TestHttpErrorMiddlewareHandleAll(TestCase):
- def setUp(self):
+class TestHttpErrorMiddlewareHandleAll:
+ def setup_method(self):
self.spider = Spider("foo")
self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True}))
self.req = Request("http://scrapytest.org")
self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
def test_process_spider_input(self):
- self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
- self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
+ assert self.mw.process_spider_input(self.res200, self.spider) is None
+ assert self.mw.process_spider_input(self.res404, self.spider) is None
def test_meta_overrides_settings(self):
request = Request(
@@ -149,7 +149,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
res402 = self.res402.copy()
res402.request = request
- self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
+ assert self.mw.process_spider_input(res404, self.spider) is None
with pytest.raises(HttpError):
self.mw.process_spider_input(res402, self.spider)
@@ -169,10 +169,10 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
with pytest.raises(HttpError):
mw.process_spider_input(res404, self.spider)
- self.assertIsNone(mw.process_spider_input(res402, self.spider))
+ assert mw.process_spider_input(res402, self.spider) is None
-class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
+class TestHttpErrorMiddlewareIntegrational(TestCase):
@classmethod
def setUpClass(cls):
cls.mockserver = MockServer()
@@ -187,28 +187,28 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
crawler = get_crawler(_HttpErrorSpider)
yield crawler.crawl(mockserver=self.mockserver)
assert not crawler.spider.skipped, crawler.spider.skipped
- self.assertEqual(crawler.spider.parsed, {"200"})
- self.assertEqual(crawler.spider.failed, {"404", "402", "500"})
+ assert crawler.spider.parsed == {"200"}
+ assert crawler.spider.failed == {"404", "402", "500"}
get_value = crawler.stats.get_value
- self.assertEqual(get_value("httperror/response_ignored_count"), 3)
- self.assertEqual(get_value("httperror/response_ignored_status_count/404"), 1)
- self.assertEqual(get_value("httperror/response_ignored_status_count/402"), 1)
- self.assertEqual(get_value("httperror/response_ignored_status_count/500"), 1)
+ assert get_value("httperror/response_ignored_count") == 3
+ assert get_value("httperror/response_ignored_status_count/404") == 1
+ assert get_value("httperror/response_ignored_status_count/402") == 1
+ assert get_value("httperror/response_ignored_status_count/500") == 1
@defer.inlineCallbacks
def test_logging(self):
crawler = get_crawler(_HttpErrorSpider)
with LogCapture() as log:
yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402})
- self.assertEqual(crawler.spider.parsed, {"200", "402"})
- self.assertEqual(crawler.spider.skipped, {"402"})
- self.assertEqual(crawler.spider.failed, {"404", "500"})
+ assert crawler.spider.parsed == {"200", "402"}
+ assert crawler.spider.skipped == {"402"}
+ assert crawler.spider.failed == {"404", "500"}
- self.assertIn("Ignoring response <404", str(log))
- self.assertIn("Ignoring response <500", str(log))
- self.assertNotIn("Ignoring response <200", str(log))
- self.assertNotIn("Ignoring response <402", str(log))
+ assert "Ignoring response <404" in str(log)
+ assert "Ignoring response <500" in str(log)
+ assert "Ignoring response <200" not in str(log)
+ assert "Ignoring response <402" not in str(log)
@defer.inlineCallbacks
def test_logging_level(self):
@@ -216,22 +216,22 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
crawler = get_crawler(_HttpErrorSpider)
with LogCapture(level=logging.INFO) as log:
yield crawler.crawl(mockserver=self.mockserver)
- self.assertEqual(crawler.spider.parsed, {"200"})
- self.assertEqual(crawler.spider.failed, {"404", "402", "500"})
+ assert crawler.spider.parsed == {"200"}
+ assert crawler.spider.failed == {"404", "402", "500"}
- self.assertIn("Ignoring response <402", str(log))
- self.assertIn("Ignoring response <404", str(log))
- self.assertIn("Ignoring response <500", str(log))
- self.assertNotIn("Ignoring response <200", str(log))
+ assert "Ignoring response <402" in str(log)
+ assert "Ignoring response <404" in str(log)
+ assert "Ignoring response <500" in str(log)
+ assert "Ignoring response <200" not in str(log)
# with level WARNING, we shouldn't capture anything from HttpError
crawler = get_crawler(_HttpErrorSpider)
with LogCapture(level=logging.WARNING) as log:
yield crawler.crawl(mockserver=self.mockserver)
- self.assertEqual(crawler.spider.parsed, {"200"})
- self.assertEqual(crawler.spider.failed, {"404", "402", "500"})
+ assert crawler.spider.parsed == {"200"}
+ assert crawler.spider.failed == {"404", "402", "500"}
- self.assertNotIn("Ignoring response <402", str(log))
- self.assertNotIn("Ignoring response <404", str(log))
- self.assertNotIn("Ignoring response <500", str(log))
- self.assertNotIn("Ignoring response <200", str(log))
+ assert "Ignoring response <402" not in str(log)
+ assert "Ignoring response <404" not in str(log)
+ assert "Ignoring response <500" not in str(log)
+ assert "Ignoring response <200" not in str(log)
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index 906928e01..f4563a0a4 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -1,5 +1,4 @@
import warnings
-from unittest import TestCase
from urllib.parse import urlparse
from scrapy.http import Request, Response
@@ -8,8 +7,8 @@ from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
-class TestOffsiteMiddleware(TestCase):
- def setUp(self):
+class TestOffsiteMiddleware:
+ def setup_method(self):
crawler = get_crawler(Spider)
self.spider = crawler._create_spider(**self._get_spiderargs())
self.mw = OffsiteMiddleware.from_crawler(crawler)
@@ -46,7 +45,7 @@ class TestOffsiteMiddleware(TestCase):
reqs = onsite_reqs + offsite_reqs
out = list(self.mw.process_spider_output(res, reqs, self.spider))
- self.assertEqual(out, onsite_reqs)
+ assert out == onsite_reqs
class TestOffsiteMiddleware2(TestOffsiteMiddleware):
@@ -57,7 +56,7 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware):
res = Response("http://scrapytest.org")
reqs = [Request("http://a.com/b.html"), Request("http://b.com/1")]
out = list(self.mw.process_spider_output(res, reqs, self.spider))
- self.assertEqual(out, reqs)
+ assert out == reqs
class TestOffsiteMiddleware3(TestOffsiteMiddleware2):
@@ -77,7 +76,7 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3):
res = Response("http://scrapytest.org")
reqs = [Request("http://scrapytest.org/1")]
out = list(self.mw.process_spider_output(res, reqs, self.spider))
- self.assertEqual(out, reqs)
+ assert out == reqs
class TestOffsiteMiddleware5(TestOffsiteMiddleware4):
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index e51957497..6e26a85ea 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -324,9 +324,9 @@ class TestSpiderMiddleware(TestCase):
was enqueued from the recovery middleware)
"""
log = yield self.crawl_log(RecoverySpider)
- self.assertIn("Middleware: TabError exception caught", str(log))
- self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1)
- self.assertIn("'item_scraped_count': 3", str(log))
+ assert "Middleware: TabError exception caught" in str(log)
+ assert str(log).count("Middleware: TabError exception caught") == 1
+ assert "'item_scraped_count': 3" in str(log)
@defer.inlineCallbacks
def test_recovery_asyncgen(self):
@@ -334,9 +334,9 @@ class TestSpiderMiddleware(TestCase):
Same as test_recovery but with an async callback.
"""
log = yield self.crawl_log(RecoveryAsyncGenSpider)
- self.assertIn("Middleware: TabError exception caught", str(log))
- self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1)
- self.assertIn("'item_scraped_count': 3", str(log))
+ assert "Middleware: TabError exception caught" in str(log)
+ assert str(log).count("Middleware: TabError exception caught") == 1
+ assert "'item_scraped_count': 3" in str(log)
@defer.inlineCallbacks
def test_process_spider_input_without_errback(self):
@@ -345,8 +345,8 @@ class TestSpiderMiddleware(TestCase):
process_spider_exception chain from the start if the Request has no errback
"""
log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithoutErrback)
- self.assertIn("Middleware: will raise IndexError", str(log1))
- self.assertIn("Middleware: IndexError exception caught", str(log1))
+ assert "Middleware: will raise IndexError" in str(log1)
+ assert "Middleware: IndexError exception caught" in str(log1)
@defer.inlineCallbacks
def test_process_spider_input_with_errback(self):
@@ -355,12 +355,12 @@ class TestSpiderMiddleware(TestCase):
process_spider_exception chain if the Request has an errback
"""
log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithErrback)
- self.assertNotIn("Middleware: IndexError exception caught", str(log1))
- self.assertIn("Middleware: will raise IndexError", str(log1))
- self.assertIn("Got a Failure on the Request errback", str(log1))
- self.assertIn("{'from': 'errback'}", str(log1))
- self.assertNotIn("{'from': 'callback'}", str(log1))
- self.assertIn("'item_scraped_count': 1", str(log1))
+ assert "Middleware: IndexError exception caught" not in str(log1)
+ assert "Middleware: will raise IndexError" in str(log1)
+ assert "Got a Failure on the Request errback" in str(log1)
+ assert "{'from': 'errback'}" in str(log1)
+ assert "{'from': 'callback'}" not in str(log1)
+ assert "'item_scraped_count': 1" in str(log1)
@defer.inlineCallbacks
def test_generator_callback(self):
@@ -370,8 +370,8 @@ class TestSpiderMiddleware(TestCase):
exception is raised should be processed normally.
"""
log2 = yield self.crawl_log(GeneratorCallbackSpider)
- self.assertIn("Middleware: ImportError exception caught", str(log2))
- self.assertIn("'item_scraped_count': 2", str(log2))
+ assert "Middleware: ImportError exception caught" in str(log2)
+ assert "'item_scraped_count': 2" in str(log2)
@defer.inlineCallbacks
def test_async_generator_callback(self):
@@ -379,8 +379,8 @@ class TestSpiderMiddleware(TestCase):
Same as test_generator_callback but with an async callback.
"""
log2 = yield self.crawl_log(AsyncGeneratorCallbackSpider)
- self.assertIn("Middleware: ImportError exception caught", str(log2))
- self.assertIn("'item_scraped_count': 2", str(log2))
+ assert "Middleware: ImportError exception caught" in str(log2)
+ assert "'item_scraped_count': 2" in str(log2)
@defer.inlineCallbacks
def test_generator_callback_right_after_callback(self):
@@ -389,8 +389,8 @@ class TestSpiderMiddleware(TestCase):
even if the middleware is placed right after the spider
"""
log21 = yield self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider)
- self.assertIn("Middleware: ImportError exception caught", str(log21))
- self.assertIn("'item_scraped_count': 2", str(log21))
+ assert "Middleware: ImportError exception caught" in str(log21)
+ assert "'item_scraped_count': 2" in str(log21)
@defer.inlineCallbacks
def test_not_a_generator_callback(self):
@@ -399,8 +399,8 @@ class TestSpiderMiddleware(TestCase):
be caught by the process_spider_exception chain. No items should be processed.
"""
log3 = yield self.crawl_log(NotGeneratorCallbackSpider)
- self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3))
- self.assertNotIn("item_scraped_count", str(log3))
+ assert "Middleware: ZeroDivisionError exception caught" in str(log3)
+ assert "item_scraped_count" not in str(log3)
@defer.inlineCallbacks
def test_not_a_generator_callback_right_after_callback(self):
@@ -411,8 +411,8 @@ class TestSpiderMiddleware(TestCase):
log31 = yield self.crawl_log(
NotGeneratorCallbackSpiderMiddlewareRightAfterSpider
)
- self.assertIn("Middleware: ZeroDivisionError exception caught", str(log31))
- self.assertNotIn("item_scraped_count", str(log31))
+ assert "Middleware: ZeroDivisionError exception caught" in str(log31)
+ assert "item_scraped_count" not in str(log31)
@defer.inlineCallbacks
def test_generator_output_chain(self):
@@ -425,22 +425,22 @@ class TestSpiderMiddleware(TestCase):
process_spider_exception chain)
"""
log4 = yield self.crawl_log(GeneratorOutputChainSpider)
- self.assertIn("'item_scraped_count': 2", str(log4))
- self.assertIn(
- "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught",
- str(log4),
+ assert "'item_scraped_count': 2" in str(log4)
+ assert (
+ "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught"
+ in str(log4)
)
- self.assertIn(
- "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught",
- str(log4),
+ assert (
+ "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught"
+ in str(log4)
)
- self.assertNotIn(
- "GeneratorFailMiddleware.process_spider_exception: LookupError caught",
- str(log4),
+ assert (
+ "GeneratorFailMiddleware.process_spider_exception: LookupError caught"
+ not in str(log4)
)
- self.assertNotIn(
- "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught",
- str(log4),
+ assert (
+ "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught"
+ not in str(log4)
)
item_from_callback = {
"processed": [
@@ -457,9 +457,9 @@ class TestSpiderMiddleware(TestCase):
"GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output",
]
}
- self.assertIn(str(item_from_callback), str(log4))
- self.assertIn(str(item_recovered), str(log4))
- self.assertNotIn("parse-second-item", str(log4))
+ assert str(item_from_callback) in str(log4)
+ assert str(item_recovered) in str(log4)
+ assert "parse-second-item" not in str(log4)
@defer.inlineCallbacks
def test_not_a_generator_output_chain(self):
@@ -472,22 +472,22 @@ class TestSpiderMiddleware(TestCase):
from the spider callback are lost)
"""
log5 = yield self.crawl_log(NotGeneratorOutputChainSpider)
- self.assertIn("'item_scraped_count': 1", str(log5))
- self.assertIn(
- "GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught",
- str(log5),
+ assert "'item_scraped_count': 1" in str(log5)
+ assert (
+ "GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught"
+ in str(log5)
)
- self.assertIn(
- "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught",
- str(log5),
+ assert (
+ "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught"
+ in str(log5)
)
- self.assertNotIn(
- "GeneratorFailMiddleware.process_spider_exception: ReferenceError caught",
- str(log5),
+ assert (
+ "GeneratorFailMiddleware.process_spider_exception: ReferenceError caught"
+ not in str(log5)
)
- self.assertNotIn(
- "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught",
- str(log5),
+ assert (
+ "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught"
+ not in str(log5)
)
item_recovered = {
"processed": [
@@ -495,6 +495,6 @@ class TestSpiderMiddleware(TestCase):
"NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output",
]
}
- self.assertIn(str(item_recovered), str(log5))
- self.assertNotIn("parse-first-item", str(log5))
- self.assertNotIn("parse-second-item", str(log5))
+ assert str(item_recovered) in str(log5)
+ assert "parse-first-item" not in str(log5)
+ assert "parse-second-item" not in str(log5)
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 01a87c645..300a40c13 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -2,7 +2,6 @@ from __future__ import annotations
import warnings
from typing import Any
-from unittest import TestCase
from urllib.parse import urlparse
import pytest
@@ -35,7 +34,7 @@ from scrapy.spidermiddlewares.referer import (
from scrapy.spiders import Spider
-class TestRefererMiddleware(TestCase):
+class TestRefererMiddleware:
req_meta: dict[str, Any] = {}
resp_headers: dict[str, str] = {}
settings: dict[str, Any] = {}
@@ -43,7 +42,7 @@ class TestRefererMiddleware(TestCase):
("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"),
]
- def setUp(self):
+ def setup_method(self):
self.spider = Spider("foo")
settings = Settings(self.settings)
self.mw = RefererMiddleware(settings)
@@ -59,7 +58,7 @@ class TestRefererMiddleware(TestCase):
response = self.get_response(origin)
request = self.get_request(target)
out = list(self.mw.process_spider_output(response, [request], self.spider))
- self.assertEqual(out[0].headers.get("Referer"), referrer)
+ assert out[0].headers.get("Referer") == referrer
class MixinDefault:
@@ -773,7 +772,7 @@ class TestRequestMetaPrecedence003(MixinUnsafeUrl, TestRefererMiddleware):
req_meta = {"referrer_policy": POLICY_UNSAFE_URL}
-class TestRequestMetaSettingFallback(TestCase):
+class TestRequestMetaSettingFallback:
params = [
(
# When an unknown policy is referenced in Request.meta
@@ -844,14 +843,14 @@ class TestRequestMetaSettingFallback(TestCase):
with warnings.catch_warnings(record=True) as w:
policy = mw.policy(response, request)
- self.assertIsInstance(policy, policy_class)
+ assert isinstance(policy, policy_class)
if check_warning:
- self.assertEqual(len(w), 1)
- self.assertEqual(w[0].category, RuntimeWarning, w[0].message)
+ assert len(w) == 1
+ assert w[0].category is RuntimeWarning, w[0].message
-class TestSettingsPolicyByName(TestCase):
+class TestSettingsPolicyByName:
def test_valid_name(self):
for s, p in [
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
@@ -866,7 +865,7 @@ class TestSettingsPolicyByName(TestCase):
]:
settings = Settings({"REFERRER_POLICY": s})
mw = RefererMiddleware(settings)
- self.assertEqual(mw.default_policy, p)
+ assert mw.default_policy == p
def test_valid_name_casevariants(self):
for s, p in [
@@ -882,7 +881,7 @@ class TestSettingsPolicyByName(TestCase):
]:
settings = Settings({"REFERRER_POLICY": s.upper()})
mw = RefererMiddleware(settings)
- self.assertEqual(mw.default_policy, p)
+ assert mw.default_policy == p
def test_invalid_name(self):
settings = Settings({"REFERRER_POLICY": "some-custom-unknown-policy"})
@@ -902,7 +901,7 @@ class TestSettingsPolicyByName(TestCase):
}
)
mw1 = RefererMiddleware(settings1)
- self.assertEqual(mw1.default_policy, StrictOriginWhenCrossOriginPolicy)
+ assert mw1.default_policy == StrictOriginWhenCrossOriginPolicy
# test parsing with space(s) after the comma
settings2 = Settings(
@@ -915,7 +914,7 @@ class TestSettingsPolicyByName(TestCase):
}
)
mw2 = RefererMiddleware(settings2)
- self.assertEqual(mw2.default_policy, UnsafeUrlPolicy)
+ assert mw2.default_policy == UnsafeUrlPolicy
def test_multiple_policy_tokens_all_invalid(self):
settings = Settings(
@@ -1003,7 +1002,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
),
]
- def setUp(self):
+ def setup_method(self):
self.spider = Spider("foo")
settings = Settings(self.settings)
self.referrermw = RefererMiddleware(settings)
@@ -1023,7 +1022,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
out = list(
self.referrermw.process_spider_output(response, [request], self.spider)
)
- self.assertEqual(out[0].headers.get("Referer"), init_referrer)
+ assert out[0].headers.get("Referer") == init_referrer
for status, url in redirections:
response = Response(
@@ -1035,7 +1034,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
self.referrermw.request_scheduled(request, self.spider)
assert isinstance(request, Request)
- self.assertEqual(request.headers.get("Referer"), final_referrer)
+ assert request.headers.get("Referer") == final_referrer
class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py
index 1a0f2e223..5cc3cdc6c 100644
--- a/tests/test_spidermiddleware_urllength.py
+++ b/tests/test_spidermiddleware_urllength.py
@@ -1,5 +1,3 @@
-from unittest import TestCase
-
from testfixtures import LogCapture
from scrapy.http import Request, Response
@@ -8,8 +6,8 @@ from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
-class TestUrlLengthMiddleware(TestCase):
- def setUp(self):
+class TestUrlLengthMiddleware:
+ def setup_method(self):
self.maxlength = 25
crawler = get_crawler(Spider, {"URLLENGTH_LIMIT": self.maxlength})
self.spider = crawler._create_spider("foo")
@@ -27,7 +25,7 @@ class TestUrlLengthMiddleware(TestCase):
)
def test_middleware_works(self):
- self.assertEqual(self.process_spider_output(), [self.short_url_req])
+ assert self.process_spider_output() == [self.short_url_req]
def test_logging(self):
with LogCapture() as log:
@@ -36,6 +34,6 @@ class TestUrlLengthMiddleware(TestCase):
ric = self.stats.get_value(
"urllength/request_ignored_count", spider=self.spider
)
- self.assertEqual(ric, 1)
+ assert ric == 1
- self.assertIn(f"Ignoring link (url length > {self.maxlength})", str(log))
+ assert f"Ignoring link (url length > {self.maxlength})" in str(log)
diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py
index 72692afab..cd31891a0 100644
--- a/tests/test_spiderstate.py
+++ b/tests/test_spiderstate.py
@@ -3,7 +3,6 @@ from datetime import datetime, timezone
from tempfile import mkdtemp
import pytest
-from twisted.trial import unittest
from scrapy.exceptions import NotConfigured
from scrapy.extensions.spiderstate import SpiderState
@@ -11,7 +10,7 @@ from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
-class SpiderStateTest(unittest.TestCase):
+class TestSpiderState:
def test_store_load(self):
jobdir = mkdtemp()
try:
@@ -27,7 +26,7 @@ class SpiderStateTest(unittest.TestCase):
spider2 = Spider(name="default")
ss2 = SpiderState(jobdir)
ss2.spider_opened(spider2)
- self.assertEqual(spider.state, {"one": 1, "dt": dt})
+ assert spider.state == {"one": 1, "dt": dt}
ss2.spider_closed(spider2)
finally:
shutil.rmtree(jobdir)
@@ -38,7 +37,7 @@ class SpiderStateTest(unittest.TestCase):
spider = Spider(name="default")
ss = SpiderState()
ss.spider_opened(spider)
- self.assertEqual(spider.state, {})
+ assert spider.state == {}
ss.spider_closed(spider)
def test_not_configured(self):
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index 8556b75dd..6283b9ad6 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -50,9 +50,9 @@ class FifoDiskQueueTestMixin:
q.push("a")
q.push(123)
q.push({"a": "dict"})
- self.assertEqual(q.pop(), "a")
- self.assertEqual(q.pop(), 123)
- self.assertEqual(q.pop(), {"a": "dict"})
+ assert q.pop() == "a"
+ assert q.pop() == 123
+ assert q.pop() == {"a": "dict"}
test_nonserializable_object = nonserializable_object_test
@@ -92,7 +92,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
q.push(i)
i2 = q.pop()
assert isinstance(i2, MyItem)
- self.assertEqual(i, i2)
+ assert i == i2
def test_serialize_loader(self):
q = self.queue()
@@ -101,7 +101,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
loader2 = q.pop()
assert isinstance(loader2, MyLoader)
assert loader2.default_item_class is MyItem
- self.assertEqual(loader2.name_out("x"), "xx")
+ assert loader2.name_out("x") == "xx"
def test_serialize_request_recursive(self):
q = self.queue()
@@ -110,23 +110,26 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
q.push(r)
r2 = q.pop()
assert isinstance(r2, Request)
- self.assertEqual(r.url, r2.url)
+ assert r.url == r2.url
assert r2.meta["request"] is r2
def test_non_pickable_object(self):
q = self.queue()
- try:
+ with pytest.raises(
+ ValueError,
+ match="Can't (get|pickle) local object|Can't pickle .*: it's not found as",
+ ) as exc_info:
q.push(lambda x: x)
- except ValueError as exc:
- if hasattr(sys, "pypy_version_info"):
- self.assertIsInstance(exc.__context__, pickle.PicklingError)
- else:
- self.assertIsInstance(exc.__context__, AttributeError)
+ if hasattr(sys, "pypy_version_info"):
+ assert isinstance(exc_info.value.__context__, pickle.PicklingError)
+ else:
+ assert isinstance(exc_info.value.__context__, AttributeError)
sel = Selector(text="some text
")
- try:
+ with pytest.raises(
+ ValueError, match="can't pickle Selector objects"
+ ) as exc_info:
q.push(sel)
- except ValueError as exc:
- self.assertIsInstance(exc.__context__, TypeError)
+ assert isinstance(exc_info.value.__context__, TypeError)
class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
@@ -151,9 +154,9 @@ class LifoDiskQueueTestMixin:
q.push("a")
q.push(123)
q.push({"a": "dict"})
- self.assertEqual(q.pop(), {"a": "dict"})
- self.assertEqual(q.pop(), 123)
- self.assertEqual(q.pop(), "a")
+ assert q.pop() == {"a": "dict"}
+ assert q.pop() == 123
+ assert q.pop() == "a"
test_nonserializable_object = nonserializable_object_test
@@ -173,7 +176,7 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
q.push(i)
i2 = q.pop()
assert isinstance(i2, MyItem)
- self.assertEqual(i, i2)
+ assert i == i2
def test_serialize_loader(self):
q = self.queue()
@@ -182,7 +185,7 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
loader2 = q.pop()
assert isinstance(loader2, MyLoader)
assert loader2.default_item_class is MyItem
- self.assertEqual(loader2.name_out("x"), "xx")
+ assert loader2.name_out("x") == "xx"
def test_serialize_request_recursive(self):
q = self.queue()
@@ -191,5 +194,5 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
q.push(r)
r2 = q.pop()
assert isinstance(r2, Request)
- self.assertEqual(r.url, r2.url)
+ assert r.url == r2.url
assert r2.meta["request"] is r2
diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py
index 88f6657d8..6c153f40e 100644
--- a/tests/test_squeues_request.py
+++ b/tests/test_squeues_request.py
@@ -22,14 +22,14 @@ from scrapy.squeues import (
from scrapy.utils.test import get_crawler
-class BaseQueueTestCase(unittest.TestCase):
- def setUp(self):
+class TestBaseQueue:
+ def setup_method(self):
self.tmpdir = tempfile.mkdtemp(prefix="scrapy-queue-tests-")
self.qpath = self.tempfilename()
self.qdir = tempfile.mkdtemp()
self.crawler = get_crawler(Spider)
- def tearDown(self):
+ def teardown_method(self):
shutil.rmtree(self.tmpdir)
def tempfilename(self):
@@ -48,36 +48,36 @@ class RequestQueueTestMixin:
if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
raise unittest.SkipTest("The queuelib queues do not define peek")
q = self.queue()
- self.assertEqual(len(q), 0)
- self.assertIsNone(q.peek())
- self.assertIsNone(q.pop())
+ assert len(q) == 0
+ assert q.peek() is None
+ assert q.pop() is None
req = Request("http://www.example.com")
q.push(req)
- self.assertEqual(len(q), 1)
- self.assertEqual(q.peek().url, req.url)
- self.assertEqual(q.pop().url, req.url)
- self.assertEqual(len(q), 0)
- self.assertIsNone(q.peek())
- self.assertIsNone(q.pop())
+ assert len(q) == 1
+ assert q.peek().url == req.url
+ assert q.pop().url == req.url
+ assert len(q) == 0
+ assert q.peek() is None
+ assert q.pop() is None
q.close()
def test_one_element_without_peek(self):
if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
raise unittest.SkipTest("The queuelib queues define peek")
q = self.queue()
- self.assertEqual(len(q), 0)
- self.assertIsNone(q.pop())
+ assert len(q) == 0
+ assert q.pop() is None
req = Request("http://www.example.com")
q.push(req)
- self.assertEqual(len(q), 1)
+ assert len(q) == 1
with pytest.raises(
NotImplementedError,
match="The underlying queue class does not implement 'peek'",
):
q.peek()
- self.assertEqual(q.pop().url, req.url)
- self.assertEqual(len(q), 0)
- self.assertIsNone(q.pop())
+ assert q.pop().url == req.url
+ assert len(q) == 0
+ assert q.pop() is None
q.close()
@@ -86,35 +86,35 @@ class FifoQueueMixin(RequestQueueTestMixin):
if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
raise unittest.SkipTest("The queuelib queues do not define peek")
q = self.queue()
- self.assertEqual(len(q), 0)
- self.assertIsNone(q.peek())
- self.assertIsNone(q.pop())
+ assert len(q) == 0
+ assert q.peek() is None
+ assert q.pop() is None
req1 = Request("http://www.example.com/1")
req2 = Request("http://www.example.com/2")
req3 = Request("http://www.example.com/3")
q.push(req1)
q.push(req2)
q.push(req3)
- self.assertEqual(len(q), 3)
- self.assertEqual(q.peek().url, req1.url)
- self.assertEqual(q.pop().url, req1.url)
- self.assertEqual(len(q), 2)
- self.assertEqual(q.peek().url, req2.url)
- self.assertEqual(q.pop().url, req2.url)
- self.assertEqual(len(q), 1)
- self.assertEqual(q.peek().url, req3.url)
- self.assertEqual(q.pop().url, req3.url)
- self.assertEqual(len(q), 0)
- self.assertIsNone(q.peek())
- self.assertIsNone(q.pop())
+ assert len(q) == 3
+ assert q.peek().url == req1.url
+ assert q.pop().url == req1.url
+ assert len(q) == 2
+ assert q.peek().url == req2.url
+ assert q.pop().url == req2.url
+ assert len(q) == 1
+ assert q.peek().url == req3.url
+ assert q.pop().url == req3.url
+ assert len(q) == 0
+ assert q.peek() is None
+ assert q.pop() is None
q.close()
def test_fifo_without_peek(self):
if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
raise unittest.SkipTest("The queuelib queues do not define peek")
q = self.queue()
- self.assertEqual(len(q), 0)
- self.assertIsNone(q.pop())
+ assert len(q) == 0
+ assert q.pop() is None
req1 = Request("http://www.example.com/1")
req2 = Request("http://www.example.com/2")
req3 = Request("http://www.example.com/3")
@@ -126,14 +126,14 @@ class FifoQueueMixin(RequestQueueTestMixin):
match="The underlying queue class does not implement 'peek'",
):
q.peek()
- self.assertEqual(len(q), 3)
- self.assertEqual(q.pop().url, req1.url)
- self.assertEqual(len(q), 2)
- self.assertEqual(q.pop().url, req2.url)
- self.assertEqual(len(q), 1)
- self.assertEqual(q.pop().url, req3.url)
- self.assertEqual(len(q), 0)
- self.assertIsNone(q.pop())
+ assert len(q) == 3
+ assert q.pop().url == req1.url
+ assert len(q) == 2
+ assert q.pop().url == req2.url
+ assert len(q) == 1
+ assert q.pop().url == req3.url
+ assert len(q) == 0
+ assert q.pop() is None
q.close()
@@ -142,35 +142,35 @@ class LifoQueueMixin(RequestQueueTestMixin):
if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
raise unittest.SkipTest("The queuelib queues do not define peek")
q = self.queue()
- self.assertEqual(len(q), 0)
- self.assertIsNone(q.peek())
- self.assertIsNone(q.pop())
+ assert len(q) == 0
+ assert q.peek() is None
+ assert q.pop() is None
req1 = Request("http://www.example.com/1")
req2 = Request("http://www.example.com/2")
req3 = Request("http://www.example.com/3")
q.push(req1)
q.push(req2)
q.push(req3)
- self.assertEqual(len(q), 3)
- self.assertEqual(q.peek().url, req3.url)
- self.assertEqual(q.pop().url, req3.url)
- self.assertEqual(len(q), 2)
- self.assertEqual(q.peek().url, req2.url)
- self.assertEqual(q.pop().url, req2.url)
- self.assertEqual(len(q), 1)
- self.assertEqual(q.peek().url, req1.url)
- self.assertEqual(q.pop().url, req1.url)
- self.assertEqual(len(q), 0)
- self.assertIsNone(q.peek())
- self.assertIsNone(q.pop())
+ assert len(q) == 3
+ assert q.peek().url == req3.url
+ assert q.pop().url == req3.url
+ assert len(q) == 2
+ assert q.peek().url == req2.url
+ assert q.pop().url == req2.url
+ assert len(q) == 1
+ assert q.peek().url == req1.url
+ assert q.pop().url == req1.url
+ assert len(q) == 0
+ assert q.peek() is None
+ assert q.pop() is None
q.close()
def test_lifo_without_peek(self):
if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
raise unittest.SkipTest("The queuelib queues do not define peek")
q = self.queue()
- self.assertEqual(len(q), 0)
- self.assertIsNone(q.pop())
+ assert len(q) == 0
+ assert q.pop() is None
req1 = Request("http://www.example.com/1")
req2 = Request("http://www.example.com/2")
req3 = Request("http://www.example.com/3")
@@ -182,46 +182,46 @@ class LifoQueueMixin(RequestQueueTestMixin):
match="The underlying queue class does not implement 'peek'",
):
q.peek()
- self.assertEqual(len(q), 3)
- self.assertEqual(q.pop().url, req3.url)
- self.assertEqual(len(q), 2)
- self.assertEqual(q.pop().url, req2.url)
- self.assertEqual(len(q), 1)
- self.assertEqual(q.pop().url, req1.url)
- self.assertEqual(len(q), 0)
- self.assertIsNone(q.pop())
+ assert len(q) == 3
+ assert q.pop().url == req3.url
+ assert len(q) == 2
+ assert q.pop().url == req2.url
+ assert len(q) == 1
+ assert q.pop().url == req1.url
+ assert len(q) == 0
+ assert q.pop() is None
q.close()
-class PickleFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase):
+class TestPickleFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue):
def queue(self):
return PickleFifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/fifo")
-class PickleLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase):
+class TestPickleLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue):
def queue(self):
return PickleLifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/lifo")
-class MarshalFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase):
+class TestMarshalFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue):
def queue(self):
return MarshalFifoDiskQueue.from_crawler(
crawler=self.crawler, key="marshal/fifo"
)
-class MarshalLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase):
+class TestMarshalLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue):
def queue(self):
return MarshalLifoDiskQueue.from_crawler(
crawler=self.crawler, key="marshal/lifo"
)
-class FifoMemoryQueueRequestTest(FifoQueueMixin, BaseQueueTestCase):
+class TestFifoMemoryQueueRequest(FifoQueueMixin, TestBaseQueue):
def queue(self):
return FifoMemoryQueue.from_crawler(crawler=self.crawler)
-class LifoMemoryQueueRequestTest(LifoQueueMixin, BaseQueueTestCase):
+class TestLifoMemoryQueueRequest(LifoQueueMixin, TestBaseQueue):
def queue(self):
return LifoMemoryQueue.from_crawler(crawler=self.crawler)
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 3d4c7e88e..537614364 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -1,4 +1,3 @@
-import unittest
from datetime import datetime
from unittest import mock
@@ -8,8 +7,8 @@ from scrapy.statscollectors import DummyStatsCollector, StatsCollector
from scrapy.utils.test import get_crawler
-class CoreStatsExtensionTest(unittest.TestCase):
- def setUp(self):
+class TestCoreStatsExtension:
+ def setup_method(self):
self.crawler = get_crawler(Spider)
self.spider = self.crawler._create_spider("foo")
@@ -24,19 +23,16 @@ class CoreStatsExtensionTest(unittest.TestCase):
ext.response_received(self.spider)
ext.item_dropped({}, self.spider, ZeroDivisionError())
ext.spider_closed(self.spider, "finished")
- self.assertEqual(
- ext.stats._stats,
- {
- "start_time": fixed_datetime,
- "finish_time": fixed_datetime,
- "item_scraped_count": 1,
- "response_received_count": 1,
- "item_dropped_count": 1,
- "item_dropped_reasons_count/ZeroDivisionError": 1,
- "finish_reason": "finished",
- "elapsed_time_seconds": 0.0,
- },
- )
+ assert ext.stats._stats == {
+ "start_time": fixed_datetime,
+ "finish_time": fixed_datetime,
+ "item_scraped_count": 1,
+ "response_received_count": 1,
+ "item_dropped_count": 1,
+ "item_dropped_reasons_count/ZeroDivisionError": 1,
+ "finish_reason": "finished",
+ "elapsed_time_seconds": 0.0,
+ }
def test_core_stats_dummy_stats_collector(self):
self.crawler.stats = DummyStatsCollector(self.crawler)
@@ -46,51 +42,51 @@ class CoreStatsExtensionTest(unittest.TestCase):
ext.response_received(self.spider)
ext.item_dropped({}, self.spider, ZeroDivisionError())
ext.spider_closed(self.spider, "finished")
- self.assertEqual(ext.stats._stats, {})
+ assert ext.stats._stats == {}
-class StatsCollectorTest(unittest.TestCase):
- def setUp(self):
+class TestStatsCollector:
+ def setup_method(self):
self.crawler = get_crawler(Spider)
self.spider = self.crawler._create_spider("foo")
def test_collector(self):
stats = StatsCollector(self.crawler)
- self.assertEqual(stats.get_stats(), {})
- self.assertEqual(stats.get_value("anything"), None)
- self.assertEqual(stats.get_value("anything", "default"), "default")
+ assert stats.get_stats() == {}
+ assert stats.get_value("anything") is None
+ assert stats.get_value("anything", "default") == "default"
stats.set_value("test", "value")
- self.assertEqual(stats.get_stats(), {"test": "value"})
+ assert stats.get_stats() == {"test": "value"}
stats.set_value("test2", 23)
- self.assertEqual(stats.get_stats(), {"test": "value", "test2": 23})
- self.assertEqual(stats.get_value("test2"), 23)
+ assert stats.get_stats() == {"test": "value", "test2": 23}
+ assert stats.get_value("test2") == 23
stats.inc_value("test2")
- self.assertEqual(stats.get_value("test2"), 24)
+ assert stats.get_value("test2") == 24
stats.inc_value("test2", 6)
- self.assertEqual(stats.get_value("test2"), 30)
+ assert stats.get_value("test2") == 30
stats.max_value("test2", 6)
- self.assertEqual(stats.get_value("test2"), 30)
+ assert stats.get_value("test2") == 30
stats.max_value("test2", 40)
- self.assertEqual(stats.get_value("test2"), 40)
+ assert stats.get_value("test2") == 40
stats.max_value("test3", 1)
- self.assertEqual(stats.get_value("test3"), 1)
+ assert stats.get_value("test3") == 1
stats.min_value("test2", 60)
- self.assertEqual(stats.get_value("test2"), 40)
+ assert stats.get_value("test2") == 40
stats.min_value("test2", 35)
- self.assertEqual(stats.get_value("test2"), 35)
+ assert stats.get_value("test2") == 35
stats.min_value("test4", 7)
- self.assertEqual(stats.get_value("test4"), 7)
+ assert stats.get_value("test4") == 7
def test_dummy_collector(self):
stats = DummyStatsCollector(self.crawler)
- self.assertEqual(stats.get_stats(), {})
- self.assertEqual(stats.get_value("anything"), None)
- self.assertEqual(stats.get_value("anything", "default"), "default")
+ assert stats.get_stats() == {}
+ assert stats.get_value("anything") is None
+ assert stats.get_value("anything", "default") == "default"
stats.set_value("test", "value")
stats.inc_value("v1")
stats.max_value("v2", 100)
stats.min_value("v3", 100)
stats.open_spider("a")
stats.set_value("test", "value", spider=self.spider)
- self.assertEqual(stats.get_stats(), {})
- self.assertEqual(stats.get_stats("a"), {})
+ assert stats.get_stats() == {}
+ assert stats.get_stats("a") == {}