mirror of https://github.com/scrapy/scrapy.git
Converting tests to plain asserts, part 4. (#6702)
This commit is contained in:
Andrey Rakhmatullin
GitHub
parent
18f912b78f
commit
3ded1dfe31
|
|
@ -31,7 +31,7 @@ from scrapy.utils.test import get_crawler
|
|||
from tests import get_testdata, tests_datadir
|
||||
|
||||
|
||||
class SpiderTest(unittest.TestCase):
|
||||
class TestSpider(unittest.TestCase):
|
||||
spider_class = Spider
|
||||
|
||||
def setUp(self):
|
||||
|
|
@ -42,19 +42,19 @@ class SpiderTest(unittest.TestCase):
|
|||
|
||||
def test_base_spider(self):
|
||||
spider = self.spider_class("example.com")
|
||||
self.assertEqual(spider.name, "example.com")
|
||||
self.assertEqual(spider.start_urls, [])
|
||||
assert spider.name == "example.com"
|
||||
assert spider.start_urls == [] # pylint: disable=use-implicit-booleaness-not-comparison
|
||||
|
||||
def test_start_requests(self):
|
||||
spider = self.spider_class("example.com")
|
||||
start_requests = spider.start_requests()
|
||||
self.assertTrue(inspect.isgenerator(start_requests))
|
||||
self.assertEqual(list(start_requests), [])
|
||||
assert inspect.isgenerator(start_requests)
|
||||
assert not list(start_requests)
|
||||
|
||||
def test_spider_args(self):
|
||||
"""``__init__`` method arguments are assigned to spider attributes"""
|
||||
spider = self.spider_class("example.com", foo="bar")
|
||||
self.assertEqual(spider.foo, "bar")
|
||||
assert spider.foo == "bar"
|
||||
|
||||
def test_spider_without_name(self):
|
||||
"""``__init__`` method arguments are assigned to spider attributes"""
|
||||
|
|
@ -67,10 +67,10 @@ class SpiderTest(unittest.TestCase):
|
|||
def test_from_crawler_crawler_and_settings_population(self):
|
||||
crawler = get_crawler()
|
||||
spider = self.spider_class.from_crawler(crawler, "example.com")
|
||||
self.assertTrue(hasattr(spider, "crawler"))
|
||||
self.assertIs(spider.crawler, crawler)
|
||||
self.assertTrue(hasattr(spider, "settings"))
|
||||
self.assertIs(spider.settings, crawler.settings)
|
||||
assert hasattr(spider, "crawler")
|
||||
assert spider.crawler is crawler
|
||||
assert hasattr(spider, "settings")
|
||||
assert spider.settings is crawler.settings
|
||||
|
||||
def test_from_crawler_init_call(self):
|
||||
with mock.patch.object(
|
||||
|
|
@ -92,7 +92,7 @@ class SpiderTest(unittest.TestCase):
|
|||
crawler.signals.send_catch_log(
|
||||
signal=signals.spider_closed, spider=spider, reason=None
|
||||
)
|
||||
self.assertTrue(spider.closed_called)
|
||||
assert spider.closed_called
|
||||
|
||||
def test_update_settings(self):
|
||||
spider_settings = {"TEST1": "spider", "TEST2": "spider"}
|
||||
|
|
@ -101,9 +101,9 @@ class SpiderTest(unittest.TestCase):
|
|||
settings = Settings(project_settings, priority="project")
|
||||
|
||||
self.spider_class.update_settings(settings)
|
||||
self.assertEqual(settings.get("TEST1"), "spider")
|
||||
self.assertEqual(settings.get("TEST2"), "spider")
|
||||
self.assertEqual(settings.get("TEST3"), "project")
|
||||
assert settings.get("TEST1") == "spider"
|
||||
assert settings.get("TEST2") == "spider"
|
||||
assert settings.get("TEST3") == "project"
|
||||
|
||||
@inlineCallbacks
|
||||
def test_settings_in_from_crawler(self):
|
||||
|
|
@ -121,11 +121,11 @@ class SpiderTest(unittest.TestCase):
|
|||
return spider
|
||||
|
||||
crawler = Crawler(TestSpider, project_settings)
|
||||
self.assertEqual(crawler.settings.get("TEST1"), "spider")
|
||||
self.assertEqual(crawler.settings.get("TEST2"), "spider")
|
||||
self.assertEqual(crawler.settings.get("TEST3"), "project")
|
||||
assert crawler.settings.get("TEST1") == "spider"
|
||||
assert crawler.settings.get("TEST2") == "spider"
|
||||
assert crawler.settings.get("TEST3") == "project"
|
||||
yield crawler.crawl()
|
||||
self.assertEqual(crawler.settings.get("TEST1"), "spider_instance")
|
||||
assert crawler.settings.get("TEST1") == "spider_instance"
|
||||
|
||||
def test_logger(self):
|
||||
spider = self.spider_class("example.com")
|
||||
|
|
@ -134,8 +134,8 @@ class SpiderTest(unittest.TestCase):
|
|||
lc.check(("example.com", "INFO", "test log msg"))
|
||||
|
||||
record = lc.records[0]
|
||||
self.assertIn("spider", record.__dict__)
|
||||
self.assertIs(record.spider, spider)
|
||||
assert "spider" in record.__dict__
|
||||
assert record.spider is spider
|
||||
|
||||
def test_log(self):
|
||||
spider = self.spider_class("example.com")
|
||||
|
|
@ -144,11 +144,11 @@ class SpiderTest(unittest.TestCase):
|
|||
mock_logger.log.assert_called_once_with("INFO", "test log msg")
|
||||
|
||||
|
||||
class InitSpiderTest(SpiderTest):
|
||||
class TestInitSpider(TestSpider):
|
||||
spider_class = InitSpider
|
||||
|
||||
|
||||
class XMLFeedSpiderTest(SpiderTest):
|
||||
class TestXMLFeedSpider(TestSpider):
|
||||
spider_class = XMLFeedSpider
|
||||
|
||||
def test_register_namespace(self):
|
||||
|
|
@ -180,28 +180,24 @@ class XMLFeedSpiderTest(SpiderTest):
|
|||
for iterator in ("iternodes", "xml"):
|
||||
spider = _XMLSpider("example", iterator=iterator)
|
||||
output = list(spider._parse(response))
|
||||
self.assertEqual(len(output), 2, iterator)
|
||||
self.assertEqual(
|
||||
output,
|
||||
[
|
||||
{
|
||||
"loc": ["http://www.example.com/Special-Offers.html"],
|
||||
"updated": ["2009-08-16"],
|
||||
"custom": ["fuu"],
|
||||
"other": ["bar"],
|
||||
},
|
||||
{
|
||||
"loc": [],
|
||||
"updated": ["2009-08-16"],
|
||||
"other": ["foo"],
|
||||
"custom": [],
|
||||
},
|
||||
],
|
||||
iterator,
|
||||
)
|
||||
assert len(output) == 2, iterator
|
||||
assert output == [
|
||||
{
|
||||
"loc": ["http://www.example.com/Special-Offers.html"],
|
||||
"updated": ["2009-08-16"],
|
||||
"custom": ["fuu"],
|
||||
"other": ["bar"],
|
||||
},
|
||||
{
|
||||
"loc": [],
|
||||
"updated": ["2009-08-16"],
|
||||
"other": ["foo"],
|
||||
"custom": [],
|
||||
},
|
||||
], iterator
|
||||
|
||||
|
||||
class CSVFeedSpiderTest(SpiderTest):
|
||||
class TestCSVFeedSpider(TestSpider):
|
||||
spider_class = CSVFeedSpider
|
||||
|
||||
def test_parse_rows(self):
|
||||
|
|
@ -222,7 +218,7 @@ class CSVFeedSpiderTest(SpiderTest):
|
|||
assert len(rows) == 4
|
||||
|
||||
|
||||
class CrawlSpiderTest(SpiderTest):
|
||||
class TestCrawlSpider(TestSpider):
|
||||
test_body = b"""<html><head><title>Page title<title>
|
||||
<body>
|
||||
<p><a href="item/12.html">Item 12</a></p>
|
||||
|
|
@ -247,16 +243,13 @@ class CrawlSpiderTest(SpiderTest):
|
|||
|
||||
spider = _CrawlSpider()
|
||||
output = list(spider._requests_to_follow(response))
|
||||
self.assertEqual(len(output), 3)
|
||||
self.assertTrue(all(isinstance(r, Request) for r in output))
|
||||
self.assertEqual(
|
||||
[r.url for r in output],
|
||||
[
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
"http://example.org/nofollow.html",
|
||||
],
|
||||
)
|
||||
assert len(output) == 3
|
||||
assert all(isinstance(r, Request) for r in output)
|
||||
assert [r.url for r in output] == [
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
"http://example.org/nofollow.html",
|
||||
]
|
||||
|
||||
def test_process_links(self):
|
||||
response = HtmlResponse(
|
||||
|
|
@ -273,16 +266,13 @@ class CrawlSpiderTest(SpiderTest):
|
|||
|
||||
spider = _CrawlSpider()
|
||||
output = list(spider._requests_to_follow(response))
|
||||
self.assertEqual(len(output), 3)
|
||||
self.assertTrue(all(isinstance(r, Request) for r in output))
|
||||
self.assertEqual(
|
||||
[r.url for r in output],
|
||||
[
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
"http://example.org/nofollow.html",
|
||||
],
|
||||
)
|
||||
assert len(output) == 3
|
||||
assert all(isinstance(r, Request) for r in output)
|
||||
assert [r.url for r in output] == [
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
"http://example.org/nofollow.html",
|
||||
]
|
||||
|
||||
def test_process_links_filter(self):
|
||||
response = HtmlResponse(
|
||||
|
|
@ -302,15 +292,12 @@ class CrawlSpiderTest(SpiderTest):
|
|||
|
||||
spider = _CrawlSpider()
|
||||
output = list(spider._requests_to_follow(response))
|
||||
self.assertEqual(len(output), 2)
|
||||
self.assertTrue(all(isinstance(r, Request) for r in output))
|
||||
self.assertEqual(
|
||||
[r.url for r in output],
|
||||
[
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
],
|
||||
)
|
||||
assert len(output) == 2
|
||||
assert all(isinstance(r, Request) for r in output)
|
||||
assert [r.url for r in output] == [
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
]
|
||||
|
||||
def test_process_links_generator(self):
|
||||
response = HtmlResponse(
|
||||
|
|
@ -327,16 +314,13 @@ class CrawlSpiderTest(SpiderTest):
|
|||
|
||||
spider = _CrawlSpider()
|
||||
output = list(spider._requests_to_follow(response))
|
||||
self.assertEqual(len(output), 3)
|
||||
self.assertTrue(all(isinstance(r, Request) for r in output))
|
||||
self.assertEqual(
|
||||
[r.url for r in output],
|
||||
[
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
"http://example.org/nofollow.html",
|
||||
],
|
||||
)
|
||||
assert len(output) == 3
|
||||
assert all(isinstance(r, Request) for r in output)
|
||||
assert [r.url for r in output] == [
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
"http://example.org/nofollow.html",
|
||||
]
|
||||
|
||||
def test_process_request(self):
|
||||
response = HtmlResponse(
|
||||
|
|
@ -355,16 +339,13 @@ class CrawlSpiderTest(SpiderTest):
|
|||
|
||||
spider = _CrawlSpider()
|
||||
output = list(spider._requests_to_follow(response))
|
||||
self.assertEqual(len(output), 3)
|
||||
self.assertTrue(all(isinstance(r, Request) for r in output))
|
||||
self.assertEqual(
|
||||
[r.url for r in output],
|
||||
[
|
||||
"http://example.com/somepage/item/12.html",
|
||||
"http://example.com/about.html",
|
||||
"http://example.com/nofollow.html",
|
||||
],
|
||||
)
|
||||
assert len(output) == 3
|
||||
assert all(isinstance(r, Request) for r in output)
|
||||
assert [r.url for r in output] == [
|
||||
"http://example.com/somepage/item/12.html",
|
||||
"http://example.com/about.html",
|
||||
"http://example.com/nofollow.html",
|
||||
]
|
||||
|
||||
def test_process_request_with_response(self):
|
||||
response = HtmlResponse(
|
||||
|
|
@ -386,20 +367,18 @@ class CrawlSpiderTest(SpiderTest):
|
|||
|
||||
spider = _CrawlSpider()
|
||||
output = list(spider._requests_to_follow(response))
|
||||
self.assertEqual(len(output), 3)
|
||||
self.assertTrue(all(isinstance(r, Request) for r in output))
|
||||
self.assertEqual(
|
||||
[r.url for r in output],
|
||||
[
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
"http://example.org/nofollow.html",
|
||||
],
|
||||
)
|
||||
self.assertEqual(
|
||||
[r.meta["response_class"] for r in output],
|
||||
["HtmlResponse", "HtmlResponse", "HtmlResponse"],
|
||||
)
|
||||
assert len(output) == 3
|
||||
assert all(isinstance(r, Request) for r in output)
|
||||
assert [r.url for r in output] == [
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
"http://example.org/nofollow.html",
|
||||
]
|
||||
assert [r.meta["response_class"] for r in output] == [
|
||||
"HtmlResponse",
|
||||
"HtmlResponse",
|
||||
"HtmlResponse",
|
||||
]
|
||||
|
||||
def test_process_request_instance_method(self):
|
||||
response = HtmlResponse(
|
||||
|
|
@ -416,16 +395,13 @@ class CrawlSpiderTest(SpiderTest):
|
|||
|
||||
spider = _CrawlSpider()
|
||||
output = list(spider._requests_to_follow(response))
|
||||
self.assertEqual(len(output), 3)
|
||||
self.assertTrue(all(isinstance(r, Request) for r in output))
|
||||
self.assertEqual(
|
||||
[r.url for r in output],
|
||||
[
|
||||
safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"),
|
||||
safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"),
|
||||
safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"),
|
||||
],
|
||||
)
|
||||
assert len(output) == 3
|
||||
assert all(isinstance(r, Request) for r in output)
|
||||
assert [r.url for r in output] == [
|
||||
safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"),
|
||||
safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"),
|
||||
safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"),
|
||||
]
|
||||
|
||||
def test_process_request_instance_method_with_response(self):
|
||||
response = HtmlResponse(
|
||||
|
|
@ -448,32 +424,30 @@ class CrawlSpiderTest(SpiderTest):
|
|||
|
||||
spider = _CrawlSpider()
|
||||
output = list(spider._requests_to_follow(response))
|
||||
self.assertEqual(len(output), 3)
|
||||
self.assertTrue(all(isinstance(r, Request) for r in output))
|
||||
self.assertEqual(
|
||||
[r.url for r in output],
|
||||
[
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
"http://example.org/nofollow.html",
|
||||
],
|
||||
)
|
||||
self.assertEqual(
|
||||
[r.meta["response_class"] for r in output],
|
||||
["HtmlResponse", "HtmlResponse", "HtmlResponse"],
|
||||
)
|
||||
assert len(output) == 3
|
||||
assert all(isinstance(r, Request) for r in output)
|
||||
assert [r.url for r in output] == [
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
"http://example.org/nofollow.html",
|
||||
]
|
||||
assert [r.meta["response_class"] for r in output] == [
|
||||
"HtmlResponse",
|
||||
"HtmlResponse",
|
||||
"HtmlResponse",
|
||||
]
|
||||
|
||||
def test_follow_links_attribute_population(self):
|
||||
crawler = get_crawler()
|
||||
spider = self.spider_class.from_crawler(crawler, "example.com")
|
||||
self.assertTrue(hasattr(spider, "_follow_links"))
|
||||
self.assertTrue(spider._follow_links)
|
||||
assert hasattr(spider, "_follow_links")
|
||||
assert spider._follow_links
|
||||
|
||||
settings_dict = {"CRAWLSPIDER_FOLLOW_LINKS": False}
|
||||
crawler = get_crawler(settings_dict=settings_dict)
|
||||
spider = self.spider_class.from_crawler(crawler, "example.com")
|
||||
self.assertTrue(hasattr(spider, "_follow_links"))
|
||||
self.assertFalse(spider._follow_links)
|
||||
assert hasattr(spider, "_follow_links")
|
||||
assert not spider._follow_links
|
||||
|
||||
def test_start_url(self):
|
||||
spider = self.spider_class("example.com")
|
||||
|
|
@ -483,7 +457,7 @@ class CrawlSpiderTest(SpiderTest):
|
|||
list(spider.start_requests())
|
||||
|
||||
|
||||
class SitemapSpiderTest(SpiderTest):
|
||||
class TestSitemapSpider(TestSpider):
|
||||
spider_class = SitemapSpider
|
||||
|
||||
BODY = b"SITEMAP"
|
||||
|
|
@ -496,7 +470,7 @@ class SitemapSpiderTest(SpiderTest):
|
|||
def assertSitemapBody(self, response, body):
|
||||
crawler = get_crawler()
|
||||
spider = self.spider_class.from_crawler(crawler, "example.com")
|
||||
self.assertEqual(spider._get_sitemap_body(response), body)
|
||||
assert spider._get_sitemap_body(response) == body
|
||||
|
||||
def test_get_sitemap_body(self):
|
||||
r = XmlResponse(url="http://www.example.com/", body=self.BODY)
|
||||
|
|
@ -543,15 +517,12 @@ Sitemap: /sitemap-relative-url.xml
|
|||
|
||||
r = TextResponse(url="http://www.example.com/robots.txt", body=robots)
|
||||
spider = self.spider_class("example.com")
|
||||
self.assertEqual(
|
||||
[req.url for req in spider._parse_sitemap(r)],
|
||||
[
|
||||
"http://example.com/sitemap.xml",
|
||||
"http://example.com/sitemap-product-index.xml",
|
||||
"http://example.com/sitemap-uppercase.xml",
|
||||
"http://www.example.com/sitemap-relative-url.xml",
|
||||
],
|
||||
)
|
||||
assert [req.url for req in spider._parse_sitemap(r)] == [
|
||||
"http://example.com/sitemap.xml",
|
||||
"http://example.com/sitemap-product-index.xml",
|
||||
"http://example.com/sitemap-uppercase.xml",
|
||||
"http://www.example.com/sitemap-relative-url.xml",
|
||||
]
|
||||
|
||||
def test_alternate_url_locs(self):
|
||||
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
|
@ -570,21 +541,17 @@ Sitemap: /sitemap-relative-url.xml
|
|||
</urlset>"""
|
||||
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
|
||||
spider = self.spider_class("example.com")
|
||||
self.assertEqual(
|
||||
[req.url for req in spider._parse_sitemap(r)],
|
||||
["http://www.example.com/english/"],
|
||||
)
|
||||
assert [req.url for req in spider._parse_sitemap(r)] == [
|
||||
"http://www.example.com/english/"
|
||||
]
|
||||
|
||||
spider.sitemap_alternate_links = True
|
||||
self.assertEqual(
|
||||
[req.url for req in spider._parse_sitemap(r)],
|
||||
[
|
||||
"http://www.example.com/english/",
|
||||
"http://www.example.com/deutsch/",
|
||||
"http://www.example.com/schweiz-deutsch/",
|
||||
"http://www.example.com/italiano/",
|
||||
],
|
||||
)
|
||||
assert [req.url for req in spider._parse_sitemap(r)] == [
|
||||
"http://www.example.com/english/",
|
||||
"http://www.example.com/deutsch/",
|
||||
"http://www.example.com/schweiz-deutsch/",
|
||||
"http://www.example.com/italiano/",
|
||||
]
|
||||
|
||||
def test_sitemap_filter(self):
|
||||
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
|
@ -611,16 +578,15 @@ Sitemap: /sitemap-relative-url.xml
|
|||
|
||||
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
|
||||
spider = self.spider_class("example.com")
|
||||
self.assertEqual(
|
||||
[req.url for req in spider._parse_sitemap(r)],
|
||||
["http://www.example.com/english/", "http://www.example.com/portuguese/"],
|
||||
)
|
||||
assert [req.url for req in spider._parse_sitemap(r)] == [
|
||||
"http://www.example.com/english/",
|
||||
"http://www.example.com/portuguese/",
|
||||
]
|
||||
|
||||
spider = FilteredSitemapSpider("example.com")
|
||||
self.assertEqual(
|
||||
[req.url for req in spider._parse_sitemap(r)],
|
||||
["http://www.example.com/english/"],
|
||||
)
|
||||
assert [req.url for req in spider._parse_sitemap(r)] == [
|
||||
"http://www.example.com/english/"
|
||||
]
|
||||
|
||||
def test_sitemap_filter_with_alternate_links(self):
|
||||
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
|
@ -649,19 +615,15 @@ Sitemap: /sitemap-relative-url.xml
|
|||
|
||||
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
|
||||
spider = self.spider_class("example.com")
|
||||
self.assertEqual(
|
||||
[req.url for req in spider._parse_sitemap(r)],
|
||||
[
|
||||
"http://www.example.com/english/article_1/",
|
||||
"http://www.example.com/english/article_2/",
|
||||
],
|
||||
)
|
||||
assert [req.url for req in spider._parse_sitemap(r)] == [
|
||||
"http://www.example.com/english/article_1/",
|
||||
"http://www.example.com/english/article_2/",
|
||||
]
|
||||
|
||||
spider = FilteredSitemapSpider("example.com")
|
||||
self.assertEqual(
|
||||
[req.url for req in spider._parse_sitemap(r)],
|
||||
["http://www.example.com/deutsch/article_1/"],
|
||||
)
|
||||
assert [req.url for req in spider._parse_sitemap(r)] == [
|
||||
"http://www.example.com/deutsch/article_1/"
|
||||
]
|
||||
|
||||
def test_sitemapindex_filter(self):
|
||||
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
|
@ -689,19 +651,15 @@ Sitemap: /sitemap-relative-url.xml
|
|||
|
||||
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
|
||||
spider = self.spider_class("example.com")
|
||||
self.assertEqual(
|
||||
[req.url for req in spider._parse_sitemap(r)],
|
||||
[
|
||||
"http://www.example.com/sitemap1.xml",
|
||||
"http://www.example.com/sitemap2.xml",
|
||||
],
|
||||
)
|
||||
assert [req.url for req in spider._parse_sitemap(r)] == [
|
||||
"http://www.example.com/sitemap1.xml",
|
||||
"http://www.example.com/sitemap2.xml",
|
||||
]
|
||||
|
||||
spider = FilteredSitemapSpider("example.com")
|
||||
self.assertEqual(
|
||||
[req.url for req in spider._parse_sitemap(r)],
|
||||
["http://www.example.com/sitemap2.xml"],
|
||||
)
|
||||
assert [req.url for req in spider._parse_sitemap(r)] == [
|
||||
"http://www.example.com/sitemap2.xml"
|
||||
]
|
||||
|
||||
def test_compression_bomb_setting(self):
|
||||
settings = {"DOWNLOAD_MAXSIZE": 10_000_000}
|
||||
|
|
@ -711,7 +669,7 @@ Sitemap: /sitemap-relative-url.xml
|
|||
body = body_path.read_bytes()
|
||||
request = Request(url="https://example.com")
|
||||
response = Response(url="https://example.com", body=body, request=request)
|
||||
self.assertIsNone(spider._get_sitemap_body(response))
|
||||
assert spider._get_sitemap_body(response) is None
|
||||
|
||||
def test_compression_bomb_spider_attr(self):
|
||||
class DownloadMaxSizeSpider(self.spider_class):
|
||||
|
|
@ -723,7 +681,7 @@ Sitemap: /sitemap-relative-url.xml
|
|||
body = body_path.read_bytes()
|
||||
request = Request(url="https://example.com")
|
||||
response = Response(url="https://example.com", body=body, request=request)
|
||||
self.assertIsNone(spider._get_sitemap_body(response))
|
||||
assert spider._get_sitemap_body(response) is None
|
||||
|
||||
def test_compression_bomb_request_meta(self):
|
||||
crawler = get_crawler()
|
||||
|
|
@ -734,7 +692,7 @@ Sitemap: /sitemap-relative-url.xml
|
|||
url="https://example.com", meta={"download_maxsize": 10_000_000}
|
||||
)
|
||||
response = Response(url="https://example.com", body=body, request=request)
|
||||
self.assertIsNone(spider._get_sitemap_body(response))
|
||||
assert spider._get_sitemap_body(response) is None
|
||||
|
||||
def test_download_warnsize_setting(self):
|
||||
settings = {"DOWNLOAD_WARNSIZE": 10_000_000}
|
||||
|
|
@ -814,13 +772,13 @@ Sitemap: /sitemap-relative-url.xml
|
|||
)
|
||||
|
||||
|
||||
class DeprecationTest(unittest.TestCase):
|
||||
class TestDeprecation:
|
||||
def test_crawl_spider(self):
|
||||
assert issubclass(CrawlSpider, Spider)
|
||||
assert isinstance(CrawlSpider(name="foo"), Spider)
|
||||
|
||||
|
||||
class NoParseMethodSpiderTest(unittest.TestCase):
|
||||
class TestNoParseMethodSpider:
|
||||
spider_class = Spider
|
||||
|
||||
def test_undefined_parse_method(self):
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ from tempfile import mkdtemp
|
|||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
from twisted.trial import unittest
|
||||
from zope.interface.verify import verifyObject
|
||||
|
||||
# ugly hack to avoid cyclic imports of scrapy.spiders when running this test
|
||||
|
|
@ -28,8 +27,8 @@ def _copytree(source: Path, target: Path):
|
|||
shutil.copytree(source, target)
|
||||
|
||||
|
||||
class SpiderLoaderTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
class TestSpiderLoader:
|
||||
def setup_method(self):
|
||||
orig_spiders_dir = module_dir / "test_spiders"
|
||||
self.tmpdir = Path(tempfile.mkdtemp())
|
||||
self.spiders_dir = self.tmpdir / "test_spiders_xxx"
|
||||
|
|
@ -38,7 +37,7 @@ class SpiderLoaderTest(unittest.TestCase):
|
|||
settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})
|
||||
self.spider_loader = SpiderLoader.from_settings(settings)
|
||||
|
||||
def tearDown(self):
|
||||
def teardown_method(self):
|
||||
del self.spider_loader
|
||||
del sys.modules["test_spiders_xxx"]
|
||||
sys.path.remove(str(self.tmpdir))
|
||||
|
|
@ -47,37 +46,35 @@ class SpiderLoaderTest(unittest.TestCase):
|
|||
verifyObject(ISpiderLoader, self.spider_loader)
|
||||
|
||||
def test_list(self):
|
||||
self.assertEqual(
|
||||
set(self.spider_loader.list()), {"spider1", "spider2", "spider3", "spider4"}
|
||||
)
|
||||
assert set(self.spider_loader.list()) == {
|
||||
"spider1",
|
||||
"spider2",
|
||||
"spider3",
|
||||
"spider4",
|
||||
}
|
||||
|
||||
def test_load(self):
|
||||
spider1 = self.spider_loader.load("spider1")
|
||||
self.assertEqual(spider1.__name__, "Spider1")
|
||||
assert spider1.__name__ == "Spider1"
|
||||
|
||||
def test_find_by_request(self):
|
||||
self.assertEqual(
|
||||
self.spider_loader.find_by_request(Request("http://scrapy1.org/test")),
|
||||
["spider1"],
|
||||
)
|
||||
self.assertEqual(
|
||||
self.spider_loader.find_by_request(Request("http://scrapy2.org/test")),
|
||||
["spider2"],
|
||||
)
|
||||
self.assertEqual(
|
||||
set(self.spider_loader.find_by_request(Request("http://scrapy3.org/test"))),
|
||||
{"spider1", "spider2"},
|
||||
)
|
||||
self.assertEqual(
|
||||
self.spider_loader.find_by_request(Request("http://scrapy999.org/test")), []
|
||||
)
|
||||
self.assertEqual(
|
||||
self.spider_loader.find_by_request(Request("http://spider3.com")), []
|
||||
)
|
||||
self.assertEqual(
|
||||
self.spider_loader.find_by_request(Request("http://spider3.com/onlythis")),
|
||||
["spider3"],
|
||||
assert self.spider_loader.find_by_request(
|
||||
Request("http://scrapy1.org/test")
|
||||
) == ["spider1"]
|
||||
assert self.spider_loader.find_by_request(
|
||||
Request("http://scrapy2.org/test")
|
||||
) == ["spider2"]
|
||||
assert set(
|
||||
self.spider_loader.find_by_request(Request("http://scrapy3.org/test"))
|
||||
) == {"spider1", "spider2"}
|
||||
assert (
|
||||
self.spider_loader.find_by_request(Request("http://scrapy999.org/test"))
|
||||
== []
|
||||
)
|
||||
assert self.spider_loader.find_by_request(Request("http://spider3.com")) == []
|
||||
assert self.spider_loader.find_by_request(
|
||||
Request("http://spider3.com/onlythis")
|
||||
) == ["spider3"]
|
||||
|
||||
def test_load_spider_module(self):
|
||||
module = "tests.test_spiderloader.test_spiders.spider1"
|
||||
|
|
@ -113,9 +110,9 @@ class SpiderLoaderTest(unittest.TestCase):
|
|||
runner = CrawlerRunner({"ADDONS": {SpiderModuleAddon: 1}})
|
||||
|
||||
crawler = runner.create_crawler("spider_from_addon")
|
||||
self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
|
||||
self.assertEqual(crawler.spidercls.name, "spider_from_addon")
|
||||
self.assertTrue(len(crawler.settings["SPIDER_MODULES"]) == 1)
|
||||
assert issubclass(crawler.spidercls, scrapy.Spider)
|
||||
assert crawler.spidercls.name == "spider_from_addon"
|
||||
assert len(crawler.settings["SPIDER_MODULES"]) == 1
|
||||
|
||||
def test_crawler_runner_loading(self):
|
||||
module = "tests.test_spiderloader.test_spiders.spider1"
|
||||
|
|
@ -129,8 +126,8 @@ class SpiderLoaderTest(unittest.TestCase):
|
|||
runner.create_crawler("spider2")
|
||||
|
||||
crawler = runner.create_crawler("spider1")
|
||||
self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
|
||||
self.assertEqual(crawler.spidercls.name, "spider1")
|
||||
assert issubclass(crawler.spidercls, scrapy.Spider)
|
||||
assert crawler.spidercls.name == "spider1"
|
||||
|
||||
def test_bad_spider_modules_exception(self):
|
||||
module = "tests.test_spiderloader.test_spiders.doesnotexist"
|
||||
|
|
@ -150,10 +147,10 @@ class SpiderLoaderTest(unittest.TestCase):
|
|||
# at least until all six versions we can import (including botocore.vendored.six)
|
||||
# are updated to 1.16.0+
|
||||
w.pop(0)
|
||||
self.assertIn("Could not load spiders from module", str(w[0].message))
|
||||
assert "Could not load spiders from module" in str(w[0].message)
|
||||
|
||||
spiders = spider_loader.list()
|
||||
self.assertEqual(spiders, [])
|
||||
assert not spiders
|
||||
|
||||
def test_syntax_error_exception(self):
|
||||
module = "tests.test_spiderloader.test_spiders.spider1"
|
||||
|
|
@ -179,14 +176,14 @@ class SpiderLoaderTest(unittest.TestCase):
|
|||
# at least until all six versions we can import (including botocore.vendored.six)
|
||||
# are updated to 1.16.0+
|
||||
w.pop(0)
|
||||
self.assertIn("Could not load spiders from module", str(w[0].message))
|
||||
assert "Could not load spiders from module" in str(w[0].message)
|
||||
|
||||
spiders = spider_loader.list()
|
||||
self.assertEqual(spiders, [])
|
||||
assert not spiders
|
||||
|
||||
|
||||
class DuplicateSpiderNameLoaderTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
class TestDuplicateSpiderNameLoader:
|
||||
def setup_method(self):
|
||||
orig_spiders_dir = module_dir / "test_spiders"
|
||||
self.tmpdir = Path(mkdtemp())
|
||||
self.spiders_dir = self.tmpdir / "test_spiders_xxx"
|
||||
|
|
@ -194,7 +191,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
|
|||
sys.path.append(str(self.tmpdir))
|
||||
self.settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})
|
||||
|
||||
def tearDown(self):
|
||||
def teardown_method(self):
|
||||
del sys.modules["test_spiders_xxx"]
|
||||
sys.path.remove(str(self.tmpdir))
|
||||
|
||||
|
|
@ -208,18 +205,18 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
|
|||
with warnings.catch_warnings(record=True) as w:
|
||||
spider_loader = SpiderLoader.from_settings(self.settings)
|
||||
|
||||
self.assertEqual(len(w), 1)
|
||||
assert len(w) == 1
|
||||
msg = str(w[0].message)
|
||||
self.assertIn("several spiders with the same name", msg)
|
||||
self.assertIn("'spider3'", msg)
|
||||
self.assertTrue(msg.count("'spider3'") == 2)
|
||||
assert "several spiders with the same name" in msg
|
||||
assert "'spider3'" in msg
|
||||
assert msg.count("'spider3'") == 2
|
||||
|
||||
self.assertNotIn("'spider1'", msg)
|
||||
self.assertNotIn("'spider2'", msg)
|
||||
self.assertNotIn("'spider4'", msg)
|
||||
assert "'spider1'" not in msg
|
||||
assert "'spider2'" not in msg
|
||||
assert "'spider4'" not in msg
|
||||
|
||||
spiders = set(spider_loader.list())
|
||||
self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"})
|
||||
assert spiders == {"spider1", "spider2", "spider3", "spider4"}
|
||||
|
||||
def test_multiple_dupename_warning(self):
|
||||
# copy 2 spider modules so as to have duplicate spider name
|
||||
|
|
@ -236,17 +233,17 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
|
|||
with warnings.catch_warnings(record=True) as w:
|
||||
spider_loader = SpiderLoader.from_settings(self.settings)
|
||||
|
||||
self.assertEqual(len(w), 1)
|
||||
assert len(w) == 1
|
||||
msg = str(w[0].message)
|
||||
self.assertIn("several spiders with the same name", msg)
|
||||
self.assertIn("'spider1'", msg)
|
||||
self.assertTrue(msg.count("'spider1'") == 2)
|
||||
assert "several spiders with the same name" in msg
|
||||
assert "'spider1'" in msg
|
||||
assert msg.count("'spider1'") == 2
|
||||
|
||||
self.assertIn("'spider2'", msg)
|
||||
self.assertTrue(msg.count("'spider2'") == 2)
|
||||
assert "'spider2'" in msg
|
||||
assert msg.count("'spider2'") == 2
|
||||
|
||||
self.assertNotIn("'spider3'", msg)
|
||||
self.assertNotIn("'spider4'", msg)
|
||||
assert "'spider3'" not in msg
|
||||
assert "'spider4'" not in msg
|
||||
|
||||
spiders = set(spider_loader.list())
|
||||
self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"})
|
||||
assert spiders == {"spider1", "spider2", "spider3", "spider4"}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
|
|||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
class SpiderMiddlewareTestCase(TestCase):
|
||||
class TestSpiderMiddleware(TestCase):
|
||||
def setUp(self):
|
||||
self.request = Request("http://example.com/index.html")
|
||||
self.response = Response(self.request.url, request=self.request)
|
||||
|
|
@ -41,7 +41,7 @@ class SpiderMiddlewareTestCase(TestCase):
|
|||
return results[0]
|
||||
|
||||
|
||||
class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase):
|
||||
class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware):
|
||||
"""Invalid return value for process_spider_input method"""
|
||||
|
||||
def test_invalid_process_spider_input(self):
|
||||
|
|
@ -51,11 +51,11 @@ class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase):
|
|||
|
||||
self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware())
|
||||
result = self._scrape_response()
|
||||
self.assertIsInstance(result, Failure)
|
||||
self.assertIsInstance(result.value, _InvalidOutput)
|
||||
assert isinstance(result, Failure)
|
||||
assert isinstance(result.value, _InvalidOutput)
|
||||
|
||||
|
||||
class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase):
|
||||
class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware):
|
||||
"""Invalid return value for process_spider_output method"""
|
||||
|
||||
def test_invalid_process_spider_output(self):
|
||||
|
|
@ -65,11 +65,11 @@ class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase):
|
|||
|
||||
self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware())
|
||||
result = self._scrape_response()
|
||||
self.assertIsInstance(result, Failure)
|
||||
self.assertIsInstance(result.value, _InvalidOutput)
|
||||
assert isinstance(result, Failure)
|
||||
assert isinstance(result.value, _InvalidOutput)
|
||||
|
||||
|
||||
class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase):
|
||||
class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware):
|
||||
"""Invalid return value for process_spider_exception method"""
|
||||
|
||||
def test_invalid_process_spider_exception(self):
|
||||
|
|
@ -84,11 +84,11 @@ class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase):
|
|||
self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware())
|
||||
self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
|
||||
result = self._scrape_response()
|
||||
self.assertIsInstance(result, Failure)
|
||||
self.assertIsInstance(result.value, _InvalidOutput)
|
||||
assert isinstance(result, Failure)
|
||||
assert isinstance(result.value, _InvalidOutput)
|
||||
|
||||
|
||||
class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
|
||||
class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):
|
||||
"""Re raise the exception by returning None"""
|
||||
|
||||
def test_process_spider_exception_return_none(self):
|
||||
|
|
@ -103,11 +103,11 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
|
|||
self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware())
|
||||
self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
|
||||
result = self._scrape_response()
|
||||
self.assertIsInstance(result, Failure)
|
||||
self.assertIsInstance(result.value, ZeroDivisionError)
|
||||
assert isinstance(result, Failure)
|
||||
assert isinstance(result.value, ZeroDivisionError)
|
||||
|
||||
|
||||
class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
|
||||
class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
|
||||
"""Helpers for testing sync, async and mixed middlewares.
|
||||
|
||||
Should work for process_spider_output and, when it's supported, process_start_requests.
|
||||
|
|
@ -148,14 +148,13 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
|
|||
result = yield self._get_middleware_result(
|
||||
*mw_classes, start_index=start_index
|
||||
)
|
||||
self.assertIsInstance(result, Iterable)
|
||||
assert isinstance(result, Iterable)
|
||||
result_list = list(result)
|
||||
self.assertEqual(len(result_list), self.RESULT_COUNT)
|
||||
self.assertIsInstance(result_list[0], self.ITEM_TYPE)
|
||||
self.assertEqual("downgraded to a non-async" in str(log), downgrade)
|
||||
self.assertEqual(
|
||||
"doesn't support asynchronous spider output" in str(log),
|
||||
ProcessSpiderOutputSimpleMiddleware in mw_classes,
|
||||
assert len(result_list) == self.RESULT_COUNT
|
||||
assert isinstance(result_list[0], self.ITEM_TYPE)
|
||||
assert ("downgraded to a non-async" in str(log)) == downgrade
|
||||
assert ("doesn't support asynchronous spider output" in str(log)) == (
|
||||
ProcessSpiderOutputSimpleMiddleware in mw_classes
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
|
|
@ -166,11 +165,11 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
|
|||
result = yield self._get_middleware_result(
|
||||
*mw_classes, start_index=start_index
|
||||
)
|
||||
self.assertIsInstance(result, AsyncIterator)
|
||||
assert isinstance(result, AsyncIterator)
|
||||
result_list = yield deferred_from_coro(collect_asyncgen(result))
|
||||
self.assertEqual(len(result_list), self.RESULT_COUNT)
|
||||
self.assertIsInstance(result_list[0], self.ITEM_TYPE)
|
||||
self.assertEqual("downgraded to a non-async" in str(log), downgrade)
|
||||
assert len(result_list) == self.RESULT_COUNT
|
||||
assert isinstance(result_list[0], self.ITEM_TYPE)
|
||||
assert ("downgraded to a non-async" in str(log)) == downgrade
|
||||
|
||||
|
||||
class ProcessSpiderOutputSimpleMiddleware:
|
||||
|
|
@ -212,7 +211,7 @@ class ProcessSpiderExceptionAsyncIterableMiddleware:
|
|||
yield {"foo": 3}
|
||||
|
||||
|
||||
class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
|
||||
class TestProcessSpiderOutputSimple(TestBaseAsyncSpiderMiddleware):
|
||||
"""process_spider_output tests for simple callbacks"""
|
||||
|
||||
ITEM_TYPE = dict
|
||||
|
|
@ -257,7 +256,7 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
|
|||
return self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN)
|
||||
|
||||
|
||||
class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple):
|
||||
class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple):
|
||||
"""process_spider_output tests for async generator callbacks"""
|
||||
|
||||
async def _scrape_func(self, *args, **kwargs):
|
||||
|
|
@ -297,7 +296,7 @@ class ProcessSpiderOutputCoroutineMiddleware:
|
|||
return result
|
||||
|
||||
|
||||
class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
|
||||
class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware):
|
||||
@defer.inlineCallbacks
|
||||
def test_non_iterable(self):
|
||||
with pytest.raises(
|
||||
|
|
@ -324,7 +323,7 @@ class ProcessStartRequestsSimpleMiddleware:
|
|||
yield from start_requests
|
||||
|
||||
|
||||
class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase):
|
||||
class TestProcessStartRequestsSimple(TestBaseAsyncSpiderMiddleware):
|
||||
"""process_start_requests tests for simple start_requests"""
|
||||
|
||||
ITEM_TYPE = (Request, dict)
|
||||
|
|
@ -373,67 +372,65 @@ class UniversalMiddlewareBothAsync:
|
|||
yield
|
||||
|
||||
|
||||
class UniversalMiddlewareManagerTest(TestCase):
|
||||
def setUp(self):
|
||||
class TestUniversalMiddlewareManager:
|
||||
def setup_method(self):
|
||||
self.mwman = SpiderMiddlewareManager()
|
||||
|
||||
def test_simple_mw(self):
|
||||
mw = ProcessSpiderOutputSimpleMiddleware()
|
||||
self.mwman._add_middleware(mw)
|
||||
self.assertEqual(
|
||||
self.mwman.methods["process_spider_output"][0], mw.process_spider_output
|
||||
assert (
|
||||
self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable
|
||||
)
|
||||
|
||||
def test_async_mw(self):
|
||||
mw = ProcessSpiderOutputAsyncGenMiddleware()
|
||||
self.mwman._add_middleware(mw)
|
||||
self.assertEqual(
|
||||
self.mwman.methods["process_spider_output"][0], mw.process_spider_output
|
||||
assert (
|
||||
self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable
|
||||
)
|
||||
|
||||
def test_universal_mw(self):
|
||||
mw = ProcessSpiderOutputUniversalMiddleware()
|
||||
self.mwman._add_middleware(mw)
|
||||
self.assertEqual(
|
||||
self.mwman.methods["process_spider_output"][0],
|
||||
(mw.process_spider_output, mw.process_spider_output_async),
|
||||
assert self.mwman.methods["process_spider_output"][0] == (
|
||||
mw.process_spider_output,
|
||||
mw.process_spider_output_async,
|
||||
)
|
||||
|
||||
def test_universal_mw_no_sync(self):
|
||||
with LogCapture() as log:
|
||||
self.mwman._add_middleware(UniversalMiddlewareNoSync())
|
||||
self.assertIn(
|
||||
assert (
|
||||
"UniversalMiddlewareNoSync has process_spider_output_async"
|
||||
" without process_spider_output",
|
||||
str(log),
|
||||
" without process_spider_output" in str(log)
|
||||
)
|
||||
self.assertEqual(self.mwman.methods["process_spider_output"][0], None)
|
||||
assert self.mwman.methods["process_spider_output"][0] is None
|
||||
|
||||
def test_universal_mw_both_sync(self):
|
||||
mw = UniversalMiddlewareBothSync()
|
||||
with LogCapture() as log:
|
||||
self.mwman._add_middleware(mw)
|
||||
self.assertIn(
|
||||
assert (
|
||||
"UniversalMiddlewareBothSync.process_spider_output_async "
|
||||
"is not an async generator function",
|
||||
str(log),
|
||||
"is not an async generator function" in str(log)
|
||||
)
|
||||
self.assertEqual(
|
||||
self.mwman.methods["process_spider_output"][0], mw.process_spider_output
|
||||
assert (
|
||||
self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable
|
||||
)
|
||||
|
||||
def test_universal_mw_both_async(self):
|
||||
with LogCapture() as log:
|
||||
self.mwman._add_middleware(UniversalMiddlewareBothAsync())
|
||||
self.assertIn(
|
||||
assert (
|
||||
"UniversalMiddlewareBothAsync.process_spider_output "
|
||||
"is an async generator function while process_spider_output_async exists",
|
||||
str(log),
|
||||
"is an async generator function while process_spider_output_async exists"
|
||||
in str(log)
|
||||
)
|
||||
self.assertEqual(self.mwman.methods["process_spider_output"][0], None)
|
||||
assert self.mwman.methods["process_spider_output"][0] is None
|
||||
|
||||
|
||||
class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase):
|
||||
class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware):
|
||||
ITEM_TYPE = dict
|
||||
MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware
|
||||
MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
|
||||
|
|
@ -474,7 +471,7 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase):
|
|||
return self._test_simple_base(self.MW_UNIVERSAL)
|
||||
|
||||
|
||||
class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest):
|
||||
class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple):
|
||||
async def _scrape_func(self, *args, **kwargs):
|
||||
for item in super()._scrape_func():
|
||||
yield item
|
||||
|
|
@ -503,7 +500,7 @@ class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest):
|
|||
return self._test_asyncgen_base(self.MW_UNIVERSAL)
|
||||
|
||||
|
||||
class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase):
|
||||
class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):
|
||||
ITEM_TYPE = dict
|
||||
MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware
|
||||
MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
from unittest import TestCase
|
||||
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.spidermiddlewares.depth import DepthMiddleware
|
||||
from scrapy.spiders import Spider
|
||||
|
|
@ -7,8 +5,8 @@ from scrapy.statscollectors import StatsCollector
|
|||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
class TestDepthMiddleware(TestCase):
|
||||
def setUp(self):
|
||||
class TestDepthMiddleware:
|
||||
def setup_method(self):
|
||||
crawler = get_crawler(Spider)
|
||||
self.spider = crawler._create_spider("scrapytest.org")
|
||||
|
||||
|
|
@ -24,18 +22,18 @@ class TestDepthMiddleware(TestCase):
|
|||
result = [Request("http://scrapytest.org")]
|
||||
|
||||
out = list(self.mw.process_spider_output(resp, result, self.spider))
|
||||
self.assertEqual(out, result)
|
||||
assert out == result
|
||||
|
||||
rdc = self.stats.get_value("request_depth_count/1", spider=self.spider)
|
||||
self.assertEqual(rdc, 1)
|
||||
assert rdc == 1
|
||||
|
||||
req.meta["depth"] = 1
|
||||
|
||||
out2 = list(self.mw.process_spider_output(resp, result, self.spider))
|
||||
self.assertEqual(out2, [])
|
||||
assert not out2
|
||||
|
||||
rdm = self.stats.get_value("request_depth_max", spider=self.spider)
|
||||
self.assertEqual(rdm, 1)
|
||||
assert rdm == 1
|
||||
|
||||
def tearDown(self):
|
||||
def teardown_method(self):
|
||||
self.stats.close_spider(self.spider, "")
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
import logging
|
||||
from unittest import TestCase
|
||||
|
||||
import pytest
|
||||
from testfixtures import LogCapture
|
||||
from twisted.internet import defer
|
||||
from twisted.trial.unittest import TestCase as TrialTestCase
|
||||
from twisted.trial.unittest import TestCase
|
||||
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.settings import Settings
|
||||
|
|
@ -59,8 +58,8 @@ def _responses(request, status_codes):
|
|||
return responses
|
||||
|
||||
|
||||
class TestHttpErrorMiddleware(TestCase):
|
||||
def setUp(self):
|
||||
class TestHttpErrorMiddleware:
|
||||
def setup_method(self):
|
||||
crawler = get_crawler(Spider)
|
||||
self.spider = Spider.from_crawler(crawler, name="foo")
|
||||
self.mw = HttpErrorMiddleware(Settings({}))
|
||||
|
|
@ -68,19 +67,20 @@ class TestHttpErrorMiddleware(TestCase):
|
|||
self.res200, self.res404 = _responses(self.req, [200, 404])
|
||||
|
||||
def test_process_spider_input(self):
|
||||
self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
|
||||
assert self.mw.process_spider_input(self.res200, self.spider) is None
|
||||
with pytest.raises(HttpError):
|
||||
self.mw.process_spider_input(self.res404, self.spider)
|
||||
|
||||
def test_process_spider_exception(self):
|
||||
self.assertEqual(
|
||||
[],
|
||||
assert (
|
||||
self.mw.process_spider_exception(
|
||||
self.res404, HttpError(self.res404), self.spider
|
||||
),
|
||||
)
|
||||
== []
|
||||
)
|
||||
self.assertIsNone(
|
||||
assert (
|
||||
self.mw.process_spider_exception(self.res404, Exception(), self.spider)
|
||||
is None
|
||||
)
|
||||
|
||||
def test_handle_httpstatus_list(self):
|
||||
|
|
@ -88,26 +88,26 @@ class TestHttpErrorMiddleware(TestCase):
|
|||
res.request = Request(
|
||||
"http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
|
||||
)
|
||||
self.assertIsNone(self.mw.process_spider_input(res, self.spider))
|
||||
assert self.mw.process_spider_input(res, self.spider) is None
|
||||
|
||||
self.spider.handle_httpstatus_list = [404]
|
||||
self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
|
||||
assert self.mw.process_spider_input(self.res404, self.spider) is None
|
||||
|
||||
|
||||
class TestHttpErrorMiddlewareSettings(TestCase):
|
||||
class TestHttpErrorMiddlewareSettings:
|
||||
"""Similar test, but with settings"""
|
||||
|
||||
def setUp(self):
|
||||
def setup_method(self):
|
||||
self.spider = Spider("foo")
|
||||
self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)}))
|
||||
self.req = Request("http://scrapytest.org")
|
||||
self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
|
||||
|
||||
def test_process_spider_input(self):
|
||||
self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
|
||||
assert self.mw.process_spider_input(self.res200, self.spider) is None
|
||||
with pytest.raises(HttpError):
|
||||
self.mw.process_spider_input(self.res404, self.spider)
|
||||
self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider))
|
||||
assert self.mw.process_spider_input(self.res402, self.spider) is None
|
||||
|
||||
def test_meta_overrides_settings(self):
|
||||
request = Request(
|
||||
|
|
@ -118,27 +118,27 @@ class TestHttpErrorMiddlewareSettings(TestCase):
|
|||
res402 = self.res402.copy()
|
||||
res402.request = request
|
||||
|
||||
self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
|
||||
assert self.mw.process_spider_input(res404, self.spider) is None
|
||||
with pytest.raises(HttpError):
|
||||
self.mw.process_spider_input(res402, self.spider)
|
||||
|
||||
def test_spider_override_settings(self):
|
||||
self.spider.handle_httpstatus_list = [404]
|
||||
self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
|
||||
assert self.mw.process_spider_input(self.res404, self.spider) is None
|
||||
with pytest.raises(HttpError):
|
||||
self.mw.process_spider_input(self.res402, self.spider)
|
||||
|
||||
|
||||
class TestHttpErrorMiddlewareHandleAll(TestCase):
|
||||
def setUp(self):
|
||||
class TestHttpErrorMiddlewareHandleAll:
|
||||
def setup_method(self):
|
||||
self.spider = Spider("foo")
|
||||
self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True}))
|
||||
self.req = Request("http://scrapytest.org")
|
||||
self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
|
||||
|
||||
def test_process_spider_input(self):
|
||||
self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
|
||||
self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
|
||||
assert self.mw.process_spider_input(self.res200, self.spider) is None
|
||||
assert self.mw.process_spider_input(self.res404, self.spider) is None
|
||||
|
||||
def test_meta_overrides_settings(self):
|
||||
request = Request(
|
||||
|
|
@ -149,7 +149,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
|
|||
res402 = self.res402.copy()
|
||||
res402.request = request
|
||||
|
||||
self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
|
||||
assert self.mw.process_spider_input(res404, self.spider) is None
|
||||
with pytest.raises(HttpError):
|
||||
self.mw.process_spider_input(res402, self.spider)
|
||||
|
||||
|
|
@ -169,10 +169,10 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
|
|||
|
||||
with pytest.raises(HttpError):
|
||||
mw.process_spider_input(res404, self.spider)
|
||||
self.assertIsNone(mw.process_spider_input(res402, self.spider))
|
||||
assert mw.process_spider_input(res402, self.spider) is None
|
||||
|
||||
|
||||
class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
|
||||
class TestHttpErrorMiddlewareIntegrational(TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.mockserver = MockServer()
|
||||
|
|
@ -187,28 +187,28 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
|
|||
crawler = get_crawler(_HttpErrorSpider)
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
assert not crawler.spider.skipped, crawler.spider.skipped
|
||||
self.assertEqual(crawler.spider.parsed, {"200"})
|
||||
self.assertEqual(crawler.spider.failed, {"404", "402", "500"})
|
||||
assert crawler.spider.parsed == {"200"}
|
||||
assert crawler.spider.failed == {"404", "402", "500"}
|
||||
|
||||
get_value = crawler.stats.get_value
|
||||
self.assertEqual(get_value("httperror/response_ignored_count"), 3)
|
||||
self.assertEqual(get_value("httperror/response_ignored_status_count/404"), 1)
|
||||
self.assertEqual(get_value("httperror/response_ignored_status_count/402"), 1)
|
||||
self.assertEqual(get_value("httperror/response_ignored_status_count/500"), 1)
|
||||
assert get_value("httperror/response_ignored_count") == 3
|
||||
assert get_value("httperror/response_ignored_status_count/404") == 1
|
||||
assert get_value("httperror/response_ignored_status_count/402") == 1
|
||||
assert get_value("httperror/response_ignored_status_count/500") == 1
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_logging(self):
|
||||
crawler = get_crawler(_HttpErrorSpider)
|
||||
with LogCapture() as log:
|
||||
yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402})
|
||||
self.assertEqual(crawler.spider.parsed, {"200", "402"})
|
||||
self.assertEqual(crawler.spider.skipped, {"402"})
|
||||
self.assertEqual(crawler.spider.failed, {"404", "500"})
|
||||
assert crawler.spider.parsed == {"200", "402"}
|
||||
assert crawler.spider.skipped == {"402"}
|
||||
assert crawler.spider.failed == {"404", "500"}
|
||||
|
||||
self.assertIn("Ignoring response <404", str(log))
|
||||
self.assertIn("Ignoring response <500", str(log))
|
||||
self.assertNotIn("Ignoring response <200", str(log))
|
||||
self.assertNotIn("Ignoring response <402", str(log))
|
||||
assert "Ignoring response <404" in str(log)
|
||||
assert "Ignoring response <500" in str(log)
|
||||
assert "Ignoring response <200" not in str(log)
|
||||
assert "Ignoring response <402" not in str(log)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_logging_level(self):
|
||||
|
|
@ -216,22 +216,22 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
|
|||
crawler = get_crawler(_HttpErrorSpider)
|
||||
with LogCapture(level=logging.INFO) as log:
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
self.assertEqual(crawler.spider.parsed, {"200"})
|
||||
self.assertEqual(crawler.spider.failed, {"404", "402", "500"})
|
||||
assert crawler.spider.parsed == {"200"}
|
||||
assert crawler.spider.failed == {"404", "402", "500"}
|
||||
|
||||
self.assertIn("Ignoring response <402", str(log))
|
||||
self.assertIn("Ignoring response <404", str(log))
|
||||
self.assertIn("Ignoring response <500", str(log))
|
||||
self.assertNotIn("Ignoring response <200", str(log))
|
||||
assert "Ignoring response <402" in str(log)
|
||||
assert "Ignoring response <404" in str(log)
|
||||
assert "Ignoring response <500" in str(log)
|
||||
assert "Ignoring response <200" not in str(log)
|
||||
|
||||
# with level WARNING, we shouldn't capture anything from HttpError
|
||||
crawler = get_crawler(_HttpErrorSpider)
|
||||
with LogCapture(level=logging.WARNING) as log:
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
self.assertEqual(crawler.spider.parsed, {"200"})
|
||||
self.assertEqual(crawler.spider.failed, {"404", "402", "500"})
|
||||
assert crawler.spider.parsed == {"200"}
|
||||
assert crawler.spider.failed == {"404", "402", "500"}
|
||||
|
||||
self.assertNotIn("Ignoring response <402", str(log))
|
||||
self.assertNotIn("Ignoring response <404", str(log))
|
||||
self.assertNotIn("Ignoring response <500", str(log))
|
||||
self.assertNotIn("Ignoring response <200", str(log))
|
||||
assert "Ignoring response <402" not in str(log)
|
||||
assert "Ignoring response <404" not in str(log)
|
||||
assert "Ignoring response <500" not in str(log)
|
||||
assert "Ignoring response <200" not in str(log)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import warnings
|
||||
from unittest import TestCase
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from scrapy.http import Request, Response
|
||||
|
|
@ -8,8 +7,8 @@ from scrapy.spiders import Spider
|
|||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
class TestOffsiteMiddleware(TestCase):
|
||||
def setUp(self):
|
||||
class TestOffsiteMiddleware:
|
||||
def setup_method(self):
|
||||
crawler = get_crawler(Spider)
|
||||
self.spider = crawler._create_spider(**self._get_spiderargs())
|
||||
self.mw = OffsiteMiddleware.from_crawler(crawler)
|
||||
|
|
@ -46,7 +45,7 @@ class TestOffsiteMiddleware(TestCase):
|
|||
reqs = onsite_reqs + offsite_reqs
|
||||
|
||||
out = list(self.mw.process_spider_output(res, reqs, self.spider))
|
||||
self.assertEqual(out, onsite_reqs)
|
||||
assert out == onsite_reqs
|
||||
|
||||
|
||||
class TestOffsiteMiddleware2(TestOffsiteMiddleware):
|
||||
|
|
@ -57,7 +56,7 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware):
|
|||
res = Response("http://scrapytest.org")
|
||||
reqs = [Request("http://a.com/b.html"), Request("http://b.com/1")]
|
||||
out = list(self.mw.process_spider_output(res, reqs, self.spider))
|
||||
self.assertEqual(out, reqs)
|
||||
assert out == reqs
|
||||
|
||||
|
||||
class TestOffsiteMiddleware3(TestOffsiteMiddleware2):
|
||||
|
|
@ -77,7 +76,7 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3):
|
|||
res = Response("http://scrapytest.org")
|
||||
reqs = [Request("http://scrapytest.org/1")]
|
||||
out = list(self.mw.process_spider_output(res, reqs, self.spider))
|
||||
self.assertEqual(out, reqs)
|
||||
assert out == reqs
|
||||
|
||||
|
||||
class TestOffsiteMiddleware5(TestOffsiteMiddleware4):
|
||||
|
|
|
|||
|
|
@ -324,9 +324,9 @@ class TestSpiderMiddleware(TestCase):
|
|||
was enqueued from the recovery middleware)
|
||||
"""
|
||||
log = yield self.crawl_log(RecoverySpider)
|
||||
self.assertIn("Middleware: TabError exception caught", str(log))
|
||||
self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1)
|
||||
self.assertIn("'item_scraped_count': 3", str(log))
|
||||
assert "Middleware: TabError exception caught" in str(log)
|
||||
assert str(log).count("Middleware: TabError exception caught") == 1
|
||||
assert "'item_scraped_count': 3" in str(log)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_recovery_asyncgen(self):
|
||||
|
|
@ -334,9 +334,9 @@ class TestSpiderMiddleware(TestCase):
|
|||
Same as test_recovery but with an async callback.
|
||||
"""
|
||||
log = yield self.crawl_log(RecoveryAsyncGenSpider)
|
||||
self.assertIn("Middleware: TabError exception caught", str(log))
|
||||
self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1)
|
||||
self.assertIn("'item_scraped_count': 3", str(log))
|
||||
assert "Middleware: TabError exception caught" in str(log)
|
||||
assert str(log).count("Middleware: TabError exception caught") == 1
|
||||
assert "'item_scraped_count': 3" in str(log)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_process_spider_input_without_errback(self):
|
||||
|
|
@ -345,8 +345,8 @@ class TestSpiderMiddleware(TestCase):
|
|||
process_spider_exception chain from the start if the Request has no errback
|
||||
"""
|
||||
log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithoutErrback)
|
||||
self.assertIn("Middleware: will raise IndexError", str(log1))
|
||||
self.assertIn("Middleware: IndexError exception caught", str(log1))
|
||||
assert "Middleware: will raise IndexError" in str(log1)
|
||||
assert "Middleware: IndexError exception caught" in str(log1)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_process_spider_input_with_errback(self):
|
||||
|
|
@ -355,12 +355,12 @@ class TestSpiderMiddleware(TestCase):
|
|||
process_spider_exception chain if the Request has an errback
|
||||
"""
|
||||
log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithErrback)
|
||||
self.assertNotIn("Middleware: IndexError exception caught", str(log1))
|
||||
self.assertIn("Middleware: will raise IndexError", str(log1))
|
||||
self.assertIn("Got a Failure on the Request errback", str(log1))
|
||||
self.assertIn("{'from': 'errback'}", str(log1))
|
||||
self.assertNotIn("{'from': 'callback'}", str(log1))
|
||||
self.assertIn("'item_scraped_count': 1", str(log1))
|
||||
assert "Middleware: IndexError exception caught" not in str(log1)
|
||||
assert "Middleware: will raise IndexError" in str(log1)
|
||||
assert "Got a Failure on the Request errback" in str(log1)
|
||||
assert "{'from': 'errback'}" in str(log1)
|
||||
assert "{'from': 'callback'}" not in str(log1)
|
||||
assert "'item_scraped_count': 1" in str(log1)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_generator_callback(self):
|
||||
|
|
@ -370,8 +370,8 @@ class TestSpiderMiddleware(TestCase):
|
|||
exception is raised should be processed normally.
|
||||
"""
|
||||
log2 = yield self.crawl_log(GeneratorCallbackSpider)
|
||||
self.assertIn("Middleware: ImportError exception caught", str(log2))
|
||||
self.assertIn("'item_scraped_count': 2", str(log2))
|
||||
assert "Middleware: ImportError exception caught" in str(log2)
|
||||
assert "'item_scraped_count': 2" in str(log2)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_async_generator_callback(self):
|
||||
|
|
@ -379,8 +379,8 @@ class TestSpiderMiddleware(TestCase):
|
|||
Same as test_generator_callback but with an async callback.
|
||||
"""
|
||||
log2 = yield self.crawl_log(AsyncGeneratorCallbackSpider)
|
||||
self.assertIn("Middleware: ImportError exception caught", str(log2))
|
||||
self.assertIn("'item_scraped_count': 2", str(log2))
|
||||
assert "Middleware: ImportError exception caught" in str(log2)
|
||||
assert "'item_scraped_count': 2" in str(log2)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_generator_callback_right_after_callback(self):
|
||||
|
|
@ -389,8 +389,8 @@ class TestSpiderMiddleware(TestCase):
|
|||
even if the middleware is placed right after the spider
|
||||
"""
|
||||
log21 = yield self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider)
|
||||
self.assertIn("Middleware: ImportError exception caught", str(log21))
|
||||
self.assertIn("'item_scraped_count': 2", str(log21))
|
||||
assert "Middleware: ImportError exception caught" in str(log21)
|
||||
assert "'item_scraped_count': 2" in str(log21)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_not_a_generator_callback(self):
|
||||
|
|
@ -399,8 +399,8 @@ class TestSpiderMiddleware(TestCase):
|
|||
be caught by the process_spider_exception chain. No items should be processed.
|
||||
"""
|
||||
log3 = yield self.crawl_log(NotGeneratorCallbackSpider)
|
||||
self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3))
|
||||
self.assertNotIn("item_scraped_count", str(log3))
|
||||
assert "Middleware: ZeroDivisionError exception caught" in str(log3)
|
||||
assert "item_scraped_count" not in str(log3)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_not_a_generator_callback_right_after_callback(self):
|
||||
|
|
@ -411,8 +411,8 @@ class TestSpiderMiddleware(TestCase):
|
|||
log31 = yield self.crawl_log(
|
||||
NotGeneratorCallbackSpiderMiddlewareRightAfterSpider
|
||||
)
|
||||
self.assertIn("Middleware: ZeroDivisionError exception caught", str(log31))
|
||||
self.assertNotIn("item_scraped_count", str(log31))
|
||||
assert "Middleware: ZeroDivisionError exception caught" in str(log31)
|
||||
assert "item_scraped_count" not in str(log31)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_generator_output_chain(self):
|
||||
|
|
@ -425,22 +425,22 @@ class TestSpiderMiddleware(TestCase):
|
|||
process_spider_exception chain)
|
||||
"""
|
||||
log4 = yield self.crawl_log(GeneratorOutputChainSpider)
|
||||
self.assertIn("'item_scraped_count': 2", str(log4))
|
||||
self.assertIn(
|
||||
"GeneratorRecoverMiddleware.process_spider_exception: LookupError caught",
|
||||
str(log4),
|
||||
assert "'item_scraped_count': 2" in str(log4)
|
||||
assert (
|
||||
"GeneratorRecoverMiddleware.process_spider_exception: LookupError caught"
|
||||
in str(log4)
|
||||
)
|
||||
self.assertIn(
|
||||
"GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught",
|
||||
str(log4),
|
||||
assert (
|
||||
"GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught"
|
||||
in str(log4)
|
||||
)
|
||||
self.assertNotIn(
|
||||
"GeneratorFailMiddleware.process_spider_exception: LookupError caught",
|
||||
str(log4),
|
||||
assert (
|
||||
"GeneratorFailMiddleware.process_spider_exception: LookupError caught"
|
||||
not in str(log4)
|
||||
)
|
||||
self.assertNotIn(
|
||||
"GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught",
|
||||
str(log4),
|
||||
assert (
|
||||
"GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught"
|
||||
not in str(log4)
|
||||
)
|
||||
item_from_callback = {
|
||||
"processed": [
|
||||
|
|
@ -457,9 +457,9 @@ class TestSpiderMiddleware(TestCase):
|
|||
"GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output",
|
||||
]
|
||||
}
|
||||
self.assertIn(str(item_from_callback), str(log4))
|
||||
self.assertIn(str(item_recovered), str(log4))
|
||||
self.assertNotIn("parse-second-item", str(log4))
|
||||
assert str(item_from_callback) in str(log4)
|
||||
assert str(item_recovered) in str(log4)
|
||||
assert "parse-second-item" not in str(log4)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_not_a_generator_output_chain(self):
|
||||
|
|
@ -472,22 +472,22 @@ class TestSpiderMiddleware(TestCase):
|
|||
from the spider callback are lost)
|
||||
"""
|
||||
log5 = yield self.crawl_log(NotGeneratorOutputChainSpider)
|
||||
self.assertIn("'item_scraped_count': 1", str(log5))
|
||||
self.assertIn(
|
||||
"GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught",
|
||||
str(log5),
|
||||
assert "'item_scraped_count': 1" in str(log5)
|
||||
assert (
|
||||
"GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught"
|
||||
in str(log5)
|
||||
)
|
||||
self.assertIn(
|
||||
"GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught",
|
||||
str(log5),
|
||||
assert (
|
||||
"GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught"
|
||||
in str(log5)
|
||||
)
|
||||
self.assertNotIn(
|
||||
"GeneratorFailMiddleware.process_spider_exception: ReferenceError caught",
|
||||
str(log5),
|
||||
assert (
|
||||
"GeneratorFailMiddleware.process_spider_exception: ReferenceError caught"
|
||||
not in str(log5)
|
||||
)
|
||||
self.assertNotIn(
|
||||
"GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught",
|
||||
str(log5),
|
||||
assert (
|
||||
"GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught"
|
||||
not in str(log5)
|
||||
)
|
||||
item_recovered = {
|
||||
"processed": [
|
||||
|
|
@ -495,6 +495,6 @@ class TestSpiderMiddleware(TestCase):
|
|||
"NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output",
|
||||
]
|
||||
}
|
||||
self.assertIn(str(item_recovered), str(log5))
|
||||
self.assertNotIn("parse-first-item", str(log5))
|
||||
self.assertNotIn("parse-second-item", str(log5))
|
||||
assert str(item_recovered) in str(log5)
|
||||
assert "parse-first-item" not in str(log5)
|
||||
assert "parse-second-item" not in str(log5)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ from __future__ import annotations
|
|||
|
||||
import warnings
|
||||
from typing import Any
|
||||
from unittest import TestCase
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import pytest
|
||||
|
|
@ -35,7 +34,7 @@ from scrapy.spidermiddlewares.referer import (
|
|||
from scrapy.spiders import Spider
|
||||
|
||||
|
||||
class TestRefererMiddleware(TestCase):
|
||||
class TestRefererMiddleware:
|
||||
req_meta: dict[str, Any] = {}
|
||||
resp_headers: dict[str, str] = {}
|
||||
settings: dict[str, Any] = {}
|
||||
|
|
@ -43,7 +42,7 @@ class TestRefererMiddleware(TestCase):
|
|||
("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"),
|
||||
]
|
||||
|
||||
def setUp(self):
|
||||
def setup_method(self):
|
||||
self.spider = Spider("foo")
|
||||
settings = Settings(self.settings)
|
||||
self.mw = RefererMiddleware(settings)
|
||||
|
|
@ -59,7 +58,7 @@ class TestRefererMiddleware(TestCase):
|
|||
response = self.get_response(origin)
|
||||
request = self.get_request(target)
|
||||
out = list(self.mw.process_spider_output(response, [request], self.spider))
|
||||
self.assertEqual(out[0].headers.get("Referer"), referrer)
|
||||
assert out[0].headers.get("Referer") == referrer
|
||||
|
||||
|
||||
class MixinDefault:
|
||||
|
|
@ -773,7 +772,7 @@ class TestRequestMetaPrecedence003(MixinUnsafeUrl, TestRefererMiddleware):
|
|||
req_meta = {"referrer_policy": POLICY_UNSAFE_URL}
|
||||
|
||||
|
||||
class TestRequestMetaSettingFallback(TestCase):
|
||||
class TestRequestMetaSettingFallback:
|
||||
params = [
|
||||
(
|
||||
# When an unknown policy is referenced in Request.meta
|
||||
|
|
@ -844,14 +843,14 @@ class TestRequestMetaSettingFallback(TestCase):
|
|||
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
policy = mw.policy(response, request)
|
||||
self.assertIsInstance(policy, policy_class)
|
||||
assert isinstance(policy, policy_class)
|
||||
|
||||
if check_warning:
|
||||
self.assertEqual(len(w), 1)
|
||||
self.assertEqual(w[0].category, RuntimeWarning, w[0].message)
|
||||
assert len(w) == 1
|
||||
assert w[0].category is RuntimeWarning, w[0].message
|
||||
|
||||
|
||||
class TestSettingsPolicyByName(TestCase):
|
||||
class TestSettingsPolicyByName:
|
||||
def test_valid_name(self):
|
||||
for s, p in [
|
||||
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
||||
|
|
@ -866,7 +865,7 @@ class TestSettingsPolicyByName(TestCase):
|
|||
]:
|
||||
settings = Settings({"REFERRER_POLICY": s})
|
||||
mw = RefererMiddleware(settings)
|
||||
self.assertEqual(mw.default_policy, p)
|
||||
assert mw.default_policy == p
|
||||
|
||||
def test_valid_name_casevariants(self):
|
||||
for s, p in [
|
||||
|
|
@ -882,7 +881,7 @@ class TestSettingsPolicyByName(TestCase):
|
|||
]:
|
||||
settings = Settings({"REFERRER_POLICY": s.upper()})
|
||||
mw = RefererMiddleware(settings)
|
||||
self.assertEqual(mw.default_policy, p)
|
||||
assert mw.default_policy == p
|
||||
|
||||
def test_invalid_name(self):
|
||||
settings = Settings({"REFERRER_POLICY": "some-custom-unknown-policy"})
|
||||
|
|
@ -902,7 +901,7 @@ class TestSettingsPolicyByName(TestCase):
|
|||
}
|
||||
)
|
||||
mw1 = RefererMiddleware(settings1)
|
||||
self.assertEqual(mw1.default_policy, StrictOriginWhenCrossOriginPolicy)
|
||||
assert mw1.default_policy == StrictOriginWhenCrossOriginPolicy
|
||||
|
||||
# test parsing with space(s) after the comma
|
||||
settings2 = Settings(
|
||||
|
|
@ -915,7 +914,7 @@ class TestSettingsPolicyByName(TestCase):
|
|||
}
|
||||
)
|
||||
mw2 = RefererMiddleware(settings2)
|
||||
self.assertEqual(mw2.default_policy, UnsafeUrlPolicy)
|
||||
assert mw2.default_policy == UnsafeUrlPolicy
|
||||
|
||||
def test_multiple_policy_tokens_all_invalid(self):
|
||||
settings = Settings(
|
||||
|
|
@ -1003,7 +1002,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
|
|||
),
|
||||
]
|
||||
|
||||
def setUp(self):
|
||||
def setup_method(self):
|
||||
self.spider = Spider("foo")
|
||||
settings = Settings(self.settings)
|
||||
self.referrermw = RefererMiddleware(settings)
|
||||
|
|
@ -1023,7 +1022,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
|
|||
out = list(
|
||||
self.referrermw.process_spider_output(response, [request], self.spider)
|
||||
)
|
||||
self.assertEqual(out[0].headers.get("Referer"), init_referrer)
|
||||
assert out[0].headers.get("Referer") == init_referrer
|
||||
|
||||
for status, url in redirections:
|
||||
response = Response(
|
||||
|
|
@ -1035,7 +1034,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
|
|||
self.referrermw.request_scheduled(request, self.spider)
|
||||
|
||||
assert isinstance(request, Request)
|
||||
self.assertEqual(request.headers.get("Referer"), final_referrer)
|
||||
assert request.headers.get("Referer") == final_referrer
|
||||
|
||||
|
||||
class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
from unittest import TestCase
|
||||
|
||||
from testfixtures import LogCapture
|
||||
|
||||
from scrapy.http import Request, Response
|
||||
|
|
@ -8,8 +6,8 @@ from scrapy.spiders import Spider
|
|||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
class TestUrlLengthMiddleware(TestCase):
|
||||
def setUp(self):
|
||||
class TestUrlLengthMiddleware:
|
||||
def setup_method(self):
|
||||
self.maxlength = 25
|
||||
crawler = get_crawler(Spider, {"URLLENGTH_LIMIT": self.maxlength})
|
||||
self.spider = crawler._create_spider("foo")
|
||||
|
|
@ -27,7 +25,7 @@ class TestUrlLengthMiddleware(TestCase):
|
|||
)
|
||||
|
||||
def test_middleware_works(self):
|
||||
self.assertEqual(self.process_spider_output(), [self.short_url_req])
|
||||
assert self.process_spider_output() == [self.short_url_req]
|
||||
|
||||
def test_logging(self):
|
||||
with LogCapture() as log:
|
||||
|
|
@ -36,6 +34,6 @@ class TestUrlLengthMiddleware(TestCase):
|
|||
ric = self.stats.get_value(
|
||||
"urllength/request_ignored_count", spider=self.spider
|
||||
)
|
||||
self.assertEqual(ric, 1)
|
||||
assert ric == 1
|
||||
|
||||
self.assertIn(f"Ignoring link (url length > {self.maxlength})", str(log))
|
||||
assert f"Ignoring link (url length > {self.maxlength})" in str(log)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ from datetime import datetime, timezone
|
|||
from tempfile import mkdtemp
|
||||
|
||||
import pytest
|
||||
from twisted.trial import unittest
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.extensions.spiderstate import SpiderState
|
||||
|
|
@ -11,7 +10,7 @@ from scrapy.spiders import Spider
|
|||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
class SpiderStateTest(unittest.TestCase):
|
||||
class TestSpiderState:
|
||||
def test_store_load(self):
|
||||
jobdir = mkdtemp()
|
||||
try:
|
||||
|
|
@ -27,7 +26,7 @@ class SpiderStateTest(unittest.TestCase):
|
|||
spider2 = Spider(name="default")
|
||||
ss2 = SpiderState(jobdir)
|
||||
ss2.spider_opened(spider2)
|
||||
self.assertEqual(spider.state, {"one": 1, "dt": dt})
|
||||
assert spider.state == {"one": 1, "dt": dt}
|
||||
ss2.spider_closed(spider2)
|
||||
finally:
|
||||
shutil.rmtree(jobdir)
|
||||
|
|
@ -38,7 +37,7 @@ class SpiderStateTest(unittest.TestCase):
|
|||
spider = Spider(name="default")
|
||||
ss = SpiderState()
|
||||
ss.spider_opened(spider)
|
||||
self.assertEqual(spider.state, {})
|
||||
assert spider.state == {}
|
||||
ss.spider_closed(spider)
|
||||
|
||||
def test_not_configured(self):
|
||||
|
|
|
|||
|
|
@ -50,9 +50,9 @@ class FifoDiskQueueTestMixin:
|
|||
q.push("a")
|
||||
q.push(123)
|
||||
q.push({"a": "dict"})
|
||||
self.assertEqual(q.pop(), "a")
|
||||
self.assertEqual(q.pop(), 123)
|
||||
self.assertEqual(q.pop(), {"a": "dict"})
|
||||
assert q.pop() == "a"
|
||||
assert q.pop() == 123
|
||||
assert q.pop() == {"a": "dict"}
|
||||
|
||||
test_nonserializable_object = nonserializable_object_test
|
||||
|
||||
|
|
@ -92,7 +92,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
|
|||
q.push(i)
|
||||
i2 = q.pop()
|
||||
assert isinstance(i2, MyItem)
|
||||
self.assertEqual(i, i2)
|
||||
assert i == i2
|
||||
|
||||
def test_serialize_loader(self):
|
||||
q = self.queue()
|
||||
|
|
@ -101,7 +101,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
|
|||
loader2 = q.pop()
|
||||
assert isinstance(loader2, MyLoader)
|
||||
assert loader2.default_item_class is MyItem
|
||||
self.assertEqual(loader2.name_out("x"), "xx")
|
||||
assert loader2.name_out("x") == "xx"
|
||||
|
||||
def test_serialize_request_recursive(self):
|
||||
q = self.queue()
|
||||
|
|
@ -110,23 +110,26 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
|
|||
q.push(r)
|
||||
r2 = q.pop()
|
||||
assert isinstance(r2, Request)
|
||||
self.assertEqual(r.url, r2.url)
|
||||
assert r.url == r2.url
|
||||
assert r2.meta["request"] is r2
|
||||
|
||||
def test_non_pickable_object(self):
|
||||
q = self.queue()
|
||||
try:
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Can't (get|pickle) local object|Can't pickle .*: it's not found as",
|
||||
) as exc_info:
|
||||
q.push(lambda x: x)
|
||||
except ValueError as exc:
|
||||
if hasattr(sys, "pypy_version_info"):
|
||||
self.assertIsInstance(exc.__context__, pickle.PicklingError)
|
||||
else:
|
||||
self.assertIsInstance(exc.__context__, AttributeError)
|
||||
if hasattr(sys, "pypy_version_info"):
|
||||
assert isinstance(exc_info.value.__context__, pickle.PicklingError)
|
||||
else:
|
||||
assert isinstance(exc_info.value.__context__, AttributeError)
|
||||
sel = Selector(text="<html><body><p>some text</p></body></html>")
|
||||
try:
|
||||
with pytest.raises(
|
||||
ValueError, match="can't pickle Selector objects"
|
||||
) as exc_info:
|
||||
q.push(sel)
|
||||
except ValueError as exc:
|
||||
self.assertIsInstance(exc.__context__, TypeError)
|
||||
assert isinstance(exc_info.value.__context__, TypeError)
|
||||
|
||||
|
||||
class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
|
||||
|
|
@ -151,9 +154,9 @@ class LifoDiskQueueTestMixin:
|
|||
q.push("a")
|
||||
q.push(123)
|
||||
q.push({"a": "dict"})
|
||||
self.assertEqual(q.pop(), {"a": "dict"})
|
||||
self.assertEqual(q.pop(), 123)
|
||||
self.assertEqual(q.pop(), "a")
|
||||
assert q.pop() == {"a": "dict"}
|
||||
assert q.pop() == 123
|
||||
assert q.pop() == "a"
|
||||
|
||||
test_nonserializable_object = nonserializable_object_test
|
||||
|
||||
|
|
@ -173,7 +176,7 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
|
|||
q.push(i)
|
||||
i2 = q.pop()
|
||||
assert isinstance(i2, MyItem)
|
||||
self.assertEqual(i, i2)
|
||||
assert i == i2
|
||||
|
||||
def test_serialize_loader(self):
|
||||
q = self.queue()
|
||||
|
|
@ -182,7 +185,7 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
|
|||
loader2 = q.pop()
|
||||
assert isinstance(loader2, MyLoader)
|
||||
assert loader2.default_item_class is MyItem
|
||||
self.assertEqual(loader2.name_out("x"), "xx")
|
||||
assert loader2.name_out("x") == "xx"
|
||||
|
||||
def test_serialize_request_recursive(self):
|
||||
q = self.queue()
|
||||
|
|
@ -191,5 +194,5 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
|
|||
q.push(r)
|
||||
r2 = q.pop()
|
||||
assert isinstance(r2, Request)
|
||||
self.assertEqual(r.url, r2.url)
|
||||
assert r.url == r2.url
|
||||
assert r2.meta["request"] is r2
|
||||
|
|
|
|||
|
|
@ -22,14 +22,14 @@ from scrapy.squeues import (
|
|||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
class BaseQueueTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
class TestBaseQueue:
|
||||
def setup_method(self):
|
||||
self.tmpdir = tempfile.mkdtemp(prefix="scrapy-queue-tests-")
|
||||
self.qpath = self.tempfilename()
|
||||
self.qdir = tempfile.mkdtemp()
|
||||
self.crawler = get_crawler(Spider)
|
||||
|
||||
def tearDown(self):
|
||||
def teardown_method(self):
|
||||
shutil.rmtree(self.tmpdir)
|
||||
|
||||
def tempfilename(self):
|
||||
|
|
@ -48,36 +48,36 @@ class RequestQueueTestMixin:
|
|||
if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
|
||||
raise unittest.SkipTest("The queuelib queues do not define peek")
|
||||
q = self.queue()
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.peek())
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 0
|
||||
assert q.peek() is None
|
||||
assert q.pop() is None
|
||||
req = Request("http://www.example.com")
|
||||
q.push(req)
|
||||
self.assertEqual(len(q), 1)
|
||||
self.assertEqual(q.peek().url, req.url)
|
||||
self.assertEqual(q.pop().url, req.url)
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.peek())
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 1
|
||||
assert q.peek().url == req.url
|
||||
assert q.pop().url == req.url
|
||||
assert len(q) == 0
|
||||
assert q.peek() is None
|
||||
assert q.pop() is None
|
||||
q.close()
|
||||
|
||||
def test_one_element_without_peek(self):
|
||||
if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
|
||||
raise unittest.SkipTest("The queuelib queues define peek")
|
||||
q = self.queue()
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 0
|
||||
assert q.pop() is None
|
||||
req = Request("http://www.example.com")
|
||||
q.push(req)
|
||||
self.assertEqual(len(q), 1)
|
||||
assert len(q) == 1
|
||||
with pytest.raises(
|
||||
NotImplementedError,
|
||||
match="The underlying queue class does not implement 'peek'",
|
||||
):
|
||||
q.peek()
|
||||
self.assertEqual(q.pop().url, req.url)
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.pop())
|
||||
assert q.pop().url == req.url
|
||||
assert len(q) == 0
|
||||
assert q.pop() is None
|
||||
q.close()
|
||||
|
||||
|
||||
|
|
@ -86,35 +86,35 @@ class FifoQueueMixin(RequestQueueTestMixin):
|
|||
if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
|
||||
raise unittest.SkipTest("The queuelib queues do not define peek")
|
||||
q = self.queue()
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.peek())
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 0
|
||||
assert q.peek() is None
|
||||
assert q.pop() is None
|
||||
req1 = Request("http://www.example.com/1")
|
||||
req2 = Request("http://www.example.com/2")
|
||||
req3 = Request("http://www.example.com/3")
|
||||
q.push(req1)
|
||||
q.push(req2)
|
||||
q.push(req3)
|
||||
self.assertEqual(len(q), 3)
|
||||
self.assertEqual(q.peek().url, req1.url)
|
||||
self.assertEqual(q.pop().url, req1.url)
|
||||
self.assertEqual(len(q), 2)
|
||||
self.assertEqual(q.peek().url, req2.url)
|
||||
self.assertEqual(q.pop().url, req2.url)
|
||||
self.assertEqual(len(q), 1)
|
||||
self.assertEqual(q.peek().url, req3.url)
|
||||
self.assertEqual(q.pop().url, req3.url)
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.peek())
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 3
|
||||
assert q.peek().url == req1.url
|
||||
assert q.pop().url == req1.url
|
||||
assert len(q) == 2
|
||||
assert q.peek().url == req2.url
|
||||
assert q.pop().url == req2.url
|
||||
assert len(q) == 1
|
||||
assert q.peek().url == req3.url
|
||||
assert q.pop().url == req3.url
|
||||
assert len(q) == 0
|
||||
assert q.peek() is None
|
||||
assert q.pop() is None
|
||||
q.close()
|
||||
|
||||
def test_fifo_without_peek(self):
|
||||
if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
|
||||
raise unittest.SkipTest("The queuelib queues do not define peek")
|
||||
q = self.queue()
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 0
|
||||
assert q.pop() is None
|
||||
req1 = Request("http://www.example.com/1")
|
||||
req2 = Request("http://www.example.com/2")
|
||||
req3 = Request("http://www.example.com/3")
|
||||
|
|
@ -126,14 +126,14 @@ class FifoQueueMixin(RequestQueueTestMixin):
|
|||
match="The underlying queue class does not implement 'peek'",
|
||||
):
|
||||
q.peek()
|
||||
self.assertEqual(len(q), 3)
|
||||
self.assertEqual(q.pop().url, req1.url)
|
||||
self.assertEqual(len(q), 2)
|
||||
self.assertEqual(q.pop().url, req2.url)
|
||||
self.assertEqual(len(q), 1)
|
||||
self.assertEqual(q.pop().url, req3.url)
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 3
|
||||
assert q.pop().url == req1.url
|
||||
assert len(q) == 2
|
||||
assert q.pop().url == req2.url
|
||||
assert len(q) == 1
|
||||
assert q.pop().url == req3.url
|
||||
assert len(q) == 0
|
||||
assert q.pop() is None
|
||||
q.close()
|
||||
|
||||
|
||||
|
|
@ -142,35 +142,35 @@ class LifoQueueMixin(RequestQueueTestMixin):
|
|||
if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
|
||||
raise unittest.SkipTest("The queuelib queues do not define peek")
|
||||
q = self.queue()
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.peek())
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 0
|
||||
assert q.peek() is None
|
||||
assert q.pop() is None
|
||||
req1 = Request("http://www.example.com/1")
|
||||
req2 = Request("http://www.example.com/2")
|
||||
req3 = Request("http://www.example.com/3")
|
||||
q.push(req1)
|
||||
q.push(req2)
|
||||
q.push(req3)
|
||||
self.assertEqual(len(q), 3)
|
||||
self.assertEqual(q.peek().url, req3.url)
|
||||
self.assertEqual(q.pop().url, req3.url)
|
||||
self.assertEqual(len(q), 2)
|
||||
self.assertEqual(q.peek().url, req2.url)
|
||||
self.assertEqual(q.pop().url, req2.url)
|
||||
self.assertEqual(len(q), 1)
|
||||
self.assertEqual(q.peek().url, req1.url)
|
||||
self.assertEqual(q.pop().url, req1.url)
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.peek())
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 3
|
||||
assert q.peek().url == req3.url
|
||||
assert q.pop().url == req3.url
|
||||
assert len(q) == 2
|
||||
assert q.peek().url == req2.url
|
||||
assert q.pop().url == req2.url
|
||||
assert len(q) == 1
|
||||
assert q.peek().url == req1.url
|
||||
assert q.pop().url == req1.url
|
||||
assert len(q) == 0
|
||||
assert q.peek() is None
|
||||
assert q.pop() is None
|
||||
q.close()
|
||||
|
||||
def test_lifo_without_peek(self):
|
||||
if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
|
||||
raise unittest.SkipTest("The queuelib queues do not define peek")
|
||||
q = self.queue()
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 0
|
||||
assert q.pop() is None
|
||||
req1 = Request("http://www.example.com/1")
|
||||
req2 = Request("http://www.example.com/2")
|
||||
req3 = Request("http://www.example.com/3")
|
||||
|
|
@ -182,46 +182,46 @@ class LifoQueueMixin(RequestQueueTestMixin):
|
|||
match="The underlying queue class does not implement 'peek'",
|
||||
):
|
||||
q.peek()
|
||||
self.assertEqual(len(q), 3)
|
||||
self.assertEqual(q.pop().url, req3.url)
|
||||
self.assertEqual(len(q), 2)
|
||||
self.assertEqual(q.pop().url, req2.url)
|
||||
self.assertEqual(len(q), 1)
|
||||
self.assertEqual(q.pop().url, req1.url)
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 3
|
||||
assert q.pop().url == req3.url
|
||||
assert len(q) == 2
|
||||
assert q.pop().url == req2.url
|
||||
assert len(q) == 1
|
||||
assert q.pop().url == req1.url
|
||||
assert len(q) == 0
|
||||
assert q.pop() is None
|
||||
q.close()
|
||||
|
||||
|
||||
class PickleFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase):
|
||||
class TestPickleFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue):
|
||||
def queue(self):
|
||||
return PickleFifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/fifo")
|
||||
|
||||
|
||||
class PickleLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase):
|
||||
class TestPickleLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue):
|
||||
def queue(self):
|
||||
return PickleLifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/lifo")
|
||||
|
||||
|
||||
class MarshalFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase):
|
||||
class TestMarshalFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue):
|
||||
def queue(self):
|
||||
return MarshalFifoDiskQueue.from_crawler(
|
||||
crawler=self.crawler, key="marshal/fifo"
|
||||
)
|
||||
|
||||
|
||||
class MarshalLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase):
|
||||
class TestMarshalLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue):
|
||||
def queue(self):
|
||||
return MarshalLifoDiskQueue.from_crawler(
|
||||
crawler=self.crawler, key="marshal/lifo"
|
||||
)
|
||||
|
||||
|
||||
class FifoMemoryQueueRequestTest(FifoQueueMixin, BaseQueueTestCase):
|
||||
class TestFifoMemoryQueueRequest(FifoQueueMixin, TestBaseQueue):
|
||||
def queue(self):
|
||||
return FifoMemoryQueue.from_crawler(crawler=self.crawler)
|
||||
|
||||
|
||||
class LifoMemoryQueueRequestTest(LifoQueueMixin, BaseQueueTestCase):
|
||||
class TestLifoMemoryQueueRequest(LifoQueueMixin, TestBaseQueue):
|
||||
def queue(self):
|
||||
return LifoMemoryQueue.from_crawler(crawler=self.crawler)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import unittest
|
||||
from datetime import datetime
|
||||
from unittest import mock
|
||||
|
||||
|
|
@ -8,8 +7,8 @@ from scrapy.statscollectors import DummyStatsCollector, StatsCollector
|
|||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
class CoreStatsExtensionTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
class TestCoreStatsExtension:
|
||||
def setup_method(self):
|
||||
self.crawler = get_crawler(Spider)
|
||||
self.spider = self.crawler._create_spider("foo")
|
||||
|
||||
|
|
@ -24,19 +23,16 @@ class CoreStatsExtensionTest(unittest.TestCase):
|
|||
ext.response_received(self.spider)
|
||||
ext.item_dropped({}, self.spider, ZeroDivisionError())
|
||||
ext.spider_closed(self.spider, "finished")
|
||||
self.assertEqual(
|
||||
ext.stats._stats,
|
||||
{
|
||||
"start_time": fixed_datetime,
|
||||
"finish_time": fixed_datetime,
|
||||
"item_scraped_count": 1,
|
||||
"response_received_count": 1,
|
||||
"item_dropped_count": 1,
|
||||
"item_dropped_reasons_count/ZeroDivisionError": 1,
|
||||
"finish_reason": "finished",
|
||||
"elapsed_time_seconds": 0.0,
|
||||
},
|
||||
)
|
||||
assert ext.stats._stats == {
|
||||
"start_time": fixed_datetime,
|
||||
"finish_time": fixed_datetime,
|
||||
"item_scraped_count": 1,
|
||||
"response_received_count": 1,
|
||||
"item_dropped_count": 1,
|
||||
"item_dropped_reasons_count/ZeroDivisionError": 1,
|
||||
"finish_reason": "finished",
|
||||
"elapsed_time_seconds": 0.0,
|
||||
}
|
||||
|
||||
def test_core_stats_dummy_stats_collector(self):
|
||||
self.crawler.stats = DummyStatsCollector(self.crawler)
|
||||
|
|
@ -46,51 +42,51 @@ class CoreStatsExtensionTest(unittest.TestCase):
|
|||
ext.response_received(self.spider)
|
||||
ext.item_dropped({}, self.spider, ZeroDivisionError())
|
||||
ext.spider_closed(self.spider, "finished")
|
||||
self.assertEqual(ext.stats._stats, {})
|
||||
assert ext.stats._stats == {}
|
||||
|
||||
|
||||
class StatsCollectorTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
class TestStatsCollector:
|
||||
def setup_method(self):
|
||||
self.crawler = get_crawler(Spider)
|
||||
self.spider = self.crawler._create_spider("foo")
|
||||
|
||||
def test_collector(self):
|
||||
stats = StatsCollector(self.crawler)
|
||||
self.assertEqual(stats.get_stats(), {})
|
||||
self.assertEqual(stats.get_value("anything"), None)
|
||||
self.assertEqual(stats.get_value("anything", "default"), "default")
|
||||
assert stats.get_stats() == {}
|
||||
assert stats.get_value("anything") is None
|
||||
assert stats.get_value("anything", "default") == "default"
|
||||
stats.set_value("test", "value")
|
||||
self.assertEqual(stats.get_stats(), {"test": "value"})
|
||||
assert stats.get_stats() == {"test": "value"}
|
||||
stats.set_value("test2", 23)
|
||||
self.assertEqual(stats.get_stats(), {"test": "value", "test2": 23})
|
||||
self.assertEqual(stats.get_value("test2"), 23)
|
||||
assert stats.get_stats() == {"test": "value", "test2": 23}
|
||||
assert stats.get_value("test2") == 23
|
||||
stats.inc_value("test2")
|
||||
self.assertEqual(stats.get_value("test2"), 24)
|
||||
assert stats.get_value("test2") == 24
|
||||
stats.inc_value("test2", 6)
|
||||
self.assertEqual(stats.get_value("test2"), 30)
|
||||
assert stats.get_value("test2") == 30
|
||||
stats.max_value("test2", 6)
|
||||
self.assertEqual(stats.get_value("test2"), 30)
|
||||
assert stats.get_value("test2") == 30
|
||||
stats.max_value("test2", 40)
|
||||
self.assertEqual(stats.get_value("test2"), 40)
|
||||
assert stats.get_value("test2") == 40
|
||||
stats.max_value("test3", 1)
|
||||
self.assertEqual(stats.get_value("test3"), 1)
|
||||
assert stats.get_value("test3") == 1
|
||||
stats.min_value("test2", 60)
|
||||
self.assertEqual(stats.get_value("test2"), 40)
|
||||
assert stats.get_value("test2") == 40
|
||||
stats.min_value("test2", 35)
|
||||
self.assertEqual(stats.get_value("test2"), 35)
|
||||
assert stats.get_value("test2") == 35
|
||||
stats.min_value("test4", 7)
|
||||
self.assertEqual(stats.get_value("test4"), 7)
|
||||
assert stats.get_value("test4") == 7
|
||||
|
||||
def test_dummy_collector(self):
|
||||
stats = DummyStatsCollector(self.crawler)
|
||||
self.assertEqual(stats.get_stats(), {})
|
||||
self.assertEqual(stats.get_value("anything"), None)
|
||||
self.assertEqual(stats.get_value("anything", "default"), "default")
|
||||
assert stats.get_stats() == {}
|
||||
assert stats.get_value("anything") is None
|
||||
assert stats.get_value("anything", "default") == "default"
|
||||
stats.set_value("test", "value")
|
||||
stats.inc_value("v1")
|
||||
stats.max_value("v2", 100)
|
||||
stats.min_value("v3", 100)
|
||||
stats.open_spider("a")
|
||||
stats.set_value("test", "value", spider=self.spider)
|
||||
self.assertEqual(stats.get_stats(), {})
|
||||
self.assertEqual(stats.get_stats("a"), {})
|
||||
assert stats.get_stats() == {}
|
||||
assert stats.get_stats("a") == {}
|
||||
|
|
|
|||
Loading…
Reference in New Issue