RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

move eagerness testing to separate function

This commit is contained in:
Vostretsov Nikita 2020-06-30 06:57:37 +00:00
parent 04369d3061
commit 96143881e4
1 changed files with 20 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
tests/test_crawl.py
View File

@ -161,12 +161,24 @@ class CrawlTestCase(TestCase):
self.assertIsNotNone(record.exc_info)
self.assertIs(record.exc_info[0], ZeroDivisionError)
@defer.inlineCallbacks
def test_start_requests_eagerness(self):
def is_eager(_, requests_in_order):
"""
All start requests(depth=0) are scheduled before
any other requests(depth!=0)
"""
depths_in_order = [r.meta.get('depth', 0) for r in requests_in_order]
order_of_start_requests = [
o for o, d in enumerate(depths_in_order) if d == 0
]
order_of_other_requests = [
o for o, d in enumerate(depths_in_order) if d != 0
]
last_start_request = max(order_of_start_requests)
first_other_request = min(order_of_other_requests)
return last_start_request < first_other_request
@defer.inlineCallbacks
def test_start_requests_eagerness(self):
class EagerSpider(YeldingRequestsSpider):
def start_requests_with_control(self):
yield from self.start_requests()
@ -186,17 +198,9 @@ class CrawlTestCase(TestCase):
mockserver=self.mockserver,
number_of_start_requests=100,
)
requests_in_order = crawler.spider.requests_in_order_of_scheduling
depths_in_order = [r.meta.get('depth', 0) for r in requests_in_order]
order_of_start_requests = [
o for o, d in enumerate(depths_in_order) if d == 0
]
order_of_other_requests = [
o for o, d in enumerate(depths_in_order) if d != 0
]
last_start_request = max(order_of_start_requests)
first_other_request = min(order_of_other_requests)
assert last_start_request < first_other_request
self.assertTrue(
self.is_eager(crawler.spider.requests_in_order_of_scheduling)
)
@defer.inlineCallbacks
def test_start_requests_lazyness(self):
@ -218,17 +222,9 @@ class CrawlTestCase(TestCase):
mockserver=self.mockserver,
number_of_start_requests=100,
)
requests_in_order = crawler.spider.requests_in_order_of_scheduling
depths_in_order = [r.meta.get('depth', 0) for r in requests_in_order]
order_of_start_requests = [
o for o, d in enumerate(depths_in_order) if d == 0
]
order_of_other_requests = [
o for o, d in enumerate(depths_in_order) if d != 0
]
last_start_request = max(order_of_start_requests)
first_other_request = min(order_of_other_requests)
assert last_start_request > first_other_request
self.assertFalse(
self.is_eager(crawler.spider.requests_in_order_of_scheduling)
)
@defer.inlineCallbacks
def test_start_requests_dupes(self):