RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

move number of start request to separate function

This commit is contained in:
Vostretsov Nikita 2020-06-30 07:12:04 +00:00
parent 96143881e4
commit 34ecb94dfe
1 changed files with 9 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
tests/test_crawl.py
View File

@ -177,6 +177,13 @@ class CrawlTestCase(TestCase):
first_other_request = min(order_of_other_requests)
return last_start_request < first_other_request
def number_of_start_requests_fro_eagernees_testing(_):
"""
number_of_start_requests should be big enough, so scheduling such
amount of requests takes longer than crawling first of them
"""
return 100
@defer.inlineCallbacks
def test_start_requests_eagerness(self):
class EagerSpider(YeldingRequestsSpider):
@ -190,13 +197,9 @@ class CrawlTestCase(TestCase):
}
}
crawler = CrawlerRunner(settings).create_crawler(EagerSpider)
"""
number_of_start_requests should be big enough, so scheduling such amount
of requests takes longer than crawling first of them
"""
yield crawler.crawl(
mockserver=self.mockserver,
number_of_start_requests=100,
number_of_start_requests=self.number_of_start_requests_fro_eagernees_testing(),
)
self.assertTrue(
self.is_eager(crawler.spider.requests_in_order_of_scheduling)
@ -214,13 +217,9 @@ class CrawlTestCase(TestCase):
}
}
crawler = CrawlerRunner(settings).create_crawler(YeldingRequestsSpider)
"""
number_of_start_requests should be big enough, so scheduling such amount
of requests takes longer than crawling first of them
"""
yield crawler.crawl(
mockserver=self.mockserver,
number_of_start_requests=100,
number_of_start_requests=self.number_of_start_requests_fro_eagernees_testing(),
)
self.assertFalse(
self.is_eager(crawler.spider.requests_in_order_of_scheduling)