RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix tests that had multiple calls to crawl()

This commit is contained in:
Jakob de Maeyer 2015-11-09 16:54:22 +01:00
parent fc26397319
commit 380f76d35f
2 changed files with 13 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
tests/test_crawl.py
View File

@ -61,14 +61,16 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_timeout_failure(self):
crawler = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35}).create_crawler(DelaySpider)
yield crawler.crawl(n=0.5)
runner = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35})
crawler = runner.create_crawler(DelaySpider)
yield runner.crawl(crawler, n=0.5)
self.assertTrue(crawler.spider.t1 > 0)
self.assertTrue(crawler.spider.t2 == 0)
self.assertTrue(crawler.spider.t2_err > 0)
self.assertTrue(crawler.spider.t2_err > crawler.spider.t1)
# server hangs after receiving response headers
yield crawler.crawl(n=0.5, b=1)
crawler = runner.create_crawler(DelaySpider)
yield runner.crawl(crawler, n=0.5, b=1)
self.assertTrue(crawler.spider.t1 > 0)
self.assertTrue(crawler.spider.t2 == 0)
self.assertTrue(crawler.spider.t2_err > 0)
@ -131,11 +133,15 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_start_requests_dupes(self):
settings = {"CONCURRENT_REQUESTS": 1}
crawler = CrawlerRunner(settings).create_crawler(DuplicateStartRequestsSpider)
yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3)
runner = CrawlerRunner(settings)
crawler = runner.create_crawler(DuplicateStartRequestsSpider)
yield runner.crawl(crawler, dont_filter=True, distinct_urls=2,
dupe_factor=3)
self.assertEqual(crawler.spider.visited, 6)
yield crawler.crawl(dont_filter=False, distinct_urls=3, dupe_factor=4)
crawler = runner.create_crawler(DuplicateStartRequestsSpider)
yield runner.crawl(crawler, dont_filter=False, distinct_urls=3,
dupe_factor=4)
self.assertEqual(crawler.spider.visited, 3)
@defer.inlineCallbacks

1
tests/test_downloader_handlers.py
View File

@ -308,6 +308,7 @@ class Http11MockServerTestCase(unittest.TestCase):
# download_maxsize < 100, hence the CancelledError
self.assertIsInstance(failure.value, defer.CancelledError)
crawler = get_crawler(SingleRequestSpider)
request.headers.setdefault('Accept-Encoding', 'gzip,deflate')
request = request.replace(url='http://localhost:8998/xpayload')
yield crawler.crawl(seed=request)