diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index bec9bdda9..b1110cbd7 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -30,7 +30,7 @@ def get_crawler(spidercls=None, settings_dict=None): from scrapy.spiders import Spider runner = CrawlerRunner(Settings(settings_dict)) - return runner._create_crawler(spidercls or Spider) + return runner.create_crawler(spidercls or Spider) def get_pythonpath(): """Return a PYTHONPATH suitable to use in processes so that they find this diff --git a/tests/test_crawl.py b/tests/test_crawl.py index b2105dcfa..814eb30d2 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -6,12 +6,12 @@ from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase -from scrapy.utils.test import get_crawler +from scrapy.http import Request +from scrapy.crawler import CrawlerRunner from tests import mock from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \ BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider from tests.mockserver import MockServer -from scrapy.http import Request class CrawlTestCase(TestCase): @@ -19,13 +19,14 @@ class CrawlTestCase(TestCase): def setUp(self): self.mockserver = MockServer() self.mockserver.__enter__() + self.runner = CrawlerRunner() def tearDown(self): self.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def test_follow_all(self): - crawler = get_crawler(FollowAllSpider) + crawler = self.runner.create_crawler(FollowAllSpider) yield crawler.crawl() self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url @@ -41,7 +42,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def _test_delay(self, delay, randomize): settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize} - crawler = get_crawler(FollowAllSpider, settings) + crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider) yield crawler.crawl(maxlatency=delay * 2) t = crawler.spider.times totaltime = t[-1] - t[0] @@ -52,7 +53,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_timeout_success(self): - crawler = get_crawler(DelaySpider) + crawler = self.runner.create_crawler(DelaySpider) yield crawler.crawl(n=0.5) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 > 0) @@ -60,7 +61,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_timeout_failure(self): - crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) + crawler = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35}).create_crawler(DelaySpider) yield crawler.crawl(n=0.5) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 == 0) @@ -75,14 +76,14 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_retry_503(self): - crawler = get_crawler(SimpleSpider) + crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: yield crawler.crawl("http://localhost:8998/status?n=503") self._assert_retried(l) @defer.inlineCallbacks def test_retry_conn_failed(self): - crawler = get_crawler(SimpleSpider) + crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: yield crawler.crawl("http://localhost:65432/status?n=503") self._assert_retried(l) @@ -91,7 +92,7 @@ class CrawlTestCase(TestCase): def test_retry_dns_error(self): with mock.patch('socket.gethostbyname', side_effect=socket.gaierror(-5, 'No address associated with hostname')): - crawler = get_crawler(SimpleSpider) + crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: yield crawler.crawl("http://example.com/") self._assert_retried(l) @@ -99,7 +100,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_bug_before_yield(self): with LogCapture('scrapy', level=logging.ERROR) as l: - crawler = get_crawler(BrokenStartRequestsSpider) + crawler = self.runner.create_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_before_yield=1) self.assertEqual(len(l.records), 1) @@ -110,7 +111,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_bug_yielding(self): with LogCapture('scrapy', level=logging.ERROR) as l: - crawler = get_crawler(BrokenStartRequestsSpider) + crawler = self.runner.create_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_yielding=1) self.assertEqual(len(l.records), 1) @@ -121,7 +122,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_lazyness(self): settings = {"CONCURRENT_REQUESTS": 1} - crawler = get_crawler(BrokenStartRequestsSpider, settings) + crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider) yield crawler.crawl() #self.assertTrue(False, crawler.spider.seedsseen) #self.assertTrue(crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99), @@ -130,7 +131,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} - crawler = get_crawler(DuplicateStartRequestsSpider, settings) + crawler = CrawlerRunner(settings).create_crawler(DuplicateStartRequestsSpider) yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3) self.assertEqual(crawler.spider.visited, 6) @@ -159,7 +160,7 @@ Connection: close foo body with multiples lines '''}) - crawler = get_crawler(SimpleSpider) + crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: yield crawler.crawl("http://localhost:8998/raw?{0}".format(query)) self.assertEqual(str(l).count("Got response 200"), 1) @@ -167,7 +168,7 @@ with multiples lines @defer.inlineCallbacks def test_retry_conn_lost(self): # connection lost after receiving data - crawler = get_crawler(SimpleSpider) + crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: yield crawler.crawl("http://localhost:8998/drop?abort=0") self._assert_retried(l) @@ -175,7 +176,7 @@ with multiples lines @defer.inlineCallbacks def test_retry_conn_aborted(self): # connection lost before receiving data - crawler = get_crawler(SimpleSpider) + crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: yield crawler.crawl("http://localhost:8998/drop?abort=1") self._assert_retried(l) @@ -194,7 +195,7 @@ with multiples lines req0.meta['next'] = req1 req1.meta['next'] = req2 req2.meta['next'] = req3 - crawler = get_crawler(SingleRequestSpider) + crawler = self.runner.create_crawler(SingleRequestSpider) yield crawler.crawl(seed=req0) # basic asserts in case of weird communication errors self.assertIn('responses', crawler.spider.meta) @@ -220,7 +221,7 @@ with multiples lines def cb(response): est.append(get_engine_status(crawler.engine)) - crawler = get_crawler(SingleRequestSpider) + crawler = self.runner.create_crawler(SingleRequestSpider) yield crawler.crawl(seed='http://localhost:8998/', callback_func=cb) self.assertEqual(len(est), 1, est) s = dict(est[0]) @@ -244,6 +245,24 @@ with multiples lines def start_requests(self): raise TestError - crawler = get_crawler(FaultySpider) + crawler = self.runner.create_crawler(FaultySpider) yield self.assertFailure(crawler.crawl(), TestError) self.assertFalse(crawler.crawling) + + @defer.inlineCallbacks + def test_crawlerrunner_accepts_crawler(self): + crawler = self.runner.create_crawler(SimpleSpider) + with LogCapture() as log: + yield self.runner.crawl(crawler, "http://localhost:8998/status?n=200") + self.assertIn("Got response 200", str(log)) + + @defer.inlineCallbacks + def test_crawl_multiple(self): + self.runner.crawl(SimpleSpider, "http://localhost:8998/status?n=200") + self.runner.crawl(SimpleSpider, "http://localhost:8998/status?n=503") + + with LogCapture() as log: + yield self.runner.join() + + self._assert_retried(log) + self.assertIn("Got response 200", str(log)) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 7cb5e299b..68dca2e98 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -8,10 +8,12 @@ from twisted.trial import unittest # ugly hack to avoid cyclic imports of scrapy.spiders when running this test # alone +import scrapy from scrapy.interfaces import ISpiderLoader from scrapy.spiderloader import SpiderLoader from scrapy.settings import Settings from scrapy.http import Request +from scrapy.crawler import CrawlerRunner module_dir = os.path.dirname(os.path.abspath(__file__)) @@ -76,3 +78,14 @@ class SpiderLoaderTest(unittest.TestCase): settings = Settings({'SPIDER_MODULES': [module]}) self.spider_loader = SpiderLoader.from_settings(settings) assert len(self.spider_loader._spiders) == 0 + + def test_crawler_runner_loading(self): + module = 'tests.test_spiderloader.test_spiders.spider1' + runner = CrawlerRunner({'SPIDER_MODULES': [module]}) + + self.assertRaisesRegexp(KeyError, 'Spider not found', + runner.create_crawler, 'spider2') + + crawler = runner.create_crawler('spider1') + self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) + self.assertEqual(crawler.spidercls.name, 'spider1')