From e7e18db179f2e45aa38a5bbdb0abba7d983cdce7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Jul 2018 14:04:35 -0300 Subject: [PATCH] Fix tests --- tests/test_spidermiddleware.py | 57 +++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 2565ef7af..5622c3179 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -8,10 +8,7 @@ from twisted.internet import defer from scrapy import Spider, Request from scrapy.utils.test import get_crawler from tests.mockserver import MockServer - - -# TEST_URL = 'http://example.org' -TEST_URL = 'http://localhost:8998' +from tests.spiders import MockServerSpider class LogExceptionMiddleware: @@ -21,16 +18,18 @@ class LogExceptionMiddleware: # ================================================================================ -# recover from an exception on a spider callback +# (0) recover from an exception on a spider callback class RecoverySpider(Spider): name = 'RecoverySpider' - start_urls = [TEST_URL] custom_settings = { 'SPIDER_MIDDLEWARES': { __name__ + '.RecoveryMiddleware': 10, }, } + def start_requests(self): + yield Request(self.mockserver.url('/status?n=200')) + def parse(self, response): yield {'test': 1} self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail')) @@ -61,10 +60,11 @@ class ProcessSpiderInputSpider(Spider): } def start_requests(self): - yield Request(TEST_URL, callback=self.parse, errback=self.errback) + yield Request(url=self.mockserver.url('/status?n=200'), + callback=self.parse, errback=self.errback) def parse(self, response): - return [{'test': 1}, {'test': 2}] + return {'from': 'callback'} def errback(self, failure): self.logger.warn('Got a Failure on the Request errback') @@ -80,13 +80,15 @@ class FailProcessSpiderInputMiddleware: # (2) exceptions from a spider callback (generator) class GeneratorCallbackSpider(Spider): name = 'GeneratorCallbackSpider' - start_urls = [TEST_URL] custom_settings = { 'SPIDER_MIDDLEWARES': { __name__ + '.LogExceptionMiddleware': 10, }, } + def start_requests(self): + yield Request(self.mockserver.url('/status?n=200')) + def parse(self, response): yield {'test': 1} yield {'test': 2} @@ -97,13 +99,15 @@ class GeneratorCallbackSpider(Spider): # (3) exceptions from a spider callback (not a generator) class NotAGeneratorCallbackSpider(Spider): name = 'NotAGeneratorCallbackSpider' - start_urls = [TEST_URL] custom_settings = { 'SPIDER_MIDDLEWARES': { __name__ + '.LogExceptionMiddleware': 10, }, } + def start_requests(self): + yield Request(self.mockserver.url('/status?n=200')) + def parse(self, response): return [{'test': 1}, {'test': 1/0}] @@ -123,20 +127,20 @@ class TestSpiderMiddleware(TestCase): def crawl_log(self, spider): crawler = get_crawler(spider) with LogCapture() as log: - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) raise defer.returnValue(log) - # @defer.inlineCallbacks - # def test_recovery(self): - # """ - # Recover from an exception from a spider's callback. The final item count should be 3 - # (one from the spider before raising the exception, one from the middleware and one - # from the spider when processing the response that was enqueued from the middleware) - # """ - # log = yield self.crawl_log(RecoverySpider) - # self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log)) - # self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1) - # self.assertIn("'item_scraped_count': 3", str(log)) + @defer.inlineCallbacks + def test_recovery(self): + """ + (0) Recover from an exception in a spider callback. The final item count should be 2 + (one directly from the recovery middleware and one from the spider when processing + the request that was enqueued from the recovery middleware) + """ + log = yield self.crawl_log(RecoverySpider) + self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log)) + self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1) + self.assertIn("'item_scraped_count': 2", str(log)) @defer.inlineCallbacks def test_process_spider_input_errback(self): @@ -148,21 +152,24 @@ class TestSpiderMiddleware(TestCase): self.assertNotIn("Middleware: IndexError exception caught", str(log1)) self.assertIn("Middleware: will raise IndexError", str(log1)) self.assertIn("Got a Failure on the Request errback", str(log1)) + self.assertIn("{'from': 'errback'}", str(log1)) + self.assertNotIn("{'from': 'callback'}", str(log1)) + self.assertIn("'item_scraped_count': 1", str(log1)) @defer.inlineCallbacks def test_generator_callback(self): """ - (2) An exception from a spider's callback should + (2) An exception from a spider callback (returning a generator) should be caught by the process_spider_exception chain """ log2 = yield self.crawl_log(GeneratorCallbackSpider) self.assertIn("Middleware: ImportError exception caught", str(log2)) - self.assertIn("'item_scraped_count': 2", str(log2)) + self.assertNotIn("item_scraped_count", str(log2)) @defer.inlineCallbacks def test_not_a_generator_callback(self): """ - (3) An exception from a spider's callback should + (3) An exception from a spider callback (returning a list) should be caught by the process_spider_exception chain """ log3 = yield self.crawl_log(NotAGeneratorCallbackSpider)