diff --git a/tests/spiders.py b/tests/spiders.py index 5ea8a4a21..7952e3d47 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -419,6 +419,17 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): self.logger.info('[errback] status %i', failure.value.response.status) +class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParseMethod): + name = 'crawl_spider_with_process_request_cb_kwargs' + rules = ( + Rule(LinkExtractor(), callback='parse', follow=True, process_request="process_request"), + ) + + def process_request(self, request, response): + request.cb_kwargs["foo"] = "process_request" + return request + + class BytesReceivedCallbackSpider(MetaSpider): full_response_length = 2**18 diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 5383ec652..5ec96e4a7 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -41,6 +41,7 @@ from tests.spiders import ( CrawlSpiderWithAsyncGeneratorCallback, CrawlSpiderWithErrback, CrawlSpiderWithParseMethod, + CrawlSpiderWithProcessRequestCallbackKeywordArguments, DelaySpider, DuplicateStartRequestsSpider, FollowAllSpider, @@ -426,6 +427,16 @@ class CrawlSpiderTestCase(TestCase): self.assertIn("[errback] status 500", str(log)) self.assertIn("[errback] status 501", str(log)) + @defer.inlineCallbacks + def test_crawlspider_process_request_cb_kwargs(self): + crawler = get_crawler(CrawlSpiderWithProcessRequestCallbackKeywordArguments) + with LogCapture() as log: + yield crawler.crawl(mockserver=self.mockserver) + + self.assertIn("[parse] status 200 (foo: process_request)", str(log)) + self.assertIn("[parse] status 201 (foo: process_request)", str(log)) + self.assertIn("[parse] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks def test_async_def_parse(self): crawler = get_crawler(AsyncDefSpider)