Merge pull request #5699 from elacuesta/crawlspider-callback-keyword-arguments

CrawlSpider: pass cb_kwargs from process_request

scrapy/spiders/crawl.py

@@ -102,9 +102,9 @@ class CrawlSpider(Spider):
                 request = self._build_request(rule_index, link)
                 yield rule.process_request(request, response)
 
-    def _callback(self, response):
+    def _callback(self, response, **cb_kwargs):
         rule = self._rules[response.meta['rule']]
-        return self._parse_response(response, rule.callback, rule.cb_kwargs, rule.follow)
+        return self._parse_response(response, rule.callback, {**rule.cb_kwargs, **cb_kwargs}, rule.follow)
 
     def _errback(self, failure):
         rule = self._rules[failure.request.meta['rule']]

tests/spiders.py

@@ -419,6 +419,17 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
         self.logger.info('[errback] status %i', failure.value.response.status)
 
 
+class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParseMethod):
+    name = 'crawl_spider_with_process_request_cb_kwargs'
+    rules = (
+        Rule(LinkExtractor(), callback='parse', follow=True, process_request="process_request"),
+    )
+
+    def process_request(self, request, response):
+        request.cb_kwargs["foo"] = "process_request"
+        return request
+
+
 class BytesReceivedCallbackSpider(MetaSpider):
 
     full_response_length = 2**18

tests/test_crawl.py

@@ -41,6 +41,7 @@ from tests.spiders import (
     CrawlSpiderWithAsyncGeneratorCallback,
     CrawlSpiderWithErrback,
     CrawlSpiderWithParseMethod,
+    CrawlSpiderWithProcessRequestCallbackKeywordArguments,
     DelaySpider,
     DuplicateStartRequestsSpider,
     FollowAllSpider,
@@ -426,6 +427,16 @@ class CrawlSpiderTestCase(TestCase):
         self.assertIn("[errback] status 500", str(log))
         self.assertIn("[errback] status 501", str(log))
 
+    @defer.inlineCallbacks
+    def test_crawlspider_process_request_cb_kwargs(self):
+        crawler = get_crawler(CrawlSpiderWithProcessRequestCallbackKeywordArguments)
+        with LogCapture() as log:
+            yield crawler.crawl(mockserver=self.mockserver)
+
+        self.assertIn("[parse] status 200 (foo: process_request)", str(log))
+        self.assertIn("[parse] status 201 (foo: process_request)", str(log))
+        self.assertIn("[parse] status 202 (foo: bar)", str(log))
+
     @defer.inlineCallbacks
     def test_async_def_parse(self):
         crawler = get_crawler(AsyncDefSpider)