Merge pull request #3988 from elacuesta/contracts_cb_kwargs

CallbackKeywordArgumentsContract

docs/topics/contracts.rst

@@ -35,12 +35,20 @@ This callback is tested using three built-in contracts:
 
 .. class:: UrlContract
 
-    This contract (``@url``) sets the sample url used when checking other
+    This contract (``@url``) sets the sample URL used when checking other
     contract conditions for this spider. This contract is mandatory. All
     callbacks lacking this contract are ignored when running the checks::
 
     @url url
 
+.. class:: CallbackKeywordArgumentsContract
+
+    This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs <scrapy.http.Request.cb_kwargs>`
+    attribute for the sample request. It must be a valid JSON dictionary.
+    ::
+
+    @cb_kwargs {"arg1": "value1", "arg2": "value2", ...}
+
 .. class:: ReturnsContract
 
     This contract (``@returns``) sets lower and upper bounds for the items and

scrapy/contracts/__init__.py

@@ -90,9 +90,9 @@ class ContractsManager(object):
         cb = request.callback
 
         @wraps(cb)
-        def cb_wrapper(response):
+        def cb_wrapper(response, **cb_kwargs):
             try:
-                output = cb(response)
+                output = cb(response, **cb_kwargs)
                 output = list(iterate_spider_output(output))
             except Exception:
                 case = _create_testcase(method, 'callback')
@@ -121,7 +121,7 @@ class Contract(object):
             cb = request.callback
 
             @wraps(cb)
-            def wrapper(response):
+            def wrapper(response, **cb_kwargs):
                 try:
                     results.startTest(self.testcase_pre)
                     self.pre_process(response)
@@ -133,7 +133,7 @@ class Contract(object):
                 else:
                     results.addSuccess(self.testcase_pre)
                 finally:
-                    return list(iterate_spider_output(cb(response)))
+                    return list(iterate_spider_output(cb(response, **cb_kwargs)))
 
             request.callback = wrapper
 
@@ -144,8 +144,8 @@ class Contract(object):
             cb = request.callback
 
             @wraps(cb)
-            def wrapper(response):
-                output = list(iterate_spider_output(cb(response)))
+            def wrapper(response, **cb_kwargs):
+                output = list(iterate_spider_output(cb(response, **cb_kwargs)))
                 try:
                     results.startTest(self.testcase_post)
                     self.post_process(output)

scrapy/contracts/default.py

@@ -1,3 +1,5 @@
+import json
+
 from scrapy.item import BaseItem
 from scrapy.http import Request
 from scrapy.exceptions import ContractFail
@@ -18,6 +20,20 @@ class UrlContract(Contract):
         return args
 
 
+class CallbackKeywordArgumentsContract(Contract):
+    """ Contract to set the keyword arguments for the request.
+        The value should be a JSON-encoded dictionary, e.g.:
+
+        @cb_kwargs {"arg1": "some value"}
+    """
+
+    name = 'cb_kwargs'
+
+    def adjust_request_args(self, args):
+        args['cb_kwargs'] = json.loads(' '.join(self.args))
+        return args
+
+
 class ReturnsContract(Contract):
     """ Contract to check the output of a callback
 

scrapy/settings/default_settings.py

@@ -291,6 +291,7 @@ TELNETCONSOLE_PASSWORD = None
 SPIDER_CONTRACTS = {}
 SPIDER_CONTRACTS_BASE = {
     'scrapy.contracts.default.UrlContract': 1,
+    'scrapy.contracts.default.CallbackKeywordArgumentsContract': 1,
     'scrapy.contracts.default.ReturnsContract': 2,
     'scrapy.contracts.default.ScrapesContract': 3,
 }

tests/test_contracts.py

@@ -14,6 +14,7 @@ from scrapy.item import Item, Field
 from scrapy.contracts import ContractsManager, Contract
 from scrapy.contracts.default import (
     UrlContract,
+    CallbackKeywordArgumentsContract,
     ReturnsContract,
     ScrapesContract,
 )
@@ -70,6 +71,37 @@ class TestSpider(Spider):
         """
         return TestItem(url=response.url)
 
+    def returns_request_cb_kwargs(self, response, url):
+        """ method which returns request
+        @url https://example.org
+        @cb_kwargs {"url": "http://scrapy.org"}
+        @returns requests 1
+        """
+        return Request(url, callback=self.returns_item_cb_kwargs)
+
+    def returns_item_cb_kwargs(self, response, name):
+        """ method which returns item
+        @url http://scrapy.org
+        @cb_kwargs {"name": "Scrapy"}
+        @returns items 1 1
+        """
+        return TestItem(name=name, url=response.url)
+
+    def returns_item_cb_kwargs_error_unexpected_keyword(self, response):
+        """ method which returns item
+        @url http://scrapy.org
+        @cb_kwargs {"arg": "value"}
+        @returns items 1 1
+        """
+        return TestItem(url=response.url)
+
+    def returns_item_cb_kwargs_error_missing_argument(self, response, arg):
+        """ method which returns item
+        @url http://scrapy.org
+        @returns items 1 1
+        """
+        return TestItem(url=response.url)
+
     def returns_dict_item(self, response):
         """ method which returns item
         @url http://scrapy.org
@@ -172,6 +204,7 @@ class InheritsTestSpider(TestSpider):
 class ContractsManagerTest(unittest.TestCase):
     contracts = [
         UrlContract,
+        CallbackKeywordArgumentsContract,
         ReturnsContract,
         ScrapesContract,
         CustomFormContract,
@@ -211,6 +244,51 @@ class ContractsManagerTest(unittest.TestCase):
         request = self.conman.from_method(spider.parse_no_url, self.results)
         self.assertEqual(request, None)
 
+    def test_cb_kwargs(self):
+        spider = TestSpider()
+        response = ResponseMock()
+
+        # extract contracts correctly
+        contracts = self.conman.extract_contracts(spider.returns_request_cb_kwargs)
+        self.assertEqual(len(contracts), 3)
+        self.assertEqual(frozenset(type(x) for x in contracts),
+                         frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]))
+        
+        contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs)
+        self.assertEqual(len(contracts), 3)
+        self.assertEqual(frozenset(type(x) for x in contracts),
+                         frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]))
+
+        contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs_error_unexpected_keyword)
+        self.assertEqual(len(contracts), 3)
+        self.assertEqual(frozenset(type(x) for x in contracts),
+                         frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]))
+
+        contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs_error_missing_argument)
+        self.assertEqual(len(contracts), 2)
+        self.assertEqual(frozenset(type(x) for x in contracts),
+                         frozenset([UrlContract, ReturnsContract]))
+
+        # returns_request
+        request = self.conman.from_method(spider.returns_request_cb_kwargs, self.results)
+        request.callback(response, **request.cb_kwargs)
+        self.should_succeed()
+
+        # returns_item
+        request = self.conman.from_method(spider.returns_item_cb_kwargs, self.results)
+        request.callback(response, **request.cb_kwargs)
+        self.should_succeed()
+
+        # returns_item (error, callback doesn't take keyword arguments)
+        request = self.conman.from_method(spider.returns_item_cb_kwargs_error_unexpected_keyword, self.results)
+        request.callback(response, **request.cb_kwargs)
+        self.should_error()
+
+        # returns_item (error, contract doesn't provide keyword arguments)
+        request = self.conman.from_method(spider.returns_item_cb_kwargs_error_missing_argument, self.results)
+        request.callback(response, **request.cb_kwargs)
+        self.should_error()
+
     def test_returns(self):
         spider = TestSpider()
         response = ResponseMock()