diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 957761b76..62f9a743b 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -35,12 +35,20 @@ This callback is tested using three built-in contracts: .. class:: UrlContract - This contract (``@url``) sets the sample url used when checking other + This contract (``@url``) sets the sample URL used when checking other contract conditions for this spider. This contract is mandatory. All callbacks lacking this contract are ignored when running the checks:: @url url +.. class:: CallbackKeywordArgumentsContract + + This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs ` + attribute for the sample request. It must be a valid JSON dictionary. + :: + + @cb_kwargs {"arg1": "value1", "arg2": "value2", ...} + .. class:: ReturnsContract This contract (``@returns``) sets lower and upper bounds for the items and diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 536bbdafb..7b6591d86 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -90,9 +90,9 @@ class ContractsManager(object): cb = request.callback @wraps(cb) - def cb_wrapper(response): + def cb_wrapper(response, **cb_kwargs): try: - output = cb(response) + output = cb(response, **cb_kwargs) output = list(iterate_spider_output(output)) except Exception: case = _create_testcase(method, 'callback') @@ -121,7 +121,7 @@ class Contract(object): cb = request.callback @wraps(cb) - def wrapper(response): + def wrapper(response, **cb_kwargs): try: results.startTest(self.testcase_pre) self.pre_process(response) @@ -133,7 +133,7 @@ class Contract(object): else: results.addSuccess(self.testcase_pre) finally: - return list(iterate_spider_output(cb(response))) + return list(iterate_spider_output(cb(response, **cb_kwargs))) request.callback = wrapper @@ -144,8 +144,8 @@ class Contract(object): cb = request.callback @wraps(cb) - def wrapper(response): - output = list(iterate_spider_output(cb(response))) + def wrapper(response, **cb_kwargs): + output = list(iterate_spider_output(cb(response, **cb_kwargs))) try: results.startTest(self.testcase_post) self.post_process(output) diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 7745959a7..24f6c2e77 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -1,3 +1,5 @@ +import json + from scrapy.item import BaseItem from scrapy.http import Request from scrapy.exceptions import ContractFail @@ -18,6 +20,20 @@ class UrlContract(Contract): return args +class CallbackKeywordArgumentsContract(Contract): + """ Contract to set the keyword arguments for the request. + The value should be a JSON-encoded dictionary, e.g.: + + @cb_kwargs {"arg1": "some value"} + """ + + name = 'cb_kwargs' + + def adjust_request_args(self, args): + args['cb_kwargs'] = json.loads(' '.join(self.args)) + return args + + class ReturnsContract(Contract): """ Contract to check the output of a callback diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 5884dfc60..05ab4b628 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -291,6 +291,7 @@ TELNETCONSOLE_PASSWORD = None SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { 'scrapy.contracts.default.UrlContract': 1, + 'scrapy.contracts.default.CallbackKeywordArgumentsContract': 1, 'scrapy.contracts.default.ReturnsContract': 2, 'scrapy.contracts.default.ScrapesContract': 3, } diff --git a/tests/test_contracts.py b/tests/test_contracts.py index a728099c0..b2e358700 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -14,6 +14,7 @@ from scrapy.item import Item, Field from scrapy.contracts import ContractsManager, Contract from scrapy.contracts.default import ( UrlContract, + CallbackKeywordArgumentsContract, ReturnsContract, ScrapesContract, ) @@ -70,6 +71,37 @@ class TestSpider(Spider): """ return TestItem(url=response.url) + def returns_request_cb_kwargs(self, response, url): + """ method which returns request + @url https://example.org + @cb_kwargs {"url": "http://scrapy.org"} + @returns requests 1 + """ + return Request(url, callback=self.returns_item_cb_kwargs) + + def returns_item_cb_kwargs(self, response, name): + """ method which returns item + @url http://scrapy.org + @cb_kwargs {"name": "Scrapy"} + @returns items 1 1 + """ + return TestItem(name=name, url=response.url) + + def returns_item_cb_kwargs_error_unexpected_keyword(self, response): + """ method which returns item + @url http://scrapy.org + @cb_kwargs {"arg": "value"} + @returns items 1 1 + """ + return TestItem(url=response.url) + + def returns_item_cb_kwargs_error_missing_argument(self, response, arg): + """ method which returns item + @url http://scrapy.org + @returns items 1 1 + """ + return TestItem(url=response.url) + def returns_dict_item(self, response): """ method which returns item @url http://scrapy.org @@ -172,6 +204,7 @@ class InheritsTestSpider(TestSpider): class ContractsManagerTest(unittest.TestCase): contracts = [ UrlContract, + CallbackKeywordArgumentsContract, ReturnsContract, ScrapesContract, CustomFormContract, @@ -211,6 +244,51 @@ class ContractsManagerTest(unittest.TestCase): request = self.conman.from_method(spider.parse_no_url, self.results) self.assertEqual(request, None) + def test_cb_kwargs(self): + spider = TestSpider() + response = ResponseMock() + + # extract contracts correctly + contracts = self.conman.extract_contracts(spider.returns_request_cb_kwargs) + self.assertEqual(len(contracts), 3) + self.assertEqual(frozenset(type(x) for x in contracts), + frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract])) + + contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs) + self.assertEqual(len(contracts), 3) + self.assertEqual(frozenset(type(x) for x in contracts), + frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract])) + + contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs_error_unexpected_keyword) + self.assertEqual(len(contracts), 3) + self.assertEqual(frozenset(type(x) for x in contracts), + frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract])) + + contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs_error_missing_argument) + self.assertEqual(len(contracts), 2) + self.assertEqual(frozenset(type(x) for x in contracts), + frozenset([UrlContract, ReturnsContract])) + + # returns_request + request = self.conman.from_method(spider.returns_request_cb_kwargs, self.results) + request.callback(response, **request.cb_kwargs) + self.should_succeed() + + # returns_item + request = self.conman.from_method(spider.returns_item_cb_kwargs, self.results) + request.callback(response, **request.cb_kwargs) + self.should_succeed() + + # returns_item (error, callback doesn't take keyword arguments) + request = self.conman.from_method(spider.returns_item_cb_kwargs_error_unexpected_keyword, self.results) + request.callback(response, **request.cb_kwargs) + self.should_error() + + # returns_item (error, contract doesn't provide keyword arguments) + request = self.conman.from_method(spider.returns_item_cb_kwargs_error_missing_argument, self.results) + request.callback(response, **request.cb_kwargs) + self.should_error() + def test_returns(self): spider = TestSpider() response = ResponseMock()