diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 2d61026e9..a912ff986 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -46,6 +46,14 @@ This callback is tested using three built-in contracts: @cb_kwargs {"arg1": "value1", "arg2": "value2", ...} +.. class:: MetadataContract + + This contract (``@meta``) sets the :attr:` meta ` + attribute for the sample request. It must be a valid JSON dictionary. + :: + + @meta {"arg1": "value1", "arg2": "value2", ...} + .. class:: ReturnsContract This contract (``@returns``) sets lower and upper bounds for the items and diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 71ca4168a..87099b950 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -35,6 +35,20 @@ class CallbackKeywordArgumentsContract(Contract): return args +class MetadataContract(Contract): + """Contract to key metadata arguments for the request. + The value should be JSON-encoded dictionary, e.g.: + + @meta {"arg1": "some value"} + """ + + name = "meta" + + def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: + args["meta"] = json.loads(" ".join(self.args)) + return args + + class ReturnsContract(Contract): """Contract to check the output of a callback diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 932475fb5..7ba0128a5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -333,6 +333,7 @@ SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { "scrapy.contracts.default.UrlContract": 1, "scrapy.contracts.default.CallbackKeywordArgumentsContract": 1, + "scrapy.contracts.default.MetadataContract": 1, "scrapy.contracts.default.ReturnsContract": 2, "scrapy.contracts.default.ScrapesContract": 3, } diff --git a/tests/test_contracts.py b/tests/test_contracts.py index c9c12f0d8..d578b3af4 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -8,6 +8,7 @@ from scrapy import FormRequest from scrapy.contracts import Contract, ContractsManager from scrapy.contracts.default import ( CallbackKeywordArgumentsContract, + MetadataContract, ReturnsContract, ScrapesContract, UrlContract, @@ -29,6 +30,10 @@ class ResponseMock: url = "http://scrapy.org" +class ResponseMetaMock(ResponseMock): + meta = None + + class CustomSuccessContract(Contract): name = "custom_success_contract" @@ -195,6 +200,33 @@ class TestSpider(Spider): """ pass + def returns_request_meta(self, response): + """method which returns request + @url https://example.org + @meta {"cookiejar": "session1"} + @returns requests 1 + """ + return Request( + "https://example.org", meta=response.meta, callback=self.returns_item_meta + ) + + def returns_item_meta(self, response): + """method which returns item + @url http://scrapy.org + @meta {"key": "example"} + @returns items 1 1 + """ + return TestItem(name="example", url=response.url) + + def returns_error_missing_meta(self, response): + """method which depends of metadata be defined + + @url http://scrapy.org + @returns items 1 + """ + key = response.meta["key"] + yield {key: "value"} + class CustomContractSuccessSpider(Spider): name = "custom_contract_success_spider" @@ -224,6 +256,7 @@ class ContractsManagerTest(unittest.TestCase): contracts = [ UrlContract, CallbackKeywordArgumentsContract, + MetadataContract, ReturnsContract, ScrapesContract, CustomFormContract, @@ -328,6 +361,52 @@ class ContractsManagerTest(unittest.TestCase): request.callback(response, **request.cb_kwargs) self.should_error() + def test_meta(self): + spider = TestSpider() + + # extract contracts correctly + contracts = self.conman.extract_contracts(spider.returns_request_meta) + self.assertEqual(len(contracts), 3) + self.assertEqual( + frozenset(type(x) for x in contracts), + frozenset([UrlContract, MetadataContract, ReturnsContract]), + ) + + contracts = self.conman.extract_contracts(spider.returns_item_meta) + self.assertEqual(len(contracts), 3) + self.assertEqual( + frozenset(type(x) for x in contracts), + frozenset([UrlContract, MetadataContract, ReturnsContract]), + ) + + response = ResponseMetaMock() + + # returns_request + request = self.conman.from_method(spider.returns_request_meta, self.results) + assert request.meta["cookiejar"] == "session1" + response.meta = request.meta + request.callback(response) + assert response.meta["cookiejar"] == "session1" + self.should_succeed() + + response = ResponseMetaMock() + + # returns_item + request = self.conman.from_method(spider.returns_item_meta, self.results) + assert request.meta["key"] == "example" + response.meta = request.meta + request.callback(ResponseMetaMock) + assert response.meta["key"] == "example" + self.should_succeed() + + response = ResponseMetaMock() + + request = self.conman.from_method( + spider.returns_error_missing_meta, self.results + ) + request.callback(response) + self.should_error() + def test_returns(self): spider = TestSpider() response = ResponseMock()