Merge pull request #3939 from Gallaecio/improve-scrapes-contract

Report all missing fields when a scrapes contract fails

scrapy/contracts/default.py

@@ -84,6 +84,7 @@ class ScrapesContract(Contract):
     def post_process(self, output):
         for x in output:
             if isinstance(x, (BaseItem, dict)):
-                for arg in self.args:
-                    if not arg in x:
-                        raise ContractFail("'%s' field is missing" % arg)
+                missing = [arg for arg in self.args if arg not in x]
+                if missing:
+                    raise ContractFail(
+                        "Missing fields: %s" % ", ".join(missing))

tests/test_contracts.py

@@ -123,6 +123,14 @@ class TestSpider(Spider):
         """
         return {'url': response.url}
 
+    def scrapes_multiple_missing_fields(self, response):
+        """ returns item with no name
+        @url http://scrapy.org
+        @returns items 1 1
+        @scrapes name url
+        """
+        return {}
+
     def parse_no_url(self, response):
         """ method with no url
         @returns items 1 1
@@ -256,6 +264,13 @@ class ContractsManagerTest(unittest.TestCase):
         request.callback(response)
         self.should_fail()
 
+        # scrapes_multiple_missing_fields
+        request = self.conman.from_method(spider.scrapes_multiple_missing_fields, self.results)
+        request.callback(response)
+        self.should_fail()
+        message = 'ContractFail: Missing fields: name, url'
+        assert message in self.results.failures[-1][-1]
+
     def test_custom_contracts(self):
         self.conman.from_spider(CustomContractSuccessSpider(), self.results)
         self.should_succeed()