diff --git a/docs/index.rst b/docs/index.rst index 6f33ff525..b32c1c42d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -129,6 +129,7 @@ Solving specific problems faq topics/debug + topics/contracts topics/firefox topics/firebug topics/leaks @@ -145,6 +146,9 @@ Solving specific problems :doc:`topics/debug` Learn how to debug common problems of your scrapy spider. +:doc:`topics/contracts` + Learn how to use contracts for testing your spiders. + :doc:`topics/firefox` Learn how to scrape with Firefox and some useful add-ons. diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 8e2150e5f..af65701f4 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -142,6 +142,7 @@ Global commands: Project-only commands: * :command:`crawl` +* :command:`check` * :command:`list` * :command:`edit` * :command:`parse` @@ -221,6 +222,33 @@ Usage examples:: [ ... myspider starts crawling ... ] +.. command:: check + +check +----- + +* Syntax: ``scrapy check [-l] `` +* Requires project: *yes* + +Run contract checks. + +Usage examples:: + + $ scrapy check -l + first_spider + * parse + * parse_item + second_spider + * parse + * parse_item + + $ scrapy check + [FAILED] first_spider:parse_item + >>> 'RetailPricex' field is missing + + [FAILED] first_spider:parse + >>> Returned 92 requests, expected 0..4 + .. command:: server server diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst new file mode 100644 index 000000000..8e98bf749 --- /dev/null +++ b/docs/topics/contracts.rst @@ -0,0 +1,113 @@ +.. _topics-contracts: + +================= +Spiders Contracts +================= + +Testing spiders can get particularly annoying and while nothing prevents you +from writing unit tests the task gets cumbersome quickly. Scrapy offers an +integrated way of testing your spiders by the means of contracts. + +This allows you to test each callback of your spider by hardcoding a sample url +and check various constraints for how the callback processes the response. Each +contract is prefixed with an ``@`` and included in the docstring. See the +following example:: + + def parse(self, response): + """ This function parses a sample response. Some contracts are mingled + with this docstring. + + @url http://www.amazon.com/s?field-keywords=selfish+gene + @returns items 1 16 + @returns requests 0 0 + @scrapes Title Author Year Price + """ + +This callback is tested using three built-in contracts: + +.. module:: scrapy.contracts.default + +.. class:: UrlContract + + This contract (``@url``) sets the sample url used when checking other + contract conditions for this spider. This contract is mandatory. All + callbacks lacking this contract are ignored when running the checks:: + + @url url + +.. class:: ReturnsContract + + This contract (``@returns``) sets lower and upper bounds for the items and + requests returned by the spider. The upper bound is optional:: + + @returns item(s)|request(s) [min [max]] + +.. class:: ScrapesContract + + This contract (``@scrapes``) checks that all the items returned by the + callback have the specified fields:: + + @scrapes field_1 field_2 ... + +Use the :command:`check` command to run the contract checks. + +Custom Contracts +================ + +If you find you need more power than the built-in scrapy contracts you can +create and load your own contracts in the project by using the +:setting:`SPIDER_CONTRACTS` setting:: + + SPIDER_CONTRACTS = { + 'myproject.contracts.ResponseCheck': 10, + 'myproject.contracts.ItemValidate': 10, + } + +Each contract must inherit from :class:`scrapy.contracts.Contract` and can +override three methods: + +.. module:: scrapy.contracts + +.. class:: Contract(method, \*args) + + :param method: callback function to which the contract is associated + :type method: function + + :param args: list of arguments passed into the docstring (whitespace + separated) + :type args: list + + .. method:: Contract.adjust_request_args(args) + + This receives a ``dict`` as an argument containing default arguments + for :class:`~scrapy.http.Request` object. Must return the same or a + modified version of it. + + .. method:: Contract.pre_process(response) + + This allows hooking in various checks on the response received from the + sample request, before it's being passed to the callback. + + .. method:: Contract.post_process(output) + + This allows processing the output of the callback. Iterators are + converted listified before being passed to this hook. + +Here is a demo contract which checks the presence of a custom header in the +response received. Raise :class:`scrapy.exceptions.ContractFail` in order to +get the failures pretty printed:: + + from scrapy.contracts import Contract + from scrapy.exceptions import ContractFail + + class HasHeaderContract(Contract): + """ Demo contract which checks the presence of a custom header + @has_header X-CustomHeader + """ + + name = 'has_header' + + def pre_process(self, response): + for header in self.args: + if header not in response.headers: + raise ContractFail('X-CustomHeader not present') diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index d479408aa..23339a502 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -694,6 +694,30 @@ The scheduler to use for crawling. .. setting:: SPIDER_MIDDLEWARES + +SPIDER_CONTRACTS +---------------- + +Default:: ``{}`` + +A dict containing the scrapy contracts enabled in your project, used for +testing spiders. For more info see :ref:`topics-testing`. + +SPIDER_CONTRACTS_BASE +--------------------- + +Default:: + + { + 'scrapy.contracts.default.UrlContract' : 1, + 'scrapy.contracts.default.ReturnsContract': 2, + 'scrapy.contracts.default.ScrapesContract': 3, + } + +A dict containing the scrapy contracts enabled by default in Scrapy. You should +never modify this setting in your project, modify :setting:`SPIDER_CONTRACTS` +instead. For more info see :ref:`topics-testing`. + SPIDER_MIDDLEWARES ------------------ diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py new file mode 100644 index 000000000..13190e025 --- /dev/null +++ b/scrapy/commands/check.py @@ -0,0 +1,78 @@ +from collections import defaultdict +from functools import wraps + +from scrapy.command import ScrapyCommand +from scrapy.contracts import ContractsManager +from scrapy.utils.misc import load_object +from scrapy.utils.spider import iterate_spider_output +from scrapy.utils.conf import build_component_list + + +def _generate(cb): + """ create a callback which does not return anything """ + @wraps(cb) + def wrapper(response): + output = cb(response) + output = list(iterate_spider_output(output)) + return wrapper + + +class Command(ScrapyCommand): + requires_project = True + default_settings = {'LOG_ENABLED': False} + + def syntax(self): + return "[options] " + + def short_desc(self): + return "Check contracts for given spider" + + def add_options(self, parser): + ScrapyCommand.add_options(self, parser) + parser.add_option("-l", "--list", dest="list", action="store_true", + help="only list contracts, without checking them") + + def run(self, args, opts): + # load contracts + contracts = build_component_list( + self.settings['SPIDER_CONTRACTS_BASE'], + self.settings['SPIDER_CONTRACTS'], + ) + self.conman = ContractsManager([load_object(c) for c in contracts]) + + # contract requests + contract_reqs = defaultdict(list) + self.crawler.engine.has_capacity = lambda: True + + for spider in args or self.crawler.spiders.list(): + spider = self.crawler.spiders.create(spider) + requests = self.get_requests(spider) + + if opts.list: + for req in requests: + contract_reqs[spider.name].append(req.callback.__name__) + else: + self.crawler.crawl(spider, requests) + + # start checks + if opts.list: + for spider, methods in sorted(contract_reqs.iteritems()): + print spider + for method in sorted(methods): + print ' * %s' % method + else: + self.crawler.start() + + def get_requests(self, spider): + requests = [] + + for key, value in vars(type(spider)).items(): + if callable(value) and value.__doc__: + bound_method = value.__get__(spider, type(spider)) + request = self.conman.from_method(bound_method) + + if request: + request.callback = _generate(request.callback) + requests.append(request) + + return requests diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py new file mode 100644 index 000000000..fac226822 --- /dev/null +++ b/scrapy/contracts/__init__.py @@ -0,0 +1,102 @@ +import re +from functools import wraps + +from scrapy.http import Request +from scrapy.utils.spider import iterate_spider_output +from scrapy.utils.python import get_spec +from scrapy.exceptions import ContractFail + + +class ContractsManager(object): + contracts = {} + + def __init__(self, contracts): + for contract in contracts: + self.contracts[contract.name] = contract + + def extract_contracts(self, method): + contracts = [] + for line in method.__doc__.split('\n'): + line = line.strip() + + if line.startswith('@'): + name, args = re.match(r'@(\w+)\s*(.*)', line).groups() + args = re.split(r'\s+', args) + + contracts.append(self.contracts[name](method, *args)) + + return contracts + + def from_method(self, method, fail=False): + contracts = self.extract_contracts(method) + if contracts: + # calculate request args + args, kwargs = get_spec(Request.__init__) + kwargs['callback'] = method + for contract in contracts: + kwargs = contract.adjust_request_args(kwargs) + + # create and prepare request + args.remove('self') + if set(args).issubset(set(kwargs)): + request = Request(**kwargs) + + # execute pre and post hooks in order + for contract in reversed(contracts): + request = contract.add_pre_hook(request, fail) + for contract in contracts: + request = contract.add_post_hook(request, fail) + + return request + + +class Contract(object): + """ Abstract class for contracts """ + + def __init__(self, method, *args): + self.method = method + self.args = args + + def add_pre_hook(self, request, fail=False): + cb = request.callback + + @wraps(cb) + def wrapper(response): + try: + self.pre_process(response) + except ContractFail as e: + if fail: + raise + else: + print e.format(self.method) + return list(iterate_spider_output(cb(response))) + + request.callback = wrapper + return request + + def add_post_hook(self, request, fail=False): + cb = request.callback + + @wraps(cb) + def wrapper(response): + output = list(iterate_spider_output(cb(response))) + try: + self.post_process(output) + except ContractFail as e: + if fail: + raise + else: + print e.format(self.method) + return output + + request.callback = wrapper + return request + + def adjust_request_args(self, args): + return args + + def pre_process(self, response): + pass + + def post_process(self, output): + pass diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py new file mode 100644 index 000000000..73a1447f0 --- /dev/null +++ b/scrapy/contracts/default.py @@ -0,0 +1,89 @@ +from scrapy.item import BaseItem +from scrapy.http import Request +from scrapy.exceptions import ContractFail + +from . import Contract + + +# contracts +class UrlContract(Contract): + """ Contract to set the url of the request (mandatory) + @url http://scrapy.org + """ + + name = 'url' + + def adjust_request_args(self, args): + args['url'] = self.args[0] + return args + + +class ReturnsContract(Contract): + """ Contract to check the output of a callback + + general form: + @returns request(s)/item(s) [min=1 [max]] + + e.g.: + @returns request + @returns request 2 + @returns request 2 10 + @returns request 0 10 + """ + + name = 'returns' + objects = { + 'request': Request, + 'requests': Request, + 'item': BaseItem, + 'items': BaseItem, + } + + def __init__(self, *args, **kwargs): + super(ReturnsContract, self).__init__(*args, **kwargs) + + assert len(self.args) in [1, 2, 3] + self.obj_name = self.args[0] or None + self.obj_type = self.objects[self.obj_name] + + try: + self.min_bound = int(self.args[1]) + except IndexError: + self.min_bound = 1 + + try: + self.max_bound = int(self.args[2]) + except IndexError: + self.max_bound = float('inf') + + def post_process(self, output): + occurences = 0 + for x in output: + if isinstance(x, self.obj_type): + occurences += 1 + + assertion = (self.min_bound <= occurences <= self.max_bound) + + if not assertion: + if self.min_bound == self.max_bound: + expected = self.min_bound + else: + expected = '%s..%s' % (self.min_bound, self.max_bound) + + raise ContractFail("Returned %s %s, expected %s" % \ + (occurences, self.obj_name, expected)) + + +class ScrapesContract(Contract): + """ Contract to check presence of fields in scraped items + @scrapes page_name page_body + """ + + name = 'scrapes' + + def post_process(self, output): + for x in output: + if isinstance(x, BaseItem): + for arg in self.args: + if not arg in x: + raise ContractFail("'%s' field is missing" % arg) diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 8f29a4c8d..f41403bb4 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -50,3 +50,9 @@ class ScrapyDeprecationWarning(Warning): """ pass +class ContractFail(AssertionError): + """Error raised in case of a failing contract""" + + def format(self, method): + return '[FAILED] %s:%s\n>>> %s\n' % \ + (method.im_class.name, method.__name__, self) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index b688de178..b70b2cadc 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -237,3 +237,10 @@ WEBSERVICE_RESOURCES_BASE = { 'scrapy.contrib.webservice.enginestatus.EngineStatusResource': 1, 'scrapy.contrib.webservice.stats.StatsResource': 1, } + +SPIDER_CONTRACTS = {} +SPIDER_CONTRACTS_BASE = { + 'scrapy.contracts.default.UrlContract' : 1, + 'scrapy.contracts.default.ReturnsContract': 2, + 'scrapy.contracts.default.ScrapesContract': 3, +} diff --git a/scrapy/tests/test_contracts.py b/scrapy/tests/test_contracts.py new file mode 100644 index 000000000..b35e57519 --- /dev/null +++ b/scrapy/tests/test_contracts.py @@ -0,0 +1,124 @@ +from twisted.trial import unittest + +from scrapy.spider import BaseSpider +from scrapy.http import Request +from scrapy.item import Item, Field +from scrapy.exceptions import ContractFail +from scrapy.contracts import ContractsManager +from scrapy.contracts.default import ( + UrlContract, + ReturnsContract, + ScrapesContract, +) + + +class TestItem(Item): + name = Field() + url = Field() + + +class ResponseMock(object): + url = 'http://scrapy.org' + + +class TestSpider(BaseSpider): + name = 'demo_spider' + + def returns_request(self, response): + """ method which returns request + @url http://scrapy.org + @returns requests 1 + """ + return Request('http://scrapy.org', callback=self.returns_item) + + def returns_item(self, response): + """ method which returns item + @url http://scrapy.org + @returns items 1 1 + """ + return TestItem(url=response.url) + + def returns_fail(self, response): + """ method which returns item + @url http://scrapy.org + @returns items 0 0 + """ + return TestItem(url=response.url) + + def scrapes_item_ok(self, response): + """ returns item with name and url + @url http://scrapy.org + @returns items 1 1 + @scrapes name url + """ + return TestItem(name='test', url=response.url) + + def scrapes_item_fail(self, response): + """ returns item with no name + @url http://scrapy.org + @returns items 1 1 + @scrapes name url + """ + return TestItem(url=response.url) + + def parse_no_url(self, response): + """ method with no url + @returns items 1 1 + """ + pass + + +class ContractsManagerTest(unittest.TestCase): + contracts = [UrlContract, ReturnsContract, ScrapesContract] + + def test_contracts(self): + conman = ContractsManager(self.contracts) + + # extract contracts correctly + contracts = conman.extract_contracts(TestSpider.returns_request) + self.assertEqual(len(contracts), 2) + self.assertEqual(frozenset(map(type, contracts)), + frozenset([UrlContract, ReturnsContract])) + + # returns request for valid method + request = conman.from_method(TestSpider.returns_request) + self.assertIsNotNone(request) + + # no request for missing url + request = conman.from_method(TestSpider.parse_no_url) + self.assertIsNone(request) + + def test_returns(self): + conman = ContractsManager(self.contracts) + + spider = TestSpider() + response = ResponseMock() + + # returns_item + request = conman.from_method(spider.returns_item, fail=True) + output = request.callback(response) + self.assertEqual(map(type, output), [TestItem]) + + # returns_request + request = conman.from_method(spider.returns_request, fail=True) + output = request.callback(response) + self.assertEqual(map(type, output), [Request]) + + # returns_fail + request = conman.from_method(spider.returns_fail, fail=True) + self.assertRaises(ContractFail, request.callback, response) + + def test_scrapes(self): + conman = ContractsManager(self.contracts) + + spider = TestSpider() + response = ResponseMock() + + # scrapes_item_ok + request = conman.from_method(spider.scrapes_item_ok, fail=True) + output = request.callback(response) + self.assertEqual(map(type, output), [TestItem]) + + # scrapes_item_fail + request = conman.from_method(spider.scrapes_item_fail, fail=True) + self.assertRaises(ContractFail, request.callback, response) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index fe9b6d058..543892d7e 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -103,4 +103,3 @@ def md5sum(file): break m.update(d) return m.hexdigest() - diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 53158fa92..44808e488 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -167,6 +167,42 @@ def get_func_args(func, stripself=False): func_args.pop(0) return func_args +def get_spec(func): + """Returns (args, kwargs) tuple for a function + >>> import re + >>> get_spec(re.match) + (['pattern', 'string'], {'flags': 0}) + + >>> class Test(object): + ... def __call__(self, val): + ... pass + ... def method(self, val, flags=0): + ... pass + + >>> get_spec(Test) + (['self', 'val'], {}) + + >>> get_spec(Test.method) + (['self', 'val'], {'flags': 0}) + + >>> get_spec(Test().method) + (['self', 'val'], {'flags': 0}) + """ + + if inspect.isfunction(func) or inspect.ismethod(func): + spec = inspect.getargspec(func) + elif hasattr(func, '__call__'): + spec = inspect.getargspec(func.__call__) + else: + raise TypeError('%s is not callable' % type(func)) + + defaults = spec.defaults or [] + + firstdefault = len(spec.args) - len(defaults) + args = spec.args[:firstdefault] + kwargs = dict(zip(spec.args[firstdefault:], defaults)) + return args, kwargs + def equal_attributes(obj1, obj2, attributes): """Compare two objects attributes""" # not attributes given return False by default