RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #177 from alexcepoi/contracts_fixes 

Improve feedback on spider contracts run
This commit is contained in:
Pablo Hoffman 2012-11-07 09:40:21 -08:00
parent aa0e02dc54 0350920f1d
commit c01f81ab44
4 changed files with 97 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
scrapy/commands/check.py
View File

@ -1,11 +1,14 @@
from collections import defaultdict
from functools import wraps
from unittest import TextTestRunner
from scrapy import signals
from scrapy.command import ScrapyCommand
from scrapy.contracts import ContractsManager
from scrapy.utils.misc import load_object
from scrapy.utils.spider import iterate_spider_output
from scrapy.utils.conf import build_component_list
from scrapy.xlib.pydispatch import dispatcher
def _generate(cb):
@ -31,6 +34,8 @@ class Command(ScrapyCommand):
ScrapyCommand.add_options(self, parser)
parser.add_option("-l", "--list", dest="list", action="store_true",
help="only list contracts, without checking them")
parser.add_option("-v", "--verbose", dest="verbose", default=1, action="count",
help="print all contract hooks")
def run(self, args, opts):
# load contracts
@ -39,6 +44,7 @@ class Command(ScrapyCommand):
self.settings['SPIDER_CONTRACTS'],
)
self.conman = ContractsManager([load_object(c) for c in contracts])
self.results = TextTestRunner(verbosity=opts.verbose)._makeResult()
# contract requests
contract_reqs = defaultdict(list)
@ -61,6 +67,8 @@ class Command(ScrapyCommand):
for method in sorted(methods):
print ' * %s' % method
else:
dispatcher.connect(self.results.printErrors,
signals.engine_stopped)
self.crawler.start()
def get_requests(self, spider):
@ -69,7 +77,7 @@ class Command(ScrapyCommand):
for key, value in vars(type(spider)).items():
if callable(value) and value.__doc__:
bound_method = value.__get__(spider, type(spider))
request = self.conman.from_method(bound_method)
request = self.conman.from_method(bound_method, self.results)
if request:
request.callback = _generate(request.callback)

91
scrapy/contracts/__init__.py
View File

@ -1,10 +1,11 @@
import sys
import re
from functools import wraps
from unittest import TestCase
from scrapy.http import Request
from scrapy.utils.spider import iterate_spider_output
from scrapy.utils.python import get_spec
from scrapy.exceptions import ContractFail
class ContractsManager(object):
@ -27,7 +28,7 @@ class ContractsManager(object):
return contracts
def from_method(self, method, fail=False):
def from_method(self, method, results):
contracts = self.extract_contracts(method)
if contracts:
# calculate request args
@ -43,9 +44,9 @@ class ContractsManager(object):
# execute pre and post hooks in order
for contract in reversed(contracts):
request = contract.add_pre_hook(request, fail)
request = contract.add_pre_hook(request, results)
for contract in contracts:
request = contract.add_post_hook(request, fail)
request = contract.add_post_hook(request, results)
return request
@ -54,49 +55,67 @@ class Contract(object):
""" Abstract class for contracts """
def __init__(self, method, *args):
self.method = method
self.testcase_pre = self.create_testcase(method, 'pre-hook')
self.testcase_post = self.create_testcase(method, 'post-hook')
self.args = args
def add_pre_hook(self, request, fail=False):
cb = request.callback
def create_testcase(self, method, hook):
spider = method.__self__.name
@wraps(cb)
def wrapper(response):
try:
self.pre_process(response)
except ContractFail as e:
if fail:
raise
class ContractTestCase(TestCase):
def __str__(_self):
return "[%s] %s (@%s %s)" % (spider, method.__name__, self.name, hook)
name = '%s_%s' % (spider, method.__name__)
setattr(ContractTestCase, name, lambda x: x)
return ContractTestCase(name)
def add_pre_hook(self, request, results):
if hasattr(self, 'pre_process'):
cb = request.callback
@wraps(cb)
def wrapper(response):
try:
results.startTest(self.testcase_pre)
self.pre_process(response)
results.stopTest(self.testcase_pre)
except AssertionError:
results.addFailure(self.testcase_pre, sys.exc_info())
except Exception:
results.addError(self.testcase_pre, sys.exc_info())
else:
print e.format(self.method)
return list(iterate_spider_output(cb(response)))
results.addSuccess(self.testcase_pre)
finally:
return list(iterate_spider_output(cb(response)))
request.callback = wrapper
request.callback = wrapper
return request
def add_post_hook(self, request, fail=False):
cb = request.callback
def add_post_hook(self, request, results):
if hasattr(self, 'post_process'):
cb = request.callback
@wraps(cb)
def wrapper(response):
output = list(iterate_spider_output(cb(response)))
try:
self.post_process(output)
except ContractFail as e:
if fail:
raise
@wraps(cb)
def wrapper(response):
try:
output = list(iterate_spider_output(cb(response)))
results.startTest(self.testcase_post)
self.post_process(output)
results.stopTest(self.testcase_post)
except AssertionError:
results.addFailure(self.testcase_post, sys.exc_info())
except Exception:
results.addError(self.testcase_post, sys.exc_info())
else:
print e.format(self.method)
return output
results.addSuccess(self.testcase_post)
finally:
return output
request.callback = wrapper
request.callback = wrapper
return request
def adjust_request_args(self, args):
return args
def pre_process(self, response):
pass
def post_process(self, output):
pass

5
scrapy/exceptions.py
View File

@ -52,7 +52,4 @@ class ScrapyDeprecationWarning(Warning):
class ContractFail(AssertionError):
"""Error raised in case of a failing contract"""
def format(self, method):
return '[FAILED] %s:%s\n>>> %s\n' % \
(method.im_class.name, method.__name__, self)
pass

48
scrapy/tests/test_contracts.py
View File

@ -1,9 +1,10 @@
from unittest import TextTestRunner
from twisted.trial import unittest
from scrapy.spider import BaseSpider
from scrapy.http import Request
from scrapy.item import Item, Field
from scrapy.exceptions import ContractFail
from scrapy.contracts import ContractsManager
from scrapy.contracts.default import (
UrlContract,
@ -71,54 +72,69 @@ class TestSpider(BaseSpider):
class ContractsManagerTest(unittest.TestCase):
contracts = [UrlContract, ReturnsContract, ScrapesContract]
def setUp(self):
self.conman = ContractsManager(self.contracts)
self.results = TextTestRunner()._makeResult()
self.results.stream = None
def should_succeed(self):
self.assertFalse(self.results.failures)
self.assertFalse(self.results.errors)
def should_fail(self):
self.assertTrue(self.results.failures)
self.assertFalse(self.results.errors)
def test_contracts(self):
conman = ContractsManager(self.contracts)
spider = TestSpider()
# extract contracts correctly
contracts = conman.extract_contracts(TestSpider.returns_request)
contracts = self.conman.extract_contracts(spider.returns_request)
self.assertEqual(len(contracts), 2)
self.assertEqual(frozenset(map(type, contracts)),
frozenset([UrlContract, ReturnsContract]))
# returns request for valid method
request = conman.from_method(TestSpider.returns_request)
request = self.conman.from_method(spider.returns_request, self.results)
self.assertNotEqual(request, None)
# no request for missing url
request = conman.from_method(TestSpider.parse_no_url)
request = self.conman.from_method(spider.parse_no_url, self.results)
self.assertEqual(request, None)
def test_returns(self):
conman = ContractsManager(self.contracts)
spider = TestSpider()
response = ResponseMock()
# returns_item
request = conman.from_method(spider.returns_item, fail=True)
request = self.conman.from_method(spider.returns_item, self.results)
output = request.callback(response)
self.assertEqual(map(type, output), [TestItem])
self.should_succeed()
# returns_request
request = conman.from_method(spider.returns_request, fail=True)
request = self.conman.from_method(spider.returns_request, self.results)
output = request.callback(response)
self.assertEqual(map(type, output), [Request])
self.should_succeed()
# returns_fail
request = conman.from_method(spider.returns_fail, fail=True)
self.assertRaises(ContractFail, request.callback, response)
request = self.conman.from_method(spider.returns_fail, self.results)
request.callback(response)
self.should_fail()
def test_scrapes(self):
conman = ContractsManager(self.contracts)
spider = TestSpider()
response = ResponseMock()
# scrapes_item_ok
request = conman.from_method(spider.scrapes_item_ok, fail=True)
request = self.conman.from_method(spider.scrapes_item_ok, self.results)
output = request.callback(response)
self.assertEqual(map(type, output), [TestItem])
self.should_succeed()
# scrapes_item_fail
request = conman.from_method(spider.scrapes_item_fail, fail=True)
self.assertRaises(ContractFail, request.callback, response)
request = self.conman.from_method(spider.scrapes_item_fail,
self.results)
request.callback(response)
self.should_fail()