From 727e67af5e82128839f084196809fa745bc8921f Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Thu, 28 May 2009 21:41:02 -0300 Subject: [PATCH] spidermw: ignore and warn about requests without callback returned by spiders --- scrapy/spider/middleware.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/scrapy/spider/middleware.py b/scrapy/spider/middleware.py index 8aac7079b..913c9a065 100644 --- a/scrapy/spider/middleware.py +++ b/scrapy/spider/middleware.py @@ -11,6 +11,7 @@ from scrapy.core.exceptions import NotConfigured from scrapy.utils.misc import load_object, arg_to_iter from scrapy.utils.defer import mustbe_deferred, defer_result from scrapy.utils.middleware import build_middleware_list +from scrapy.http import Request from scrapy.conf import settings def _isiterable(possible_iterator): @@ -79,7 +80,8 @@ class SpiderMiddlewareManager(object): assert _isiterable(result), \ 'Middleware %s must returns an iterable object, got %s ' % \ (fname(method), type(result)) - return result + + return self._validate_output(request, result) dfd = mustbe_deferred(process_spider_input, response) dfd.addErrback(process_spider_exception) @@ -90,3 +92,16 @@ class SpiderMiddlewareManager(object): defer_result(result).chainDeferred(request.deferred) request.deferred.addCallback(arg_to_iter) return request.deferred + + def _validate_output(self, request, result): + """Every request returned by spiders must be instanciate with a callback""" + for r in result: + if isinstance(r, Request) and not r.deferred.callbacks: + log.msg('Ignoring %s returned from spider while processing %s: ' \ + 'Request has no callback, try adding callback=self.parse ' \ + 'argument when instanciating Request objects inside your spiders' \ + % (r, request), level=log.WARNING) + continue + yield r + +