From aacb7965b41b0fa805b1e095cefe8c037a4fc9c1 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sat, 3 Oct 2009 14:08:40 -0300 Subject: [PATCH] improved check for missing callback in Requests --- scrapy/core/engine.py | 4 ++++ scrapy/spider/middleware.py | 16 +--------------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 004a65b98..5b03d4231 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -161,6 +161,10 @@ class ExecutionEngine(object): return self.downloader.sites.keys() def crawl(self, request, spider): + if not request.deferred.callbacks: + log.msg("Unable to crawl Request with no callback: %s" % request, + level=log.ERROR, domain=spider.domain_name) + return schd = mustbe_deferred(self.schedule, request, spider) # FIXME: we can't log errors because we would be preventing them from # propagating to the request errback. This should be fixed after the diff --git a/scrapy/spider/middleware.py b/scrapy/spider/middleware.py index e446a6a24..416b3cc9b 100644 --- a/scrapy/spider/middleware.py +++ b/scrapy/spider/middleware.py @@ -85,23 +85,9 @@ class SpiderMiddlewareManager(object): assert _isiterable(result), \ 'Middleware %s must returns an iterable object, got %s ' % \ (fname(method), type(result)) - - return self._validate_output(request, result, spider) + return result dfd = mustbe_deferred(process_spider_input, response) dfd.addErrback(process_spider_exception) dfd.addCallback(process_spider_output) return dfd - - def _validate_output(self, request, result, spider): - """Every request returned by spiders must be instanciate with a callback""" - for r in result: - if isinstance(r, Request) and not r.deferred.callbacks: - log.msg('Ignoring %s returned from spider while processing %s: ' \ - 'Request has no callback, try adding callback=self.parse ' \ - 'argument when instanciating Request objects inside your spiders' \ - % (r, request), level=log.WARNING, domain=spider.domain_name) - continue - yield r - -