From fb3a2dab2f6931d9c0a043790fc3227624c7edc4 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Tue, 17 Mar 2009 21:30:33 +0000 Subject: [PATCH] core: allow returning None from spiders to support pipeline/download throttling --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40990 --- scrapy/trunk/scrapy/core/engine.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/trunk/scrapy/core/engine.py b/scrapy/trunk/scrapy/core/engine.py index d4b058427..05f6e1b95 100644 --- a/scrapy/trunk/scrapy/core/engine.py +++ b/scrapy/trunk/scrapy/core/engine.py @@ -250,8 +250,10 @@ class ExecutionEngine(object): elif isinstance(item, Request): signals.send_catch_log(signal=signals.request_received, sender=self.__class__, request=item, spider=spider, response=response) self.crawl(request=item, spider=spider, priority=priority) + elif item is None: + pass # may be next time. else: - log.msg("Spider must return Request or ScrapedItem objects, got '%s' while processing %s" % (type(item).__name__, request), log.WARNING, domain=domain) + log.msg("Spider can return Request, ScrapedItem or None, got '%s' while processing %s" % (type(item).__name__, request), log.WARNING, domain=domain) class _ResultContainer(object): def append(self, item):