From be206ca5abf73b06a9639e34068f67c67c7f0d7b Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 31 Aug 2012 16:40:46 -0300 Subject: [PATCH] added process_start_requests method to spider middlewares --- docs/news.rst | 2 ++ docs/topics/spider-middleware.rst | 28 ++++++++++++++++++++++++++++ scrapy/core/engine.py | 5 +++-- scrapy/core/spidermw.py | 5 +++++ 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index e32e6bcb3..5ca1b71c1 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -6,6 +6,8 @@ Release notes Scrapy changes: +- added :meth:`~scrapy.contrib.spidermiddleware.SpiderMiddleware.process_start_requests` method to spider middlewares +- dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info. - dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info. - dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info. - documented :ref:`topics-api` diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 995d6689d..1c7a7dd6d 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -131,6 +131,34 @@ single Python class that defines one or more of the following methods: :param spider: the spider which raised the exception :type spider: :class:`scrapy.spider.BaseSpider` object + .. method:: process_start_requests(start_requests, spider) + + .. versionadded:: 0.15 + + This method is called with the start requests of the spider, and works + similarly to the :meth:`process_spider_output` method, except that it + doesn't have a response associated and must return only requests (not + items). + + It receives an iterable (in the ``start_requests`` parameter) and must + return another iterable of :class:`~scrapy.http.Request` objects. + + .. note:: When implementing this method in your spider middleware, you + should always return an iterable (that follows the input one) and + not consume all ``start_requests`` iterator because it can be very + large (or even unbounded) and cause a memory overflow. The Scrapy + engine is designed to pull start requests while it has capacity to + process them, so the start requests iterator can be effectively + endless where there is some other condition for stopping the spider + (like a time limit or item/page count). + + :param start_requests: the start requests + :type start_requests: an iterable of :class:`~scrapy.http.Request` + + :param spider: the spider to whom the start requests belong + :type spider: :class:`~scrapy.item.BaseSpider` object + + .. _Exception: http://docs.python.org/library/exceptions.html#exceptions.Exception diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index f0b4e8696..3bb6fe19a 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -213,13 +213,14 @@ class ExecutionEngine(object): return dwld @defer.inlineCallbacks - def open_spider(self, spider, start_requests=None, close_if_idle=True): + def open_spider(self, spider, start_requests=(), close_if_idle=True): assert self.has_capacity(), "No free spider slots when opening %r" % \ spider.name log.msg("Spider opened", spider=spider) nextcall = CallLaterOnce(self._next_request, spider) scheduler = self.scheduler_cls.from_crawler(self.crawler) - slot = Slot(start_requests or (), close_if_idle, nextcall, scheduler) + start_requests = yield self.scraper.spidermw.process_start_requests(start_requests, spider) + slot = Slot(start_requests, close_if_idle, nextcall, scheduler) self.slots[spider] = slot yield scheduler.open(spider) yield self.scraper.open_spider(spider) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 2cab31d9b..f6bb62afb 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -29,6 +29,8 @@ class SpiderMiddlewareManager(MiddlewareManager): self.methods['process_spider_output'].insert(0, mw.process_spider_output) if hasattr(mw, 'process_spider_exception'): self.methods['process_spider_exception'].insert(0, mw.process_spider_exception) + if hasattr(mw, 'process_start_requests'): + self.methods['process_start_requests'].insert(0, mw.process_start_requests) def scrape_response(self, scrape_func, response, request, spider): fname = lambda f:'%s.%s' % (f.im_self.__class__.__name__, f.im_func.__name__) @@ -68,3 +70,6 @@ class SpiderMiddlewareManager(MiddlewareManager): dfd.addErrback(process_spider_exception) dfd.addCallback(process_spider_output) return dfd + + def process_start_requests(self, start_requests, spider): + return self._process_chain('process_start_requests', start_requests, spider)