added process_start_requests method to spider middlewares

docs/news.rst

@@ -6,6 +6,8 @@ Release notes
 
 Scrapy changes:
 
+- added :meth:`~scrapy.contrib.spidermiddleware.SpiderMiddleware.process_start_requests` method to spider middlewares
+- dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info.
 - dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info.
 - dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info.
 - documented :ref:`topics-api`

docs/topics/spider-middleware.rst

@@ -131,6 +131,34 @@ single Python class that defines one or more of the following methods:
         :param spider: the spider which raised the exception
         :type spider: :class:`scrapy.spider.BaseSpider` object
 
+    .. method:: process_start_requests(start_requests, spider)
+
+        .. versionadded:: 0.15
+
+        This method is called with the start requests of the spider, and works
+        similarly to the :meth:`process_spider_output` method, except that it
+        doesn't have a response associated and must return only requests (not
+        items).
+
+        It receives an iterable (in the ``start_requests`` parameter) and must
+        return another iterable of :class:`~scrapy.http.Request` objects.
+
+        .. note:: When implementing this method in your spider middleware, you
+           should always return an iterable (that follows the input one) and
+           not consume all ``start_requests`` iterator because it can be very
+           large (or even unbounded) and cause a memory overflow. The Scrapy
+           engine is designed to pull start requests while it has capacity to
+           process them, so the start requests iterator can be effectively
+           endless where there is some other condition for stopping the spider
+           (like a time limit or item/page count).
+
+        :param start_requests: the start requests
+        :type start_requests: an iterable of :class:`~scrapy.http.Request`
+
+        :param spider: the spider to whom the start requests belong
+        :type spider: :class:`~scrapy.item.BaseSpider` object
+
+
 .. _Exception: http://docs.python.org/library/exceptions.html#exceptions.Exception
 
 

scrapy/core/engine.py

@@ -213,13 +213,14 @@ class ExecutionEngine(object):
         return dwld
 
     @defer.inlineCallbacks
-    def open_spider(self, spider, start_requests=None, close_if_idle=True):
+    def open_spider(self, spider, start_requests=(), close_if_idle=True):
         assert self.has_capacity(), "No free spider slots when opening %r" % \
             spider.name
         log.msg("Spider opened", spider=spider)
         nextcall = CallLaterOnce(self._next_request, spider)
         scheduler = self.scheduler_cls.from_crawler(self.crawler)
-        slot = Slot(start_requests or (), close_if_idle, nextcall, scheduler)
+        start_requests = yield self.scraper.spidermw.process_start_requests(start_requests, spider)
+        slot = Slot(start_requests, close_if_idle, nextcall, scheduler)
         self.slots[spider] = slot
         yield scheduler.open(spider)
         yield self.scraper.open_spider(spider)

scrapy/core/spidermw.py

@@ -29,6 +29,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
             self.methods['process_spider_output'].insert(0, mw.process_spider_output)
         if hasattr(mw, 'process_spider_exception'):
             self.methods['process_spider_exception'].insert(0, mw.process_spider_exception)
+        if hasattr(mw, 'process_start_requests'):
+            self.methods['process_start_requests'].insert(0, mw.process_start_requests)
 
     def scrape_response(self, scrape_func, response, request, spider):
         fname = lambda f:'%s.%s' % (f.im_self.__class__.__name__, f.im_func.__name__)
@@ -68,3 +70,6 @@ class SpiderMiddlewareManager(MiddlewareManager):
         dfd.addErrback(process_spider_exception)
         dfd.addCallback(process_spider_output)
         return dfd
+
+    def process_start_requests(self, start_requests, spider):
+        return self._process_chain('process_start_requests', start_requests, spider)