allow to directly specify which domain corresponds to a given request

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40290

scrapy/trunk/scrapy/core/manager.py

@@ -36,15 +36,38 @@ class ExecutionManager(object):
         requests = self._parse_args(args)
         self.priorities = self.prioritizer_class(requests.keys())
 
+        #applies a time (in seconds) between succesive requests crawl
+        #useful for applications that runs softly distributed in time
+        self.crawl_delay = opts.get("crawl_delay", 0)
+        
+        
     def crawl(self, *args):
         """Schedule the given args for crawling. args is a list of urls or domains"""
 
         requests = self._parse_args(args)
         # schedule initial requets to be scraped at engine start
-        for domain in requests or ():
-            spider = spiders.fromdomain(domain)
-            priority = self.priorities.get_priority(domain)
-            for request in requests[domain]:
+        
+        def _issue():
+            for domain in requests or ():
+                spider = spiders.fromdomain(domain)
+                priority = self.priorities.get_priority(domain)
+                for request in requests[domain]:
+                    yield request, spider, priority
+
+        if self.crawl_delay:
+            gen = _issue()
+            def _soft_crawl():
+                try:
+                    request, spider, priority = gen.next()
+                    scrapyengine.crawl(request, spider, domain_priority=priority)
+                    log.msg("Delaying %ss the next request." % self.crawl_delay)
+                except StopIteration:
+                    self.stop()
+
+            scrapyengine.addtask(_soft_crawl, self.crawl_delay)
+
+        else:
+            for request, spider, priority in _issue():
                 scrapyengine.crawl(request, spider, domain_priority=priority)
 
     def runonce(self, *args, **opts):
@@ -94,7 +117,7 @@ class ExecutionManager(object):
             signal.signal(signal.SIGBREAK, sig_handler_terminate)
 
     def _parse_args(self, args):
-        """ Parse crawl arguments and return a tuple of (domains, urls) """
+        """ Parse crawl arguments and return a dict domains -> [requests] """
         if not args:
             args = [p.domain_name for p in spiders.enabled]
 
@@ -133,7 +156,10 @@ class ExecutionManager(object):
 
         # requests
         for request in requests:
-            spider = spiders.fromurl(request.url)
+            if request.domain:
+                spider = spiders.fromdomain(request.domain)
+            else:
+                spider = spiders.fromurl(request.url)
             if not spider:
                 log.msg('Could not found spider for %s' % request, log.ERROR)
                 continue
@@ -142,5 +168,5 @@ class ExecutionManager(object):
 
     def _start_urls(self, spider):
         return spider.start_urls if hasattr(spider.start_urls, '__iter__') else [spider.start_urls]
-
+        
 scrapymanager = ExecutionManager()

scrapy/trunk/scrapy/http/request.py

@@ -15,7 +15,7 @@ from scrapy.utils.defer import chain_deferred
 class Request(object):
     def __init__(self, url, callback=None, context=None, method=None, body=None, headers=None, cookies=None,
             referer=None, url_encoding='utf-8', link_text='', http_user='', http_pass='', dont_filter=None, 
-            fingerprint_params=None):
+            fingerprint_params=None, domain=None):
 
         self.encoding = url_encoding  # this one has to be set first
         self.set_url(url)
@@ -55,7 +55,9 @@ class Request(object):
             self.httpauth(http_user, http_pass)
         self.depth = 0
         self.link_text = link_text
-
+        #allows to directly specify the spider for the request
+        self.domain = domain
+        
     def append_callback(self, callback, *args, **kwargs):
         if isinstance(callback, defer.Deferred):
             return chain_deferred(self.deferred, callback)