scrapy manager refactor

* ExecutionManager
    * deprecated runonce(*args)
    * changed start() to start(keep_alive=Bool)
    * changed crawl(*args) to crawl(requests, spider=None)
        * if no spider given, tries to resolve spider
          for each request
    * added crawl_url(url, spider=None)
    * added crawl_request(request, spider=None)
    * added crawl_domain(domain)
    * added crawl_spider(spider)
 * updated commands: crawl, runspider, start
 * updated webconsole
 * updated crawler
 * updated tests.test_engine
 * updated utils.fetch

scrapy/command/commands/crawl.py

@@ -1,6 +1,7 @@
 from scrapy.command import ScrapyCommand
 from scrapy.core.manager import scrapymanager
 from scrapy.conf import settings
+from scrapy.utils.url import is_url
 
 
 class Command(ScrapyCommand):
@@ -24,4 +25,12 @@ class Command(ScrapyCommand):
             settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False
 
     def run(self, args, opts):
-        scrapymanager.runonce(*args)
+        for arg in args:
+            # schedule arg as url or domain
+            if is_url(arg):
+                scrapymanager.crawl_url(arg)
+            else:
+                scrapymanager.crawl_domain(arg)
+
+        # crawl just scheduled arguments without keeping idle
+        scrapymanager.start()

scrapy/command/commands/runspider.py

@@ -52,6 +52,10 @@ class Command(ScrapyCommand):
             dispatcher.connect(exporter.export_item, signal=signals.item_passed)
             exporter.start_exporting()
         module = _import_file(args[0])
-        scrapymanager.runonce(module.SPIDER)
+
+        # schedule spider and start engine
+        scrapymanager.crawl_spider(module.SPIDER)
+        scrapymanager.start()
+
         if opts.output:
             exporter.finish_exporting()

scrapy/command/commands/start.py

@@ -9,4 +9,4 @@ class Command(ScrapyCommand):
         return "Start the Scrapy manager but don't run any spider (idle mode)"
 
     def run(self, args, opts):
-        scrapymanager.start(*args)
+        scrapymanager.start(keep_alive=True)

scrapy/contrib/webconsole/spiderctl.py

@@ -135,14 +135,14 @@ class Spiderctl(object):
         if "add_pending_domains" in args:
             for domain in args["add_pending_domains"]:
                 if domain not in scrapyengine.scheduler.pending_requests:
-                    scrapymanager.crawl(domain)
+                    scrapymanager.crawl_domain(domain)
             s += "<p>"
             s += "Scheduled spiders: <ul><li>%s</li></ul>" % "</li><li>".join(args["add_pending_domains"])
             s += "</p>"
         if "rerun_finished_domains" in args:
             for domain in args["rerun_finished_domains"]:
                 if domain not in scrapyengine.scheduler.pending_requests:
-                    scrapymanager.crawl(domain)
+                    scrapymanager.crawl_domain(domain)
                 self.finished.remove(domain)
             s += "<p>"
             s += "Re-scheduled finished spiders: <ul><li>%s</li></ul>" % "</li><li>".join(args["rerun_finished_domains"])

scrapy/core/manager.py

@@ -1,5 +1,4 @@
 import signal
-from collections import defaultdict
 
 from twisted.internet import reactor
 
@@ -12,40 +11,6 @@ from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.url import is_url
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
 
-def _get_spider_requests(*args):
-    """Collect requests and spiders from the given arguments. Returns a dict of
-    spider -> list of requests
-    """
-    spider_requests = defaultdict(list)
-    for arg in args:
-        if isinstance(arg, tuple):
-            request, spider = arg
-            spider_requests[spider] = request
-        elif isinstance(arg, Request):
-            spider = spiders.fromurl(arg.url) or BaseSpider('default')
-            if spider:
-                spider_requests[spider] += [arg]
-            else:
-                log.msg('Could not find spider for request: %s' % arg, log.ERROR)
-        elif isinstance(arg, BaseSpider):
-            spider_requests[arg] += arg.start_requests()
-        elif is_url(arg):
-            spider = spiders.fromurl(arg) or BaseSpider('default')
-            if spider:
-                for req in arg_to_iter(spider.make_requests_from_url(arg)):
-                    spider_requests[spider] += [req]
-            else:
-                log.msg('Could not find spider for url: %s' % arg, log.ERROR)
-        elif isinstance(arg, basestring):
-            spider = spiders.fromdomain(arg)
-            if spider:
-                spider_requests[spider] += spider.start_requests()
-            else:
-                log.msg('Could not find spider for domain: %s' % arg, log.ERROR)
-        else:
-            raise TypeError("Unsupported argument: %r" % arg)
-    return spider_requests
-
 
 class ExecutionManager(object):
     """Process a list of sites or urls.
@@ -78,24 +43,44 @@ class ExecutionManager(object):
         scrapyengine.configure()
         self.configured = True
         
-    def crawl(self, *args):
-        """Schedule the given args for crawling. args is a list of urls or domains"""
+    def crawl_url(self, url, spider=None):
+        """Schedule given url for crawling."""
+        spider = spider or spiders.fromurl(url)
+        if spider:
+            requests = arg_to_iter(spider.make_requests_from_url(url))
+            self._crawl_requests(requests, spider)
+        else:
+            log.msg('Could not find spider for url: %s' % url, log.ERROR)
+
+    def crawl_request(self, request, spider=None):
+        """Schedule request for crawling."""
         assert self.configured, "Scrapy Manager not yet configured"
-        spider_requests = _get_spider_requests(*args)
-        for spider, requests in spider_requests.iteritems():
-            for request in requests:
-                scrapyengine.crawl(request, spider)
+        spider = spider or spiders.fromurl(request.url)
+        if spider:
+            scrapyengine.crawl(request, spider)
+        else:
+            log.msg('Could not find spider for request: %s' % url, log.ERROR)
 
-    def runonce(self, *args):
-        """Run the engine until it finishes scraping all domains and then exit"""
-        self.crawl(*args)
-        scrapyengine.start()
-        if self.control_reactor:
-            reactor.run(installSignalHandlers=False)
+    def crawl_domain(self, domain):
+        """Schedule given domain for crawling."""
+        spider = spiders.fromdomain(domain)
+        if spider:
+            self.crawl_spider(spider)
+        else:
+            log.msg('Could not find spider for domain: %s' % domain, log.ERROR)
 
-    def start(self):
+    def crawl_spider(self, spider):
+        """Schedule spider for crawling."""
+        requests = spider.start_requests()
+        self._crawl_requests(requests, spider)
+
+    def _crawl_requests(self, requests, spider):
+        for req in requests:
+            self.crawl_request(req, spider)
+
+    def start(self, keep_alive=False):
         """Start the scrapy server, without scheduling any domains"""
-        scrapyengine.keep_alive = True
+        scrapyengine.keep_alive = keep_alive
         scrapyengine.start()
         if self.control_reactor:
             reactor.run(installSignalHandlers=False)

scrapy/shell.py

@@ -104,7 +104,7 @@ class Shell(object):
         signal.signal(signal.SIGINT, signal.SIG_IGN)
 
         reactor.callInThread(self._console_thread, url)
-        scrapymanager.start()
+        scrapymanager.start(keep_alive=True)
 
     def inspect_response(self, response):
         print

scrapy/tests/test_engine.py

@@ -97,7 +97,8 @@ class CrawlingSession(object):
             dispatcher.connect(self.response_downloaded, signals.response_downloaded)
 
             scrapymanager.configure()
-            scrapymanager.runonce(self.spider)
+            scrapymanager.crawl_spider(self.spider)
+            scrapymanager.start()
             self.port.stopListening()
             self.wasrun = True
 

scrapy/utils/fetch.py

@@ -10,8 +10,11 @@ def fetch(urls):
     commands or standalone scripts.
     """
     responses = []
-    requests = [Request(url, callback=responses.append, dont_filter=True) \
-        for url in urls]
-    scrapymanager.runonce(*requests)
+    for url in urls:
+        req = Request(url, callback=responses.append, dont_filter=True)
+        # @@@ request will require a suitable spider.
+        #     If not will not be schedule
+        scrapymanager.crawl_request(req)
+    scrapymanager.start()
     return responses