diff --git a/scrapy/command/commands/crawl.py b/scrapy/command/commands/crawl.py
index 3a04423fc..f53de139d 100644
--- a/scrapy/command/commands/crawl.py
+++ b/scrapy/command/commands/crawl.py
@@ -1,6 +1,7 @@
from scrapy.command import ScrapyCommand
from scrapy.core.manager import scrapymanager
from scrapy.conf import settings
+from scrapy.utils.url import is_url
class Command(ScrapyCommand):
@@ -24,4 +25,12 @@ class Command(ScrapyCommand):
settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False
def run(self, args, opts):
- scrapymanager.runonce(*args)
+ for arg in args:
+ # schedule arg as url or domain
+ if is_url(arg):
+ scrapymanager.crawl_url(arg)
+ else:
+ scrapymanager.crawl_domain(arg)
+
+ # crawl just scheduled arguments without keeping idle
+ scrapymanager.start()
diff --git a/scrapy/command/commands/runspider.py b/scrapy/command/commands/runspider.py
index e18f13cdd..2bfcd87f9 100644
--- a/scrapy/command/commands/runspider.py
+++ b/scrapy/command/commands/runspider.py
@@ -52,6 +52,10 @@ class Command(ScrapyCommand):
dispatcher.connect(exporter.export_item, signal=signals.item_passed)
exporter.start_exporting()
module = _import_file(args[0])
- scrapymanager.runonce(module.SPIDER)
+
+ # schedule spider and start engine
+ scrapymanager.crawl_spider(module.SPIDER)
+ scrapymanager.start()
+
if opts.output:
exporter.finish_exporting()
diff --git a/scrapy/command/commands/start.py b/scrapy/command/commands/start.py
index 032f12bcc..2c7304787 100644
--- a/scrapy/command/commands/start.py
+++ b/scrapy/command/commands/start.py
@@ -9,4 +9,4 @@ class Command(ScrapyCommand):
return "Start the Scrapy manager but don't run any spider (idle mode)"
def run(self, args, opts):
- scrapymanager.start(*args)
+ scrapymanager.start(keep_alive=True)
diff --git a/scrapy/contrib/webconsole/spiderctl.py b/scrapy/contrib/webconsole/spiderctl.py
index 9719d12f0..933801854 100644
--- a/scrapy/contrib/webconsole/spiderctl.py
+++ b/scrapy/contrib/webconsole/spiderctl.py
@@ -135,14 +135,14 @@ class Spiderctl(object):
if "add_pending_domains" in args:
for domain in args["add_pending_domains"]:
if domain not in scrapyengine.scheduler.pending_requests:
- scrapymanager.crawl(domain)
+ scrapymanager.crawl_domain(domain)
s += ""
s += "Scheduled spiders:
" % "".join(args["add_pending_domains"])
s += ""
if "rerun_finished_domains" in args:
for domain in args["rerun_finished_domains"]:
if domain not in scrapyengine.scheduler.pending_requests:
- scrapymanager.crawl(domain)
+ scrapymanager.crawl_domain(domain)
self.finished.remove(domain)
s += ""
s += "Re-scheduled finished spiders:
" % "".join(args["rerun_finished_domains"])
diff --git a/scrapy/core/manager.py b/scrapy/core/manager.py
index 2078aaa95..05586a117 100644
--- a/scrapy/core/manager.py
+++ b/scrapy/core/manager.py
@@ -1,5 +1,4 @@
import signal
-from collections import defaultdict
from twisted.internet import reactor
@@ -12,40 +11,6 @@ from scrapy.utils.misc import arg_to_iter
from scrapy.utils.url import is_url
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
-def _get_spider_requests(*args):
- """Collect requests and spiders from the given arguments. Returns a dict of
- spider -> list of requests
- """
- spider_requests = defaultdict(list)
- for arg in args:
- if isinstance(arg, tuple):
- request, spider = arg
- spider_requests[spider] = request
- elif isinstance(arg, Request):
- spider = spiders.fromurl(arg.url) or BaseSpider('default')
- if spider:
- spider_requests[spider] += [arg]
- else:
- log.msg('Could not find spider for request: %s' % arg, log.ERROR)
- elif isinstance(arg, BaseSpider):
- spider_requests[arg] += arg.start_requests()
- elif is_url(arg):
- spider = spiders.fromurl(arg) or BaseSpider('default')
- if spider:
- for req in arg_to_iter(spider.make_requests_from_url(arg)):
- spider_requests[spider] += [req]
- else:
- log.msg('Could not find spider for url: %s' % arg, log.ERROR)
- elif isinstance(arg, basestring):
- spider = spiders.fromdomain(arg)
- if spider:
- spider_requests[spider] += spider.start_requests()
- else:
- log.msg('Could not find spider for domain: %s' % arg, log.ERROR)
- else:
- raise TypeError("Unsupported argument: %r" % arg)
- return spider_requests
-
class ExecutionManager(object):
"""Process a list of sites or urls.
@@ -78,24 +43,44 @@ class ExecutionManager(object):
scrapyengine.configure()
self.configured = True
- def crawl(self, *args):
- """Schedule the given args for crawling. args is a list of urls or domains"""
+ def crawl_url(self, url, spider=None):
+ """Schedule given url for crawling."""
+ spider = spider or spiders.fromurl(url)
+ if spider:
+ requests = arg_to_iter(spider.make_requests_from_url(url))
+ self._crawl_requests(requests, spider)
+ else:
+ log.msg('Could not find spider for url: %s' % url, log.ERROR)
+
+ def crawl_request(self, request, spider=None):
+ """Schedule request for crawling."""
assert self.configured, "Scrapy Manager not yet configured"
- spider_requests = _get_spider_requests(*args)
- for spider, requests in spider_requests.iteritems():
- for request in requests:
- scrapyengine.crawl(request, spider)
+ spider = spider or spiders.fromurl(request.url)
+ if spider:
+ scrapyengine.crawl(request, spider)
+ else:
+ log.msg('Could not find spider for request: %s' % url, log.ERROR)
- def runonce(self, *args):
- """Run the engine until it finishes scraping all domains and then exit"""
- self.crawl(*args)
- scrapyengine.start()
- if self.control_reactor:
- reactor.run(installSignalHandlers=False)
+ def crawl_domain(self, domain):
+ """Schedule given domain for crawling."""
+ spider = spiders.fromdomain(domain)
+ if spider:
+ self.crawl_spider(spider)
+ else:
+ log.msg('Could not find spider for domain: %s' % domain, log.ERROR)
- def start(self):
+ def crawl_spider(self, spider):
+ """Schedule spider for crawling."""
+ requests = spider.start_requests()
+ self._crawl_requests(requests, spider)
+
+ def _crawl_requests(self, requests, spider):
+ for req in requests:
+ self.crawl_request(req, spider)
+
+ def start(self, keep_alive=False):
"""Start the scrapy server, without scheduling any domains"""
- scrapyengine.keep_alive = True
+ scrapyengine.keep_alive = keep_alive
scrapyengine.start()
if self.control_reactor:
reactor.run(installSignalHandlers=False)
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 8af6af006..82baf63eb 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -104,7 +104,7 @@ class Shell(object):
signal.signal(signal.SIGINT, signal.SIG_IGN)
reactor.callInThread(self._console_thread, url)
- scrapymanager.start()
+ scrapymanager.start(keep_alive=True)
def inspect_response(self, response):
print
diff --git a/scrapy/tests/test_engine.py b/scrapy/tests/test_engine.py
index f205ff264..873e0eb30 100644
--- a/scrapy/tests/test_engine.py
+++ b/scrapy/tests/test_engine.py
@@ -97,7 +97,8 @@ class CrawlingSession(object):
dispatcher.connect(self.response_downloaded, signals.response_downloaded)
scrapymanager.configure()
- scrapymanager.runonce(self.spider)
+ scrapymanager.crawl_spider(self.spider)
+ scrapymanager.start()
self.port.stopListening()
self.wasrun = True
diff --git a/scrapy/utils/fetch.py b/scrapy/utils/fetch.py
index b489c75bf..a1eed0fd8 100644
--- a/scrapy/utils/fetch.py
+++ b/scrapy/utils/fetch.py
@@ -10,8 +10,11 @@ def fetch(urls):
commands or standalone scripts.
"""
responses = []
- requests = [Request(url, callback=responses.append, dont_filter=True) \
- for url in urls]
- scrapymanager.runonce(*requests)
+ for url in urls:
+ req = Request(url, callback=responses.append, dont_filter=True)
+ # @@@ request will require a suitable spider.
+ # If not will not be schedule
+ scrapymanager.crawl_request(req)
+ scrapymanager.start()
return responses