From 84f518fc5ebbf3052ac38b06cbd3f966e01c97d7 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 15 Jul 2011 15:18:39 -0300 Subject: [PATCH] More core changes: * removed execution queue (replaced by newer spider queues) * added real support for returning iterators in Spider.start_requests() * removed support for passing urls to 'scrapy crawl' command --- docs/topics/commands.rst | 10 +-- scrapy/commands/crawl.py | 61 ++----------- scrapy/commands/fetch.py | 14 ++- scrapy/commands/parse.py | 2 +- scrapy/commands/runspider.py | 4 +- scrapy/commands/shell.py | 2 +- scrapy/core/engine.py | 16 +++- scrapy/crawler.py | 37 ++------ scrapy/queue.py | 96 -------------------- scrapy/settings/default_settings.py | 4 - scrapy/shell.py | 2 +- scrapy/tests/test_commands.py | 3 - scrapy/tests/test_engine.py | 2 +- scrapy/tests/test_queue.py | 130 ---------------------------- 14 files changed, 37 insertions(+), 346 deletions(-) delete mode 100644 scrapy/queue.py delete mode 100644 scrapy/tests/test_queue.py diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 984894505..22d84d2d3 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -218,22 +218,16 @@ Usage example:: crawl ----- -* Syntax: ``scrapy crawl `` +* Syntax: ``scrapy crawl `` * Requires project: *yes* -Start crawling a spider. If a URL is passed instead of a spider, it will start -from that URL instead of the spider start urls. +Start crawling a spider. Usage examples:: - $ scrapy crawl example.com - [ ... example.com spider starts crawling ... ] - $ scrapy crawl myspider [ ... myspider starts crawling ... ] - $ scrapy crawl http://example.com/some/page.html - [ ... spider that handles example.com starts crawling from that url ... ] .. command:: server diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index c2e787a71..07409be1a 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,32 +1,21 @@ -from w3lib.url import is_url - -from scrapy import log from scrapy.command import ScrapyCommand -from scrapy.conf import settings -from scrapy.http import Request from scrapy.utils.conf import arglist_to_dict from scrapy.exceptions import UsageError -from collections import defaultdict - class Command(ScrapyCommand): requires_project = True def syntax(self): - return "[options] ..." + return "[options] " def short_desc(self): return "Start crawling from a spider or URL" def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--spider", dest="spider", default=None, \ - help="always use this spider when arguments are urls") parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", \ help="set spider argument (may be repeated)") - parser.add_option("-n", "--nofollow", dest="nofollow", action="store_true", \ - help="don't follow links (for use with URLs only)") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) @@ -34,49 +23,11 @@ class Command(ScrapyCommand): opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) - if opts.nofollow: - settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False def run(self, args, opts): - q = self.crawler.queue - urls, names = self._split_urls_and_names(args) - for name in names: - q.append_spider_name(name, **opts.spargs) - - if opts.spider: - try: - spider = self.crawler.spiders.create(opts.spider, **opts.spargs) - for url in urls: - q.append_url(url, spider) - except KeyError: - log.msg('Unable to find spider: %s' % opts.spider, log.ERROR) - else: - for name, urls in self._group_urls_by_spider(urls): - spider = self.crawler.spiders.create(name, **opts.spargs) - for url in urls: - q.append_url(url, spider) + if len(args) < 1: + raise UsageError() + for spname in args: + spider = self.crawler.spiders.create(spname, **opts.spargs) + self.crawler.crawl(spider) self.crawler.start() - - def _group_urls_by_spider(self, urls): - spider_urls = defaultdict(list) - for url in urls: - spider_names = self.crawler.spiders.find_by_request(Request(url)) - if not spider_names: - log.msg('Could not find spider that handles url: %s' % url, - log.ERROR) - elif len(spider_names) > 1: - log.msg('More than one spider can handle url: %s - %s' % \ - (url, ", ".join(spider_names)), log.ERROR) - else: - spider_urls[spider_names[0]].append(url) - return spider_urls.items() - - def _split_urls_and_names(self, args): - urls = [] - names = [] - for arg in args: - if is_url(arg): - urls.append(arg) - else: - names.append(arg) - return urls, names diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index d26d3bfd4..cc4d00f94 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,10 +1,10 @@ from w3lib.url import is_url -from scrapy import log from scrapy.command import ScrapyCommand from scrapy.http import Request from scrapy.spider import BaseSpider from scrapy.exceptions import UsageError +from scrapy.utils.spider import create_spider_for_request class Command(ScrapyCommand): @@ -49,12 +49,10 @@ class Command(ScrapyCommand): spider = None if opts.spider: - try: - spider = self.crawler.spiders.create(opts.spider) - except KeyError: - log.msg("Could not find spider: %s" % opts.spider, log.ERROR) - - self.crawler.queue.append_request(request, spider, \ - default_spider=BaseSpider('default')) + spider = self.crawler.spiders.create(opts.spider) + else: + spider = create_spider_for_request(self.crawler.spiders, request, \ + default_spider=BaseSpider('default')) + self.crawler.crawl(spider, [request]) self.crawler.start() diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 8ab3ee76f..3dbda5fe3 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -84,7 +84,7 @@ class Command(ScrapyCommand): spider = self.get_spider(request, opts) if not spider: return None, None - self.crawler.queue.append_request(request, spider) + self.crawler.crawl(spider, [request]) self.crawler.start() if not responses: log.msg('No response downloaded for: %s' % request, log.ERROR, \ diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index fe7142a57..183855029 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -60,6 +60,6 @@ class Command(ScrapyCommand): if not spclasses: raise UsageError("No spider found in file: %s\n" % filename) spider = spclasses.pop()(**opts.spargs) - # schedule spider and start engine - self.crawler.queue.append_spider(spider) + + self.crawler.crawl(spider) self.crawler.start() diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index ae4426d2f..6600dfc5d 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -11,7 +11,7 @@ from scrapy import log class Command(ScrapyCommand): requires_project = False - default_settings = {'KEEP_ALIVE': True} + default_settings = {'KEEP_ALIVE': True, 'LOGSTATS_INTERVAL': 0} def syntax(self): return "[url|file]" diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index df409f3ba..bd412bc89 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -21,9 +21,11 @@ from scrapy.utils.defer import mustbe_deferred class Slot(object): - def __init__(self): + def __init__(self, start_requests, close_if_idle): self.closing = False self.inprogress = set() # requests in progress + self.requests = iter(start_requests) + self.close_if_idle = close_if_idle def add_request(self, request): self.inprogress.add(request) @@ -107,7 +109,13 @@ class ExecutionEngine(object): break if self.spider_is_idle(spider): - self._spider_idle(spider) + slot = self.slots[spider] + try: + request = slot.requests.next() + self.crawl(request, spider) + except StopIteration: + if slot.close_if_idle: + self._spider_idle(spider) def _needs_backout(self, spider): slot = self.slots[spider] @@ -212,11 +220,11 @@ class ExecutionEngine(object): return dwld @defer.inlineCallbacks - def open_spider(self, spider): + def open_spider(self, spider, start_requests=None, close_if_idle=True): assert self.has_capacity(), "No free spider slots when opening %r" % \ spider.name log.msg("Spider opened", spider=spider) - self.slots[spider] = Slot() + self.slots[spider] = Slot(start_requests or (), close_if_idle) yield self.scheduler.open_spider(spider) self.downloader.open_spider(spider) yield self.scraper.open_spider(spider) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 22d0f2f91..4c81fa417 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -3,7 +3,6 @@ import signal from twisted.internet import reactor, defer from scrapy.xlib.pydispatch import dispatcher -from scrapy.queue import ExecutionQueue from scrapy.core.engine import ExecutionEngine from scrapy.extension import ExtensionManager from scrapy.utils.ossignal import install_shutdown_handlers, signal_names @@ -34,51 +33,25 @@ class Crawler(object): self.extensions = ExtensionManager.from_settings(self.settings) spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS']) self.spiders = spman_cls.from_settings(self.settings) - spq_cls = load_object(self.settings['SPIDER_QUEUE_CLASS']) - spq = spq_cls.from_settings(self.settings) - keepalive = self.settings.getbool('KEEP_ALIVE') - pollint = self.settings.getfloat('QUEUE_POLL_INTERVAL') - self.queue = ExecutionQueue(self.spiders, spq, poll_interval=pollint, - keep_alive=keepalive) self.engine = ExecutionEngine(self.settings, self._spider_closed) - @defer.inlineCallbacks - def _start_next_spider(self): - spider, requests = yield defer.maybeDeferred(self.queue.get_next) - if spider: - self._start_spider(spider, requests) - if self.engine.has_capacity() and not self._nextcall.active(): - self._nextcall = reactor.callLater(self.queue.poll_interval, \ - self._spider_closed) - - @defer.inlineCallbacks - def _start_spider(self, spider, requests): - """Don't call this method. Use self.queue to start new spiders""" + def crawl(self, spider, requests=None): spider.set_crawler(self) - yield defer.maybeDeferred(self.engine.open_spider, spider) - for request in requests: - self.engine.crawl(request, spider) + if requests is None: + requests = spider.start_requests() + return self.engine.open_spider(spider, requests) - @defer.inlineCallbacks def _spider_closed(self, spider=None): if not self.engine.open_spiders: - is_finished = yield defer.maybeDeferred(self.queue.is_finished) - if is_finished: - self.stop() - return - if self.engine.has_capacity(): - self._start_next_spider() + self.stop() @defer.inlineCallbacks def start(self): yield defer.maybeDeferred(self.configure) yield defer.maybeDeferred(self.engine.start) - self._nextcall = reactor.callLater(0, self._start_next_spider) @defer.inlineCallbacks def stop(self): - if self._nextcall.active(): - self._nextcall.cancel() if self.engine.running: yield defer.maybeDeferred(self.engine.stop) diff --git a/scrapy/queue.py b/scrapy/queue.py deleted file mode 100644 index 2f2a17de5..000000000 --- a/scrapy/queue.py +++ /dev/null @@ -1,96 +0,0 @@ -from twisted.internet import defer - -from scrapy.http import Request -from scrapy.utils.misc import arg_to_iter -from scrapy.utils.spider import create_spider_for_request -from scrapy.utils.python import stringify_dict -from scrapy import log - - -class ExecutionQueue(object): - - def __init__(self, spiders, queue, poll_interval=5, keep_alive=False): - self.spider_requests = [] - self.poll_interval = poll_interval - self._spiders = spiders - self._queue = queue - self._keepalive = keep_alive - - @defer.inlineCallbacks - def _append_next(self): - """Called when there are no more items left in self.spider_requests. - This method is meant to be overriden in subclasses to add new (spider, - requests) tuples to self.spider_requests. It can return a Deferred. - """ - msg = yield self._queue.pop() - if msg: - name = msg.pop('name') - msg = stringify_dict(msg) # see #250 - self.append_spider_name(name, **msg) - - def get_next(self): - """Return a tuple (spider, requests) containing a list of Requests and - the Spider which will be used to crawl those Requests. If there aren't - any more spiders to crawl it must return (None, []). - - This method can return a deferred. - """ - if self.spider_requests: - return self._get_next_now() - d = defer.maybeDeferred(self._append_next) - d.addCallback(lambda _: self._get_next_now()) - return d - - def _get_next_now(self): - try: - return self.spider_requests.pop(0) - except IndexError: - return (None, []) - - def is_finished(self): - """Return True if the queue is empty and there won't be any more - spiders to crawl (this is for one-shot runs). If it returns ``False`` - Scrapy will keep polling this queue for new requests to scrape - """ - return not self._keepalive and not bool(self.spider_requests) - - def append_spider(self, spider): - """Append a Spider to crawl""" - requests = spider.start_requests() - self.spider_requests.append((spider, requests)) - - def append_request(self, request, spider=None, **kwargs): - if spider is None: - spider = create_spider_for_request(self._spiders, request, **kwargs) - if spider: - self.spider_requests.append((spider, [request])) - - def append_url(self, url=None, spider=None, **kwargs): - """Append a URL to crawl with the given spider. If the spider is not - given, a spider will be looked up based on the URL - """ - if url is None: - raise ValueError("A url is required") - if spider is None: - spider = create_spider_for_request(self._spiders, Request(url), \ - **kwargs) - if spider: - requests = arg_to_iter(spider.make_requests_from_url(url)) - self.spider_requests.append((spider, requests)) - - def append_spider_name(self, name=None, **spider_kwargs): - """Append a spider to crawl given its name and optional arguments, - which are used to instantiate it. The SpiderManager is used to lookup - the spider - """ - if name is None: - raise ValueError("A spider name is required") - try: - spider = self._spiders.create(name, **spider_kwargs) - except KeyError: - log.msg('Unable to find spider: %s' % name, log.ERROR) - except: - log.err(None, 'Error creating spider %r with arguments: %r' % \ - (name, spider_kwargs)) - else: - self.append_spider(spider) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a99d6360d..7b14f3fe1 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -174,8 +174,6 @@ ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager' # Item pipelines are typically set in specific commands settings ITEM_PIPELINES = [] -KEEP_ALIVE = False - LOG_ENABLED = True LOG_ENCODING = 'utf-8' LOG_FORMATTER = 'scrapy.logformatter.LogFormatter' @@ -203,8 +201,6 @@ MEMUSAGE_WARNING_MB = 0 NEWSPIDER_MODULE = '' -QUEUE_POLL_INTERVAL = 5 - RANDOMIZE_DOWNLOAD_DELAY = True REDIRECT_ENABLED = True diff --git a/scrapy/shell.py b/scrapy/shell.py index acfe370ca..258c03ba6 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -61,7 +61,7 @@ class Shell(object): spider = create_spider_for_request(self.crawler.spiders, request, \ BaseSpider('default'), log_multiple=True) spider.set_crawler(self.crawler) - self.crawler.engine.open_spider(spider) + self.crawler.engine.open_spider(spider, close_if_idle=False) d = request_deferred(request) d.addCallback(lambda x: (x, spider)) self.crawler.engine.crawl(request, spider) diff --git a/scrapy/tests/test_commands.py b/scrapy/tests/test_commands.py index 170030693..374872e54 100644 --- a/scrapy/tests/test_commands.py +++ b/scrapy/tests/test_commands.py @@ -103,9 +103,6 @@ class GenspiderCommandTest(CommandTest): class MiscCommandsTest(CommandTest): - def test_crawl(self): - self.assertEqual(0, self.call('crawl')) - def test_list(self): self.assertEqual(0, self.call('list')) diff --git a/scrapy/tests/test_engine.py b/scrapy/tests/test_engine.py index 2664daf69..22ae44ead 100644 --- a/scrapy/tests/test_engine.py +++ b/scrapy/tests/test_engine.py @@ -97,7 +97,7 @@ class CrawlerRun(object): self.crawler = get_crawler() self.crawler.install() self.crawler.configure() - self.crawler.queue.append_spider(self.spider) + self.crawler.crawl(self.spider) self.crawler.start() self.deferred = defer.Deferred() diff --git a/scrapy/tests/test_queue.py b/scrapy/tests/test_queue.py deleted file mode 100644 index 077dd71f9..000000000 --- a/scrapy/tests/test_queue.py +++ /dev/null @@ -1,130 +0,0 @@ -import unittest - -from scrapy.queue import ExecutionQueue -from scrapy.spider import BaseSpider -from scrapy.http import Request - -class TestSpider(BaseSpider): - - name = "default" - - def start_requests(self): - return [Request("http://www.example.com/1"), \ - Request("http://www.example.com/2")] - - def make_requests_from_url(self, url): - return [Request(url + "/make1"), Request(url + "/make2")] - - -class TestSpiderManager(object): - - def find_by_request(self, request): - return ['create_for_request'] - - def create(self, spider_name, **spider_kwargs): - return TestSpider(spider_name, **spider_kwargs) - - -class ExecutionQueueTest(unittest.TestCase): - - keep_alive = False - - def setUp(self): - self.queue = ExecutionQueue(TestSpiderManager(), None, keep_alive=self.keep_alive) - self.spider = TestSpider() - self.request = Request('about:none') - - def tearDown(self): - del self.queue, self.spider, self.request - - def test_is_finished(self): - self.assert_(self.queue.is_finished()) - self.queue.append_request(self.request, self.spider) - self.assert_(not self.queue.is_finished()) - - def test_append_spider(self): - spider = TestSpider() - self.queue.append_spider(spider) - self.assert_(self.queue.spider_requests[0][0] is spider) - self._assert_request_urls(self.queue.spider_requests[0][1], - ["http://www.example.com/1", "http://www.example.com/2"]) - - def test_append_request1(self): - spider = TestSpider() - request = Request('about:blank') - self.queue.append_request(request, spider=spider) - self.assert_(self.queue.spider_requests[0][0] is spider) - self.assert_(self.queue.spider_requests[0][1][0] is request) - - def test_append_request2(self): - request = Request('about:blank') - self.queue.append_request(request, arg='123') - spider = self.queue.spider_requests[0][0] - self.assert_(spider.name == 'create_for_request') - self.assert_(spider.arg == '123') - - def test_append_url(self): - spider = TestSpider() - url = 'http://www.example.com/asd' - self.queue.append_url(url, spider=spider) - self.assert_(self.queue.spider_requests[0][0] is spider) - self._assert_request_urls(self.queue.spider_requests[0][1], \ - ['http://www.example.com/asd/make1', 'http://www.example.com/asd/make2']) - - def test_append_url_kwarg(self): - spider = TestSpider() - url = 'http://www.example.com/asd' - self.queue.append_url(url=url, spider=spider) - self.assert_(self.queue.spider_requests[0][0] is spider) - self._assert_request_urls(self.queue.spider_requests[0][1], \ - ['http://www.example.com/asd/make1', 'http://www.example.com/asd/make2']) - - def test_append_url2(self): - url = 'http://www.example.com/asd' - self.queue.append_url(url, arg='123') - self._assert_request_urls(self.queue.spider_requests[0][1], \ - ['http://www.example.com/asd/make1', 'http://www.example.com/asd/make2']) - spider = self.queue.spider_requests[0][0] - self.assert_(spider.name == 'create_for_request') - self.assert_(spider.arg == '123') - - def test_append_spider_name(self): - self.queue.append_spider_name('test123', arg='123') - spider = self.queue.spider_requests[0][0] - self.assert_(spider.name == 'test123') - self.assert_(spider.arg == '123') - - def test_append_spider_name_kwarg(self): - self.queue.append_spider_name(name='test123', arg='123') - spider = self.queue.spider_requests[0][0] - self.assert_(spider.name == 'test123') - self.assert_(spider.arg == '123') - - def test_append_next(self): - # the reason for this test: http://dev.scrapy.org/ticket/250 - class MockQueue(object): - def pop(self): - return {u'name': u'test123', u'test': u'hello'} - self.queue._queue = MockQueue() - self.queue._append_next() - spider = self.queue.spider_requests[0][0] - self.assert_(spider.name == 'test123') - self.assert_(spider.test == 'hello') - - def _assert_request_urls(self, requests, urls): - assert all(isinstance(x, Request) for x in requests) - self.assertEqual([x.url for x in requests], urls) - -class KeepAliveExecutionQueueTest(ExecutionQueueTest): - - keep_alive = True - - def test_is_finished(self): - self.assert_(not self.queue.is_finished()) - self.queue.append_request(self.request, self.spider) - self.assert_(not self.queue.is_finished()) - - -if __name__ == "__main__": - unittest.main() -