diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 348305fb7..229943c55 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -28,9 +28,10 @@ contains a dictionary of all available extensions and their order similar to how you :ref:`configure the downloader middlewares `. -.. class:: Crawler(settings) +.. class:: Crawler(spidercls, settings) The Crawler object must be instantiated with a + :class:`scrapy.spider.Spider` subclass and a :class:`scrapy.settings.Settings` object. .. attribute:: settings @@ -90,16 +91,18 @@ how you :ref:`configure the downloader middlewares or modify the downloader and scheduler behaviour, although this is an advanced use and this API is not yet stable. - .. method:: configure() + .. attribute:: spider - Configure the crawler. + Spider currently being crawled. This is an instance of the spider class + provided while constructing the crawler, and it is created after the + arguments given in the :meth:`crawl` method. - This loads extensions, middlewares and spiders, leaving the crawler - ready to be started. It also configures the execution engine. + .. method:: crawl(\*args, \**kwargs) - .. method:: start() + Starts the crawler by instantiating its spider class with the given + `args` and `kwargs` arguments, while setting the execution engine in + motion. - Start the crawler. This calls :meth:`configure` if it hasn't been called yet. Returns a deferred that is fired when the crawl is finished. .. _topics-api-settings: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index cfd6c8003..db1a083dd 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -13,16 +13,22 @@ from scrapy import log, signals class Crawler(object): - def __init__(self, settings): - self.configured = False + def __init__(self, spidercls, settings): + self.spidercls = spidercls self.settings = settings self.signals = SignalManager(self) - self.stats = load_object(settings['STATS_CLASS'])(self) - self._start_requests = lambda: () - self._spider = None - # TODO: move SpiderManager to CrawlerProcess + self.stats = load_object(self.settings['STATS_CLASS'])(self) + lf_cls = load_object(self.settings['LOG_FORMATTER']) + self.logformatter = lf_cls.from_crawler(self) + self.extensions = ExtensionManager.from_crawler(self) + + # Attribute kept for backward compatibility (Use CrawlerRunner.spiders) spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS']) - self.spiders = spman_cls.from_crawler(self) + self.spiders = spman_cls.from_settings(self.settings) + + self.crawling = False + self.spider = None + self.engine = None def install(self): # TODO: remove together with scrapy.project.crawler usage @@ -36,39 +42,31 @@ class Crawler(object): assert hasattr(scrapy.project, 'crawler'), "crawler not installed" del scrapy.project.crawler - def configure(self): - if self.configured: - return - - self.configured = True - lf_cls = load_object(self.settings['LOG_FORMATTER']) - self.logformatter = lf_cls.from_crawler(self) - self.extensions = ExtensionManager.from_crawler(self) - self.engine = ExecutionEngine(self, self._spider_closed) - - def crawl(self, spider, requests=None): - assert self._spider is None, 'Spider already attached' - self._spider = spider - spider.set_crawler(self) - if requests is None: - self._start_requests = spider.start_requests - else: - self._start_requests = lambda: requests - - def _spider_closed(self, spider=None): - if not self.engine.open_spiders: - self.stop() - @defer.inlineCallbacks - def start(self): - yield defer.maybeDeferred(self.configure) - if self._spider: - yield self.engine.open_spider(self._spider, self._start_requests()) - yield defer.maybeDeferred(self.engine.start) + def crawl(self, *args, **kwargs): + assert not self.crawling, "Crawling already taking place" + self.crawling = True + + try: + self.spider = self._create_spider(*args, **kwargs) + self.engine = self._create_engine() + start_requests = iter(self.spider.start_requests()) + yield self.engine.open_spider(self.spider, start_requests) + yield defer.maybeDeferred(self.engine.start) + except Exception: + self.crawling = False + raise + + def _create_spider(self, *args, **kwargs): + return self.spidercls.from_crawler(self, *args, **kwargs) + + def _create_engine(self): + return ExecutionEngine(self, lambda _: self.stop()) @defer.inlineCallbacks def stop(self): - if self.configured and self.engine.running: + if self.crawling: + self.crawling = False yield defer.maybeDeferred(self.engine.stop)