From bc705843444472dbcb8eb583b8d87e055a1b9798 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Fri, 10 Apr 2015 11:52:57 -0300 Subject: [PATCH 1/3] Support crawlers as first positional arg in CrawlerRunner.crawl() --- scrapy/crawler.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 692a896be..615303528 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -82,9 +82,12 @@ class CrawlerRunner(object): self.crawlers = set() self._active = set() - def crawl(self, spidercls, *args, **kwargs): - crawler = self._create_crawler(spidercls) - self._setup_crawler_logging(crawler) + def crawl(self, crawler_or_spidercls, *args, **kwargs): + crawler = crawler_or_spidercls + if not isinstance(crawler_or_spidercls, Crawler): + crawler = self._create_crawler(crawler_or_spidercls) + self._setup_crawler_logging(crawler) + self.crawlers.add(crawler) d = crawler.crawl(*args, **kwargs) self._active.add(d) From 86b095132770f8d1ba84f6502a5929c6ae102e04 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Fri, 10 Apr 2015 12:06:27 -0300 Subject: [PATCH 2/3] Delete `crawler_deferreds` doc in CrawlerRunner This attribute is now an internal one since it's main use-case was covered by CrawlerRunner.stop(). --- docs/topics/api.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 57b8ee0cf..70b75a868 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -116,11 +116,6 @@ how you :ref:`configure the downloader middlewares Set of :class:`crawlers ` created by the :meth:`crawl` method. - .. attribute:: crawl_deferreds - - Set of the `deferreds`_ return by the :meth:`crawl` method. This - collection it's useful for keeping track of current crawling state. - .. method:: crawl(spidercls, \*args, \**kwargs) This method sets up the crawling of the given `spidercls` with the From 3dabde6706b6c2f1015167f959fefec836bab9b1 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Fri, 10 Apr 2015 12:09:07 -0300 Subject: [PATCH 3/3] Update docs for CrawlerRunner.crawl() new usage --- docs/topics/api.rst | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 70b75a868..2055682dc 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -101,7 +101,7 @@ how you :ref:`configure the downloader middlewares .. class:: CrawlerRunner(settings) - This is a convenient helper class that creates, configures and runs + This is a convenient helper class that keeps track of, manages and runs crawlers inside an already setup Twisted `reactor`_. The CrawlerRunner object must be instantiated with a @@ -116,18 +116,23 @@ how you :ref:`configure the downloader middlewares Set of :class:`crawlers ` created by the :meth:`crawl` method. - .. method:: crawl(spidercls, \*args, \**kwargs) + .. method:: crawl(crawler_or_spidercls, \*args, \**kwargs) - This method sets up the crawling of the given `spidercls` with the - provided arguments. + This method runs a crawler with the provided arguments. - It takes care of loading the spider class while configuring and starting - a crawler for it. + It will keep track of the given crawler so it can be stopped later, + while calling its :meth:`Crawler.crawl` method. + + If `crawler_or_spidercls` isn't a :class:`~scrapy.crawler.Crawler` + instance, it will try to create one using this parameter as the spider + class given to it. Returns a deferred that is fired when the crawl is finished. - :param spidercls: spider class or spider's name inside the project - :type spidercls: :class:`~scrapy.spider.Spider` subclass or str + :param crawler_or_spidercls: already created crawler, or a spider class + or spider's name inside the project to create it + :type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance, + :class:`~scrapy.spider.Spider` subclass or string :param args: arguments to initializate the spider :type args: list