Use autoclass for CrawlerRunner in docs/topics/api.rst

This commit is contained in:
Julia Medina 2015-04-24 18:26:49 -03:00
parent 353672d292
commit a2ce78db63
2 changed files with 45 additions and 46 deletions

View File

@ -99,52 +99,9 @@ how you :ref:`configure the downloader middlewares
Returns a deferred that is fired when the crawl is finished.
.. class:: CrawlerRunner(settings)
.. autoclass:: CrawlerRunner
:members:
This is a convenient helper class that keeps track of, manages and runs
crawlers inside an already setup Twisted `reactor`_.
The CrawlerRunner object must be instantiated with a
:class:`~scrapy.settings.Settings` object.
This class shouldn't be needed (since Scrapy is responsible of using it
accordingly) unless writing scripts that manually handle the crawling
process. See :ref:`run-from-script` for an example.
.. attribute:: crawlers
Set of :class:`crawlers <scrapy.crawler.Crawler>` created by the
:meth:`crawl` method.
.. method:: crawl(crawler_or_spidercls, \*args, \**kwargs)
This method runs a crawler with the provided arguments.
It will keep track of the given crawler so it can be stopped later,
while calling its :meth:`Crawler.crawl` method.
If `crawler_or_spidercls` isn't a :class:`~scrapy.crawler.Crawler`
instance, it will try to create one using this parameter as the spider
class given to it.
Returns a deferred that is fired when the crawl is finished.
:param crawler_or_spidercls: already created crawler, or a spider class
or spider's name inside the project to create it
:type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance,
:class:`~scrapy.spider.Spider` subclass or string
:param args: arguments to initializate the spider
:type args: list
:param kwargs: keyword arguments to initializate the spider
:type kwargs: dict
.. method:: stop()
Stops simultaneously all the crawling jobs taking place.
Returns a deferred that is fired when they all have ended.
.. _topics-api-settings:

View File

@ -89,6 +89,17 @@ class Crawler(object):
class CrawlerRunner(object):
"""
This is a convenient helper class that keeps track of, manages and runs
crawlers inside an already setup Twisted `reactor`_.
The CrawlerRunner object must be instantiated with a
:class:`~scrapy.settings.Settings` object.
This class shouldn't be needed (since Scrapy is responsible of using it
accordingly) unless writing scripts that manually handle the crawling
process. See :ref:`run-from-script` for an example.
"""
def __init__(self, settings):
if isinstance(settings, dict):
@ -106,6 +117,27 @@ class CrawlerRunner(object):
return self.spider_loader
def crawl(self, crawler_or_spidercls, *args, **kwargs):
"""
Run a crawler with the provided arguments.
It will call the given Crawler's :meth:`~Crawler.crawl` method, while
keeping track of it so it can be stopped later.
If `crawler_or_spidercls` isn't a :class:`~scrapy.crawler.Crawler`
instance, this method will try to create one using this parameter as
the spider class given to it.
Returns a deferred that is fired when the crawling is finished.
:param crawler_or_spidercls: already created crawler, or a spider class
or spider's name inside the project to create it
:type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance,
:class:`~scrapy.spider.Spider` subclass or string
:param list args: arguments to initialize the spider
:param dict kwargs: keyword arguments to initialize the spider
"""
crawler = crawler_or_spidercls
if not isinstance(crawler_or_spidercls, Crawler):
crawler = self._create_crawler(crawler_or_spidercls)
@ -127,11 +159,21 @@ class CrawlerRunner(object):
return Crawler(spidercls, self.settings)
def stop(self):
"""
Stops simultaneously all the crawling jobs taking place.
Returns a deferred that is fired when they all have ended.
"""
return defer.DeferredList([c.stop() for c in list(self.crawlers)])
@defer.inlineCallbacks
def join(self):
"""Wait for all managed crawlers to complete"""
"""
join()
Returns a deferred that is fired when all managed :attr:`crawlers` have
completed their executions.
"""
while self._active:
yield defer.DeferredList(self._active)