mirror of https://github.com/scrapy/scrapy.git
docs: Remove top-level reactor imports from CrawlerProces/CrawlerRunner examples
This commit is contained in:
parent
631fc65fad
commit
e676cd3ce0
|
|
@ -92,7 +92,6 @@ reactor after ``MySpider`` has finished running.
|
|||
|
||||
.. code-block:: python
|
||||
|
||||
from twisted.internet import reactor
|
||||
import scrapy
|
||||
from scrapy.crawler import CrawlerRunner
|
||||
from scrapy.utils.log import configure_logging
|
||||
|
|
@ -107,6 +106,33 @@ reactor after ``MySpider`` has finished running.
|
|||
runner = CrawlerRunner()
|
||||
|
||||
d = runner.crawl(MySpider)
|
||||
from twisted.internet import reactor
|
||||
|
||||
d.addBoth(lambda _: reactor.stop())
|
||||
reactor.run() # the script will block here until the crawling is finished
|
||||
|
||||
Same example but using a non-default reactor, is only necessary call ``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import scrapy
|
||||
from scrapy.crawler import CrawlerRunner
|
||||
from scrapy.utils.log import configure_logging
|
||||
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
# Your spider definition
|
||||
...
|
||||
|
||||
|
||||
configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"})
|
||||
from scrapy.utils.reactor import install_reactor
|
||||
|
||||
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
|
||||
runner = CrawlerRunner()
|
||||
d = runner.crawl(MySpider)
|
||||
from twisted.internet import reactor
|
||||
|
||||
d.addBoth(lambda _: reactor.stop())
|
||||
reactor.run() # the script will block here until the crawling is finished
|
||||
|
||||
|
|
@ -151,7 +177,6 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
|
|||
.. code-block:: python
|
||||
|
||||
import scrapy
|
||||
from twisted.internet import reactor
|
||||
from scrapy.crawler import CrawlerRunner
|
||||
from scrapy.utils.log import configure_logging
|
||||
from scrapy.utils.project import get_project_settings
|
||||
|
|
@ -173,6 +198,8 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
|
|||
runner.crawl(MySpider1)
|
||||
runner.crawl(MySpider2)
|
||||
d = runner.join()
|
||||
from twisted.internet import reactor
|
||||
|
||||
d.addBoth(lambda _: reactor.stop())
|
||||
|
||||
reactor.run() # the script will block here until all crawling jobs are finished
|
||||
|
|
@ -181,7 +208,7 @@ Same example but running the spiders sequentially by chaining the deferreds:
|
|||
|
||||
.. code-block:: python
|
||||
|
||||
from twisted.internet import reactor, defer
|
||||
from twisted.internet import defer
|
||||
from scrapy.crawler import CrawlerRunner
|
||||
from scrapy.utils.log import configure_logging
|
||||
from scrapy.utils.project import get_project_settings
|
||||
|
|
@ -209,6 +236,8 @@ Same example but running the spiders sequentially by chaining the deferreds:
|
|||
reactor.stop()
|
||||
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
crawl()
|
||||
reactor.run() # the script will block here until the last crawl call is finished
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue