RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Install the reactor explicitly in CrawlerRunner examples. (#6865)

This commit is contained in:
Andrey Rakhmatullin 2025-06-06 14:54:21 +05:00 committed by GitHub
parent b20995c9d8
commit d99234a33f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 12 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
docs/topics/practices.rst
View File

@ -95,6 +95,7 @@ reactor after ``MySpider`` has finished running.
import scrapy
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from scrapy.utils.reactor import install_reactor
class MySpider(scrapy.Spider):
@ -102,6 +103,7 @@ reactor after ``MySpider`` has finished running.
...
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"})
runner = CrawlerRunner()
@ -112,26 +114,26 @@ reactor after ``MySpider`` has finished running.
d.addBoth(lambda _: reactor.stop())
reactor.run() # the script will block here until the crawling is finished
Same example but using a non-default reactor, it's only necessary call
``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically.
Same example but using a different reactor.
.. code-block:: python
import scrapy
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from scrapy.utils.reactor import install_reactor
class MySpider(scrapy.Spider):
custom_settings = {
"TWISTED_REACTOR": "twisted.internet.epollreactor.EPollReactor",
}
# Your spider definition
...
install_reactor("twisted.internet.epollreactor.EPollReactor")
configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"})
from scrapy.utils.reactor import install_reactor
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
runner = CrawlerRunner()
d = runner.crawl(MySpider)
@ -184,6 +186,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from scrapy.utils.project import get_project_settings
from scrapy.utils.reactor import install_reactor
class MySpider1(scrapy.Spider):
@ -196,6 +199,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
...
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
configure_logging()
settings = get_project_settings()
runner = CrawlerRunner(settings)
@ -217,6 +221,7 @@ Same example but running the spiders sequentially by chaining the deferreds:
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from scrapy.utils.project import get_project_settings
from scrapy.utils.reactor import install_reactor
class MySpider1(scrapy.Spider):
@ -229,6 +234,7 @@ Same example but running the spiders sequentially by chaining the deferreds:
...
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
settings = get_project_settings()
configure_logging(settings)
runner = CrawlerRunner(settings)