RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5352 from wRAR/crawler-process-reactor-later

This commit is contained in:
Andrey Rahmatullin 2022-01-06 11:31:59 +05:00 committed by GitHub
parent c5ab58056c 3577c00569
commit b04cfa4832
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 151 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
docs/news.rst
View File

@ -1,3 +1,25 @@
.. note::
.. versionchanged:: VERSION
The Twisted reactor is now installed when
:meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a
:class:`scrapy.crawler.CrawlerProcess` object is created. Because of this,
:setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now
honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions
they are silently ignored when set there and you need to set these settings
in some other way.
.. note::
.. versionchanged:: VERSION
Previously this setting had no effect in a spider
:attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but
if you :ref:`run several spiders in one process <run-multiple-spiders>`,
they must not have different values for this setting, because they will use
a single reactor instance.
.. _news:
Release notes

19
docs/topics/practices.rst
View File

@ -193,6 +193,25 @@ Same example but running the spiders sequentially by chaining the deferreds:
crawl()
reactor.run() # the script will block here until the last crawl call is finished
Different spiders can set different values for the same setting, but when they
run in the same process it may be impossible, by design or because of some
limitations, to use these different values. What happens in practice is
different for different settings:
* :setting:`SPIDER_LOADER_CLASS` and the ones used by its value
(:setting:`SPIDER_MODULES`, :setting:`SPIDER_LOADER_WARN_ONLY` for the
default one) cannot be read from the per-spider settings. These are applied
when the :class:`~scrapy.crawler.CrawlerRunner` or
:class:`~scrapy.crawler.CrawlerProcess` object is created.
* For :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` the first
available value is used, and if a spider requests a different reactor an
exception will be raised. These are applied when the reactor is installed.
* For :setting:`REACTOR_THREADPOOL_MAXSIZE`, :setting:`DNS_RESOLVER` and the
ones used by the resolver (:setting:`DNSCACHE_ENABLED`,
:setting:`DNSCACHE_SIZE`, :setting:`DNS_TIMEOUT` for ones included in Scrapy)
the first available value is used. These are applied when the reactor is
started.
.. seealso:: :ref:`run-from-script`.
.. _distributed-crawls:

7
docs/topics/settings.rst
View File

@ -1638,10 +1638,9 @@ which raises :exc:`Exception`, becomes::
The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
means that Scrapy will not attempt to install any specific reactor, and the
default reactor defined by Twisted for the current platform will be used. This
is to maintain backward compatibility and avoid possible problems caused by
using a non-default reactor.
means that Scrapy will install the default reactor defined by Twisted for the
current platform. This is to maintain backward compatibility and avoid possible
problems caused by using a non-default reactor.
For additional information, see :doc:`core/howto/choosing-reactor`.

2
scrapy/commands/shell.py
View File

@ -75,6 +75,6 @@ class Command(ScrapyCommand):
def _start_crawler_thread(self):
t = Thread(target=self.crawler_process.start,
kwargs={'stop_after_crawl': False})
kwargs={'stop_after_crawl': False, 'install_signal_handlers': False})
t.daemon = True
t.start()

40
scrapy/crawler.py
View File

@ -25,6 +25,7 @@ from scrapy.utils.log import (
configure_logging,
get_scrapy_root_handler,
install_scrapy_root_handler,
log_reactor_info,
log_scrapy_info,
LogCounterHandler,
)
@ -38,7 +39,7 @@ logger = logging.getLogger(__name__)
class Crawler:
def __init__(self, spidercls, settings=None):
def __init__(self, spidercls, settings=None, init_reactor: bool = False):
if isinstance(spidercls, Spider):
raise ValueError('The spidercls argument must be a class, not an object')
@ -69,6 +70,20 @@ class Crawler:
lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self)
reactor_class = self.settings.get("TWISTED_REACTOR")
if init_reactor:
# this needs to be done after the spider settings are merged,
# but before something imports twisted.internet.reactor
if reactor_class:
install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"])
else:
from twisted.internet import default
default.install()
log_reactor_info()
if reactor_class:
verify_installed_reactor(reactor_class)
self.extensions = ExtensionManager.from_crawler(self)
self.settings.freeze()
@ -153,7 +168,6 @@ class CrawlerRunner:
self._crawlers = set()
self._active = set()
self.bootstrap_failed = False
self._handle_twisted_reactor()
@property
def spiders(self):
@ -247,10 +261,6 @@ class CrawlerRunner:
while self._active:
yield defer.DeferredList(self._active)
def _handle_twisted_reactor(self):
if self.settings.get("TWISTED_REACTOR"):
verify_installed_reactor(self.settings["TWISTED_REACTOR"])
class CrawlerProcess(CrawlerRunner):
"""
@ -278,7 +288,6 @@ class CrawlerProcess(CrawlerRunner):
def __init__(self, settings=None, install_root_handler=True):
super().__init__(settings)
install_shutdown_handlers(self._signal_shutdown)
configure_logging(self.settings, install_root_handler)
log_scrapy_info(self.settings)
@ -298,7 +307,12 @@ class CrawlerProcess(CrawlerRunner):
{'signame': signame})
reactor.callFromThread(self._stop_reactor)
def start(self, stop_after_crawl=True):
def _create_crawler(self, spidercls):
if isinstance(spidercls, str):
spidercls = self.spider_loader.load(spidercls)
return Crawler(spidercls, self.settings, init_reactor=True)
def start(self, stop_after_crawl=True, install_signal_handlers=True):
"""
This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache
@ -309,6 +323,9 @@ class CrawlerProcess(CrawlerRunner):
:param bool stop_after_crawl: stop or not the reactor when all
crawlers have finished
:param bool install_signal_handlers: whether to install the shutdown
handlers (default: True)
"""
from twisted.internet import reactor
if stop_after_crawl:
@ -318,6 +335,8 @@ class CrawlerProcess(CrawlerRunner):
return
d.addBoth(self._stop_reactor)
if install_signal_handlers:
install_shutdown_handlers(self._signal_shutdown)
resolver_class = load_object(self.settings["DNS_RESOLVER"])
resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
resolver.install_on_reactor()
@ -337,8 +356,3 @@ class CrawlerProcess(CrawlerRunner):
reactor.stop()
except RuntimeError: # raised if already stopped or in shutdown stage
pass
def _handle_twisted_reactor(self):
if self.settings.get("TWISTED_REACTOR"):
install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"])
super()._handle_twisted_reactor()

5
scrapy/utils/log.py
View File

@ -143,7 +143,7 @@ def _get_handler(settings):
return handler
def log_scrapy_info(settings):
def log_scrapy_info(settings: Settings) -> None:
logger.info("Scrapy %(version)s started (bot: %(bot)s)",
{'version': scrapy.__version__, 'bot': settings['BOT_NAME']})
versions = [
@ -152,6 +152,9 @@ def log_scrapy_info(settings):
if name != "Scrapy"
]
logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)})
def log_reactor_info() -> None:
from twisted.internet import reactor
logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
from twisted.internet import asyncioreactor

14
tests/CrawlerProcess/twisted_reactor_custom_settings.py Normal file
View File

@ -0,0 +1,14 @@
import scrapy
from scrapy.crawler import CrawlerProcess
class AsyncioReactorSpider(scrapy.Spider):
name = 'asyncio_reactor'
custom_settings = {
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
}
process = CrawlerProcess()
process.crawl(AsyncioReactorSpider)
process.start()

22
tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py Normal file
View File

@ -0,0 +1,22 @@
import scrapy
from scrapy.crawler import CrawlerProcess
class SelectReactorSpider(scrapy.Spider):
name = 'select_reactor'
custom_settings = {
"TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
}
class AsyncioReactorSpider(scrapy.Spider):
name = 'asyncio_reactor'
custom_settings = {
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
}
process = CrawlerProcess()
process.crawl(SelectReactorSpider)
process.crawl(AsyncioReactorSpider)
process.start()

21
tests/CrawlerProcess/twisted_reactor_custom_settings_same.py Normal file
View File

@ -0,0 +1,21 @@
import scrapy
from scrapy.crawler import CrawlerProcess
class AsyncioReactorSpider1(scrapy.Spider):
name = 'asyncio_reactor1'
custom_settings = {
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
}
class AsyncioReactorSpider2(scrapy.Spider):
name = 'asyncio_reactor2'
custom_settings = {
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
}
process = CrawlerProcess()
process.crawl(AsyncioReactorSpider1)
process.crawl(AsyncioReactorSpider2)
process.start()

9
tests/test_commands.py
View File

@ -762,6 +762,7 @@ class MySpider(scrapy.Spider):
self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
spider_filename = 'myspider.pyw'
@ -774,35 +775,27 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
self.assertIn("start_requests", log)
self.assertIn("badspider.pyw", log)
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
def test_run_good_spider(self):
super().test_run_good_spider()
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
def test_runspider(self):
super().test_runspider()
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
def test_runspider_dnscache_disabled(self):
super().test_runspider_dnscache_disabled()
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
def test_runspider_log_level(self):
super().test_runspider_log_level()
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
def test_runspider_log_short_names(self):
super().test_runspider_log_short_names()
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
def test_runspider_no_spider_found(self):
super().test_runspider_no_spider_found()
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
def test_output(self):
super().test_output()
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
def test_overwrite_output(self):
super().test_overwrite_output()

41
tests/test_crawler.py
View File

@ -6,7 +6,6 @@ import sys
import warnings
from pytest import raises, mark
from testfixtures import LogCapture
from twisted import version as twisted_version
from twisted.internet import defer
from twisted.python.versions import Version
@ -270,6 +269,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
self.assertEqual(runner.bootstrap_failed, True)
@defer.inlineCallbacks
def test_crawler_runner_asyncio_enabled_true(self):
if self.reactor_pytest == 'asyncio':
CrawlerRunner(settings={
@ -278,32 +278,10 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
else:
msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
with self.assertRaisesRegex(Exception, msg):
CrawlerRunner(settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
})
@defer.inlineCallbacks
def test_crawler_process_asyncio_enabled_true(self):
with LogCapture(level=logging.DEBUG) as log:
if self.reactor_pytest == 'asyncio':
runner = CrawlerProcess(settings={
runner = CrawlerRunner(settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
})
yield runner.crawl(NoRequestsSpider)
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log))
else:
msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
with self.assertRaisesRegex(Exception, msg):
runner = CrawlerProcess(settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
})
@defer.inlineCallbacks
def test_crawler_process_asyncio_enabled_false(self):
runner = CrawlerProcess(settings={"TWISTED_REACTOR": None})
with LogCapture(level=logging.DEBUG) as log:
yield runner.crawl(NoRequestsSpider)
self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log))
class ScriptRunnerMixin:
@ -372,6 +350,21 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
self.assertIn("Spider closed (finished)", log)
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
def test_reactor_asyncio_custom_settings(self):
log = self.run_script("twisted_reactor_custom_settings.py")
self.assertIn("Spider closed (finished)", log)
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
def test_reactor_asyncio_custom_settings_same(self):
log = self.run_script("twisted_reactor_custom_settings_same.py")
self.assertIn("Spider closed (finished)", log)
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
def test_reactor_asyncio_custom_settings_conflict(self):
log = self.run_script("twisted_reactor_custom_settings_conflict.py")
self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
self.assertIn("(twisted.internet.selectreactor.SelectReactor) does not match the requested one", log)
@mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
@mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
@mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')