Prevent scrapy shell from starting spider requests (#7557)

* Prevent scrapy shell from starting spider requests

* Run pre-commit fixes

* Update shell architecture notes

---------

Co-authored-by: Mayuresh <mayuresh@Mayureshs-MacBook-Air.local>
This commit is contained in:
msn 2026-06-11 01:29:03 +05:30 committed by GitHub
parent cd25ece58a
commit 93a627ba1c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 8 deletions

View File

@ -28,11 +28,7 @@ from scrapy.spiders import Spider
from scrapy.utils.conf import get_config
from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.defer import (
_schedule_coro,
deferred_f_from_coro_f,
maybe_deferred_to_future,
)
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.misc import load_object
from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop
from scrapy.utils.response import open_in_browser
@ -69,8 +65,8 @@ if TYPE_CHECKING:
# 3. When fetch() is called, it prepares a request and calls Shell._schedule()
# in the reactor thread (via threads.blockingCallFromThread()).
# 4. Shell._schedule() calls Shell._open_spider() (on the first call).
# 5. Shell._open_spider() calls engine.open_spider_async(close_if_idle=False)
# and engine._start_request_processing().
# 5. Shell._open_spider() calls
# engine.open_spider_async(close_if_idle=False).
# 6. Shell._schedule() calls engine.crawl(request), scheduling the request.
# 7. Shell._schedule() via _request_deferred() waits until the request callback
# is called. When it's called, the response becomes available.
@ -214,7 +210,6 @@ class Shell:
self.crawler.spider = spider
assert self.crawler.engine
await self.crawler.engine.open_spider_async(close_if_idle=False)
_schedule_coro(self.crawler.engine._start_request_processing())
self.spider = spider
def fetch(