RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove the deprecated spider parameter from the engine (#5998)

This commit is contained in:
Laerte Pereira 2023-08-04 02:53:04 -03:00 committed by GitHub
parent e58c8ca638
commit 09c63a178b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 18 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
scrapy/core/engine.py
View File

@ -5,7 +5,6 @@ For more information see docs/topics/architecture.rst
"""
import logging
import warnings
from time import time
from typing import (
TYPE_CHECKING,
@ -14,7 +13,6 @@ from typing import (
Generator,
Iterable,
Iterator,
List,
Optional,
Set,
Type,
@ -29,7 +27,7 @@ from twisted.python.failure import Failure
from scrapy import signals
from scrapy.core.downloader import Downloader
from scrapy.core.scraper import Scraper
from scrapy.exceptions import CloseSpider, DontCloseSpider, ScrapyDeprecationWarning
from scrapy.exceptions import CloseSpider, DontCloseSpider
from scrapy.http import Request, Response
from scrapy.logformatter import LogFormatter
from scrapy.settings import BaseSettings, Settings
@ -213,7 +211,7 @@ class ExecutionEngine:
if request is None:
return None
d = self._download(request, self.spider)
d = self._download(request)
d.addBoth(self._handle_downloader_output, request)
d.addErrback(
lambda f: logger.info(
@ -266,13 +264,7 @@ class ExecutionEngine:
)
return d
def spider_is_idle(self, spider: Optional[Spider] = None) -> bool:
if spider is not None:
warnings.warn(
"Passing a 'spider' argument to ExecutionEngine.spider_is_idle is deprecated",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
def spider_is_idle(self) -> bool:
if self.slot is None:
raise RuntimeError("Engine slot not assigned")
if not self.scraper.slot.is_idle(): # type: ignore[union-attr]
@ -285,18 +277,8 @@ class ExecutionEngine:
return False
return True
def crawl(self, request: Request, spider: Optional[Spider] = None) -> None:
def crawl(self, request: Request) -> None:
"""Inject the request into the spider <-> downloader pipeline"""
if spider is not None:
warnings.warn(
"Passing a 'spider' argument to ExecutionEngine.crawl is deprecated",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
if spider is not self.spider:
raise RuntimeError(
f"The spider {spider.name!r} does not match the open spider"
)
if self.spider is None:
raise RuntimeError(f"No open spider to crawl: {request}")
self._schedule_request(request, self.spider)
@ -311,39 +293,24 @@ class ExecutionEngine:
signals.request_dropped, request=request, spider=spider
)
def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred:
def download(self, request: Request) -> Deferred:
"""Return a Deferred which fires with a Response as result, only downloader middlewares are applied"""
if spider is not None:
warnings.warn(
"Passing a 'spider' argument to ExecutionEngine.download is deprecated",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
if spider is not self.spider:
logger.warning(
"The spider '%s' does not match the open spider", spider.name
)
if self.spider is None:
raise RuntimeError(f"No open spider to crawl: {request}")
return self._download(request, spider).addBoth(
self._downloaded, request, spider
)
return self._download(request).addBoth(self._downloaded, request)
def _downloaded(
self, result: Union[Response, Request], request: Request, spider: Spider
self, result: Union[Response, Request], request: Request
) -> Union[Deferred, Response]:
assert self.slot is not None # typing
self.slot.remove_request(request)
return self.download(result, spider) if isinstance(result, Request) else result
return self.download(result) if isinstance(result, Request) else result
def _download(self, request: Request, spider: Optional[Spider]) -> Deferred:
def _download(self, request: Request) -> Deferred:
assert self.slot is not None # typing
self.slot.add_request(request)
if spider is None:
spider = self.spider
def _on_success(result: Union[Response, Request]) -> Union[Response, Request]:
if not isinstance(result, (Response, Request)):
raise TypeError(
@ -352,15 +319,17 @@ class ExecutionEngine:
if isinstance(result, Response):
if result.request is None:
result.request = request
assert spider is not None
logkws = self.logformatter.crawled(result.request, result, spider)
assert self.spider is not None
logkws = self.logformatter.crawled(result.request, result, self.spider)
if logkws is not None:
logger.log(*logformatter_adapter(logkws), extra={"spider": spider})
logger.log(
*logformatter_adapter(logkws), extra={"spider": self.spider}
)
self.signals.send_catch_log(
signal=signals.response_received,
response=result,
request=result.request,
spider=spider,
spider=self.spider,
)
return result
@ -369,8 +338,8 @@ class ExecutionEngine:
self.slot.nextcall.schedule()
return _
assert spider is not None
dwld = self.downloader.fetch(request, spider)
assert self.spider is not None
dwld = self.downloader.fetch(request, self.spider)
dwld.addCallbacks(_on_success)
dwld.addBoth(_on_complete)
return dwld
@ -485,31 +454,3 @@ class ExecutionEngine:
dfd.addBoth(lambda _: self._spider_closed_callback(spider))
return dfd
@property
def open_spiders(self) -> List[Spider]:
warnings.warn(
"ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
return [self.spider] if self.spider is not None else []
def has_capacity(self) -> bool:
warnings.warn(
"ExecutionEngine.has_capacity is deprecated",
ScrapyDeprecationWarning,
stacklevel=2,
)
return not bool(self.slot)
def schedule(self, request: Request, spider: Spider) -> None:
warnings.warn(
"ExecutionEngine.schedule is deprecated, please use "
"ExecutionEngine.crawl or ExecutionEngine.download instead",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
if self.slot is None:
raise RuntimeError("Engine slot not assigned")
self._schedule_request(request, spider)

87
tests/test_engine.py
View File

@ -20,7 +20,6 @@ from threading import Timer
from urllib.parse import urlparse
import attr
import pytest
from itemadapter import ItemAdapter
from pydispatch import dispatcher
from twisted.internet import defer, reactor
@ -29,7 +28,7 @@ from twisted.web import server, static, util
from scrapy import signals
from scrapy.core.engine import ExecutionEngine
from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning
from scrapy.exceptions import CloseSpider
from scrapy.http import Request
from scrapy.item import Field, Item
from scrapy.linkextractors import LinkExtractor
@ -436,90 +435,6 @@ class EngineTest(unittest.TestCase):
finally:
yield e.stop()
@defer.inlineCallbacks
def test_close_spiders_downloader(self):
with pytest.warns(
ScrapyDeprecationWarning,
match="ExecutionEngine.open_spiders is deprecated, "
"please use ExecutionEngine.spider instead",
):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
yield e.open_spider(TestSpider(), [])
self.assertEqual(len(e.open_spiders), 1)
yield e.close()
self.assertEqual(len(e.open_spiders), 0)
@defer.inlineCallbacks
def test_close_engine_spiders_downloader(self):
with pytest.warns(
ScrapyDeprecationWarning,
match="ExecutionEngine.open_spiders is deprecated, "
"please use ExecutionEngine.spider instead",
):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
yield e.open_spider(TestSpider(), [])
e.start()
self.assertTrue(e.running)
yield e.close()
self.assertFalse(e.running)
self.assertEqual(len(e.open_spiders), 0)
@defer.inlineCallbacks
def test_crawl_deprecated_spider_arg(self):
with pytest.warns(
ScrapyDeprecationWarning,
match="Passing a 'spider' argument to "
"ExecutionEngine.crawl is deprecated",
):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
spider = TestSpider()
yield e.open_spider(spider, [])
e.start()
e.crawl(Request("data:,"), spider)
yield e.close()
@defer.inlineCallbacks
def test_download_deprecated_spider_arg(self):
with pytest.warns(
ScrapyDeprecationWarning,
match="Passing a 'spider' argument to "
"ExecutionEngine.download is deprecated",
):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
spider = TestSpider()
yield e.open_spider(spider, [])
e.start()
e.download(Request("data:,"), spider)
yield e.close()
@defer.inlineCallbacks
def test_deprecated_schedule(self):
with pytest.warns(
ScrapyDeprecationWarning,
match="ExecutionEngine.schedule is deprecated, please use "
"ExecutionEngine.crawl or ExecutionEngine.download instead",
):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
spider = TestSpider()
yield e.open_spider(spider, [])
e.start()
e.schedule(Request("data:,"), spider)
yield e.close()
@defer.inlineCallbacks
def test_deprecated_has_capacity(self):
with pytest.warns(
ScrapyDeprecationWarning, match="ExecutionEngine.has_capacity is deprecated"
):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
self.assertTrue(e.has_capacity())
spider = TestSpider()
yield e.open_spider(spider, [])
self.assertFalse(e.has_capacity())
e.start()
yield e.close()
self.assertTrue(e.has_capacity())
def test_short_timeout(self):
args = (
sys.executable,